From 1d9ab19914d3a61c5721e6fa4cae3fb81532bdef Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Tue, 9 Jun 2020 16:13:18 -0700 Subject: [PATCH 01/15] Support IPFIX flow records for flow exporter feature: Added support to export IPFIX flow records that are built from connection map using IPFIX library. Unit tests are added. Did testing with ipfix collector in local k8s cluster running iperf service consisting tcp client and server. Run with elastiflow in progress. Issue# 712 --- build/yamls/base/conf/antrea-agent.conf | 4 + cmd/antrea-agent/agent.go | 22 +- cmd/antrea-agent/config.go | 4 + cmd/antrea-agent/options.go | 18 +- go.mod | 4 +- go.sum | 10 +- hack/update-codegen-dockerized.sh | 1 + .../flowexporter/connections/connections.go | 33 +++ pkg/agent/flowexporter/exporter/exporter.go | 270 ++++++++++++++++++ .../flowexporter/exporter/exporter_test.go | 171 +++++++++++ .../flowexporter/flowrecords/flowrecords.go | 79 +++++ pkg/agent/flowexporter/ipfix/ipfixprocess.go | 82 ++++++ pkg/agent/flowexporter/ipfix/ipfixrecord.go | 78 +++++ .../flowexporter/ipfix/testing/mock_ipfix.go | 230 +++++++++++++++ pkg/agent/flowexporter/types.go | 15 +- plugins/octant/go.sum | 1 + 16 files changed, 1013 insertions(+), 9 deletions(-) create mode 100644 pkg/agent/flowexporter/exporter/exporter.go create mode 100644 pkg/agent/flowexporter/exporter/exporter_test.go create mode 100644 pkg/agent/flowexporter/flowrecords/flowrecords.go create mode 100644 pkg/agent/flowexporter/ipfix/ipfixprocess.go create mode 100644 pkg/agent/flowexporter/ipfix/ipfixrecord.go create mode 100644 pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 50412a91ea5..0e6110f90e9 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -60,3 +60,7 @@ featureGates: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + +# Provide flow collector address as string with format IP:port. This also enables flow exporter that sends IPFIX +# flow records of conntrack flows on OVS bridge. +#flowCollectorAddr: "" diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 85905ff39f8..5cd94724644 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -31,6 +31,8 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute" "github.com/vmware-tanzu/antrea/pkg/agent/controller/traceflow" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/exporter" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" "github.com/vmware-tanzu/antrea/pkg/agent/metrics" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" @@ -234,11 +236,23 @@ func run(o *Options) error { if features.DefaultFeatureGate.Enabled(features.Traceflow) { go ofClient.StartPacketInHandler(stopCh) } - // Create connection store that polls conntrack flows with a given polling interval. + + // Initialize flow exporter; start go routines to poll conntrack flows and export IPFIX flow records if features.DefaultFeatureGate.Enabled(features.FlowExporter) { - ctDumper := connections.NewConnTrackDumper(nodeConfig, serviceCIDRNet, connections.NewConnTrackInterfacer()) - connStore := connections.NewConnectionStore(ctDumper, ifaceStore) - go connStore.Run(stopCh) + if o.flowCollector != nil { + ctDumper := connections.NewConnTrackDumper(nodeConfig, serviceCIDRNet, connections.NewConnTrackInterfacer()) + connStore := connections.NewConnectionStore(ctDumper, ifaceStore) + flowRecords := flowrecords.NewFlowRecords(connStore) + flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords) + if err != nil { + // Antrea agent do not exit, if flow exporter cannot be initialized. + // Currently, only logging the error. + klog.Errorf("error when initializing flow exporter: %v", err) + } else { + go connStore.Run(stopCh) + go flowExporter.Run(stopCh) + } + } } <-stopCh diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 672f7a2d790..36865938623 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -86,4 +86,8 @@ type AgentConfig struct { // Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener // Defaults to false. EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"` + // Provide flow collector address as string with format IP:port. This also enables flow exporter that sends IPFIX + // flow records of conntrack flows on OVS bridge. + // Defaults to "". + FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index b8203ac43fa..700914c5db4 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -42,6 +42,8 @@ type Options struct { configFile string // The configuration object config *AgentConfig + // IPFIX flow collector + flowCollector net.Addr } func newOptions() *Options { @@ -95,8 +97,20 @@ func (o *Options) validate(args []string) error { if encapMode.SupportsNoEncap() && o.config.EnableIPSecTunnel { return fmt.Errorf("IPSec tunnel may only be enabled on %s mode", config.TrafficEncapModeEncap) } - if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev && features.DefaultFeatureGate.Enabled(features.FlowExporter) { - return fmt.Errorf("FlowExporter feature is not supported for OVS datapath type %s", o.config.OVSDatapathType) + if o.config.FlowCollectorAddr != "" && features.DefaultFeatureGate.Enabled(features.FlowExporter) { + if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { + return fmt.Errorf("exporting flows is not supported for OVS datapath type %s", o.config.OVSDatapathType) + } else { + // Convert the string input in net.Addr format + _, _, err := net.SplitHostPort(o.config.FlowCollectorAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector is given in invalid format. Error: %v", err) + } + o.flowCollector, err = net.ResolveTCPAddr("tcp", o.config.FlowCollectorAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector server over TCP proto is not resolved. Error: %v", err) + } + } } return nil } diff --git a/go.mod b/go.mod index d7bfe2122ab..f28e3c273ee 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/go-openapi/spec v0.19.3 github.com/goccy/go-graphviz v0.0.5 github.com/gogo/protobuf v1.3.1 - github.com/golang/mock v1.2.0 + github.com/golang/mock v1.4.3 github.com/golang/protobuf v1.3.2 github.com/google/uuid v1.1.1 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd @@ -35,6 +35,7 @@ require ( github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 + github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836 github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect github.com/stretchr/testify v1.5.1 github.com/ti-mo/conntrack v0.3.0 @@ -46,6 +47,7 @@ require ( golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/grpc v1.26.0 gopkg.in/yaml.v2 v2.2.8 + gotest.tools v2.2.0+incompatible k8s.io/api v0.18.4 k8s.io/apimachinery v0.18.4 k8s.io/apiserver v0.18.4 diff --git a/go.sum b/go.sum index de9713c18ea..a229f7d5b41 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -355,6 +356,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836 h1:DHd3ZLldmrmJK3xYUhfXsW0pqicrro9QNo2FxyoIMB4= +github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 h1:XklXvOrWxWCDX2n4vdEQWkjuIP820XD6C4kF0O0FzH4= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= @@ -483,6 +486,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -500,6 +504,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -549,6 +554,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -577,6 +583,8 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66 h1:Ly1Oxdu5p5ZFmiVT71LFgeZETvMfZ1iBIGeOenT2JeM= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7 h1:uuHDyjllyzRyCIvvn0OBjiRB0SgBZGqHNYAmjR7fO50= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/hack/update-codegen-dockerized.sh b/hack/update-codegen-dockerized.sh index 5b609a84eb3..c079a2a46b2 100755 --- a/hack/update-codegen-dockerized.sh +++ b/hack/update-codegen-dockerized.sh @@ -93,6 +93,7 @@ MOCKGEN_TARGETS=( "pkg/controller/querier ControllerQuerier" "pkg/querier AgentNetworkPolicyInfoQuerier" "pkg/agent/flowexporter/connections ConnTrackDumper,ConnTrackInterfacer" + "pkg/agent/flowexporter/ipfix IPFIXExportingProcess,IPFIXRecord" ) # Command mockgen does not automatically replace variable YEAR with current year diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index fb127db288c..0ffd82a45f5 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -29,6 +29,8 @@ var _ ConnectionStore = new(connectionStore) type ConnectionStore interface { Run(stopCh <-chan struct{}) + IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error + FlushConnectionStore() } type connectionStore struct { @@ -116,7 +118,25 @@ func (cs *connectionStore) getConnByKey(flowTuple flowexporter.ConnectionKey) (* return &conn, found } +func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error { + cs.mutex.Lock() + defer cs.mutex.Unlock() + + for k, v := range cs.connections { + cs.mutex.Unlock() + err := updateCallback(k, v) + if err != nil { + klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) + return err + } + klog.V(2).Infof("Flow record added or updated") + cs.mutex.Lock() + } + return nil +} + // poll returns number of filtered connections after poll cycle +// TODO: Optimize polling cycle--Only poll invalid/close connection during every poll. Poll established right before export func (cs *connectionStore) poll() (int, error) { klog.V(2).Infof("Polling conntrack") @@ -133,3 +153,16 @@ func (cs *connectionStore) poll() (int, error) { return len(filteredConns), nil } + +// FlushConnectionStore after each IPFIX export of flow records. +// Timed out conntrack connections will not be sent as IPFIX flow records. +// TODO: Enhance/optimize this logic. +func (cs *connectionStore) FlushConnectionStore() { + klog.Infof("Flushing connection map") + + cs.mutex.Lock() + defer cs.mutex.Unlock() + for conn := range cs.connections { + delete(cs.connections, conn) + } +} diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go new file mode 100644 index 00000000000..bbeb0934c35 --- /dev/null +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -0,0 +1,270 @@ +package exporter + +import ( + "fmt" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" + "hash/fnv" + "net" + "os" + "strings" + "time" + "unicode" + + ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" + "k8s.io/klog" + + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" +) + +var ( + IANAInfoElements = []string{ + "flowStartSeconds", + "flowEndSeconds", + "sourceIPv4Address", + "destinationIPv4Address", + "sourceTransportPort", + "destinationTransportPort", + "protocolIdentifier", + "packetTotalCount", + "octetTotalCount", + "packetDeltaCount", + "octetDeltaCount", + "reverse_PacketTotalCount", + "reverse_OctetTotalCount", + "reverse_PacketDeltaCount", + "reverse_OctetDeltaCount", + } + AntreaInfoElements = []string{ + "sourcePodName", + "sourcePodNamespace", + "destinationPodName", + "destinationPodNamespace", + } +) + +var _ FlowExporter = new(flowExporter) + +type FlowExporter interface { + Run(stopCh <-chan struct{}) +} + +type flowExporter struct { + flowRecords flowrecords.FlowRecords + process ipfix.IPFIXExportingProcess + elementsList []*ipfixentities.InfoElement + templateID uint16 +} + +func getNodeName() (string, error) { + const nodeNameEnvKey = "NODE_NAME" + nodeName := os.Getenv(nodeNameEnvKey) + if nodeName != "" { + return nodeName, nil + } + klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey) + var err error + nodeName, err = os.Hostname() + if err != nil { + return "", fmt.Errorf("failed to get local hostname: %v", err) + } + return nodeName, nil +} + +func genObservationID() (uint32, error) { + name, err := getNodeName() + if err != nil { + return 0, err + } + h := fnv.New32() + h.Write([]byte(name)) + return h.Sum32(), nil +} + +func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords) (*flowExporter, error) { + // Create IPFIX exporting expProcess and initialize registries and other related entities + obsID, err := genObservationID() + if err != nil { + return nil, fmt.Errorf("cannot generate obsID for IPFIX ipfixexport: %v", err) + } + + expProcess, err := ipfix.NewIPFIXExportingProcess(collector, obsID) + if err != nil { + return nil, fmt.Errorf("error while initializing IPFIX exporting expProcess: %v", err) + } + expProcess.LoadRegistries() + + flowExp := &flowExporter{ + records, + expProcess, + nil, + 0, + } + + flowExp.templateID = flowExp.process.AddTemplate() + templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(AntreaInfoElements)), flowExp.templateID) + + sentBytes, err := flowExp.sendTemplateRecord(templateRec) + if err != nil { + return nil, fmt.Errorf("error while creating and sending template record through IPFIX process: %v", err) + } + klog.V(2).Infof("Initialized flow exporter and sent %d bytes size of template record", sentBytes) + + return flowExp, nil +} + +func (exp *flowExporter) Run(stopCh <-chan struct{}) { + klog.Infof("Start exporting IPFIX flow records") + for { + select { + case <-stopCh: + exp.process.CloseConnToCollector() + break + case <-time.After(flowexporter.FlowExportInterval): + err := exp.flowRecords.BuildFlowRecords() + if err != nil { + klog.Errorf("Error when building flow records: %v", err) + return + } + err = exp.sendFlowRecords() + if err != nil { + klog.Errorf("Error when sending flow records: %v", err) + return + } + } + } +} + +func (exp *flowExporter) sendFlowRecords() error { + err := exp.flowRecords.IterateFlowRecordsWithSendCB(exp.sendDataRecord, exp.templateID) + if err != nil { + return fmt.Errorf("error in iterating flow records: %v", err) + } + return nil +} + +func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, error) { + // Initialize this every time new template is added + exp.elementsList = make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(AntreaInfoElements)) + // Add template header + _, err := templateRec.PrepareRecord() + if err != nil { + return 0, fmt.Errorf("error when writing template header: %v", err) + } + + for i, ie := range IANAInfoElements { + var element *ipfixentities.InfoElement + var err error + if !strings.Contains(ie, "reverse") { + element, err = exp.process.GetIANARegistryInfoElement(ie, false) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + } else { + split := strings.Split(ie, "_") + runeStr := []rune(split[1]) + runeStr[0] = unicode.ToLower(runeStr[0]) + element, err = exp.process.GetIANARegistryInfoElement(string(runeStr), true) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + } + _, err = templateRec.AddInfoElement(element, nil) + if err != nil { + // Add error interface to IPFIX library in future to avoid overloading of fmt.Errorf. + return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) + } + exp.elementsList[i] = element + } + + for i, ie := range AntreaInfoElements { + element, err := exp.process.GetAntreaRegistryInfoElement(ie, false) + if err != nil { + return 0, fmt.Errorf("information element %s is not present in Antrea registry", ie) + } + templateRec.AddInfoElement(element, nil) + exp.elementsList[i+len(IANAInfoElements)] = element + } + + sentBytes, err := exp.process.AddRecordAndSendMsg(ipfixentities.Template, templateRec.GetRecord()) + if err != nil { + return 0, fmt.Errorf("error in IPFIX exporting process when sending template record: %v", err) + } + + return sentBytes, nil +} + +func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowexporter.FlowRecord) error { + // Iterate over all infoElements in the list + for _, ie := range exp.elementsList { + var err error + switch ieName := ie.Name; ieName { + case "flowStartSeconds": + _, err = dataRec.AddInfoElement(ie, record.Conn.StartTime.Unix()) + case "flowEndSeconds": + _, err = dataRec.AddInfoElement(ie, record.Conn.StopTime.Unix()) + case "sourceIPv4Address": + _, err = dataRec.AddInfoElement(ie, record.Conn.TupleOrig.SourceAddress) + case "destinationIPv4Address": + _, err = dataRec.AddInfoElement(ie, record.Conn.TupleReply.SourceAddress) + case "sourceTransportPort": + _, err = dataRec.AddInfoElement(ie, record.Conn.TupleOrig.SourcePort) + case "destinationTransportPort": + _, err = dataRec.AddInfoElement(ie, record.Conn.TupleReply.SourcePort) + case "protocolIdentifier": + _, err = dataRec.AddInfoElement(ie, record.Conn.TupleOrig.Protocol) + case "packetTotalCount": + _, err = dataRec.AddInfoElement(ie, record.Conn.OriginalPackets) + case "octetTotalCount": + _, err = dataRec.AddInfoElement(ie, record.Conn.OriginalBytes) + case "packetDeltaCount": + deltaPkts := int(record.Conn.OriginalPackets) - int(record.PrevPackets) + if deltaPkts < 0 { + klog.Warningf("Delta packets is not expected to be negative: %d", deltaPkts) + } + _, err = dataRec.AddInfoElement(ie, uint64(deltaPkts)) + case "octetDeltaCount": + deltaBytes := int(record.Conn.OriginalBytes) - int(record.PrevBytes) + if deltaBytes < 0 { + klog.Warningf("Delta bytes is not expected to be negative: %d", deltaBytes) + } + _, err = dataRec.AddInfoElement(ie, uint64(deltaBytes)) + case "reverse_PacketTotalCount": + _, err = dataRec.AddInfoElement(ie, record.Conn.ReversePackets) + case "reverse_OctetTotalCount": + _, err = dataRec.AddInfoElement(ie, record.Conn.ReverseBytes) + case "reverse_PacketDeltaCount": + deltaPkts := int(record.Conn.ReversePackets) - int(record.PrevReversePackets) + if deltaPkts < 0 { + klog.Warningf("Delta packets is not expected to be negative: %d", deltaPkts) + } + _, err = dataRec.AddInfoElement(ie, uint64(deltaPkts)) + case "reverse_OctetDeltaCount": + deltaBytes := int(record.Conn.ReverseBytes) - int(record.PrevReverseBytes) + if deltaBytes < 0 { + klog.Warningf("Delta bytes is not expected to be negative: %d", deltaBytes) + } + _, err = dataRec.AddInfoElement(ie, uint64(deltaBytes)) + case "sourcePodNamespace": + _, err = dataRec.AddInfoElement(ie, record.Conn.SourcePodNamespace) + case "sourcePodName": + _, err = dataRec.AddInfoElement(ie, record.Conn.SourcePodName) + case "destinationPodNamespace": + _, err = dataRec.AddInfoElement(ie, record.Conn.DestinationPodNamespace) + case "destinationPodName": + _, err = dataRec.AddInfoElement(ie, record.Conn.DestinationPodName) + } + if err != nil { + return fmt.Errorf("error while adding info element: %s to data record: %v", ie.Name, err) + } + } + klog.V(2).Infof("Flow data record created. Number of fields: %d, Bytes added: %d", dataRec.GetFieldCount(), dataRec.GetBuffer().Len()) + + sentBytes, err := exp.process.AddRecordAndSendMsg(ipfixentities.Data, dataRec.GetRecord()) + if err != nil { + return fmt.Errorf("error in IPFIX exporting process when sending data record: %v", err) + } + klog.V(2).Infof("Flow record sent successfully. Bytes sent: %d", sentBytes) + + return nil +} diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go new file mode 100644 index 00000000000..6a88b6c791a --- /dev/null +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -0,0 +1,171 @@ +package exporter + +import ( + "bytes" + "strings" + "testing" + "time" + "unicode" + + "github.com/golang/mock/gomock" + "gotest.tools/assert" + + ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + ipfixtest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix/testing" +) + +func TestFlowExporter_sendTemplateRecord(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIPFIXExpProc := ipfixtest.NewMockIPFIXExportingProcess(ctrl) + mockTempRec := ipfixtest.NewMockIPFIXRecord(ctrl) + flowExp := &flowExporter{ + nil, + mockIPFIXExpProc, + nil, + 256, + } + // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) + // Need only element name and other are dummys + elemList := make([]*ipfixentities.InfoElement, 0) + for _, ie := range IANAInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) + } + for _, ie := range AntreaInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) + } + // Expect calls for different mock objects + tempBytes := uint16(0) + var templateRecord ipfixentities.Record + + mockTempRec.EXPECT().PrepareRecord().Return(tempBytes, nil) + for i, ie := range IANAInfoElements { + if !strings.Contains(ie, "reverse") { + mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(ie, false).Return(elemList[i], nil) + } else { + split := strings.Split(ie, "_") + runeStr := []rune(split[1]) + runeStr[0] = unicode.ToLower(runeStr[0]) + mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(string(runeStr), true).Return(elemList[i], nil) + } + mockTempRec.EXPECT().AddInfoElement(elemList[i], nil).Return(tempBytes, nil) + } + for i, ie := range AntreaInfoElements { + if !strings.Contains(ie, "reverse") { + mockIPFIXExpProc.EXPECT().GetAntreaRegistryInfoElement(ie, false).Return(elemList[i+len(IANAInfoElements)], nil) + } else { + split := strings.Split(ie, "_") + runeStr := []rune(split[1]) + runeStr[0] = unicode.ToLower(runeStr[0]) + mockIPFIXExpProc.EXPECT().GetAntreaRegistryInfoElement(string(runeStr), true).Return(elemList[i+len(IANAInfoElements)], nil) + } + mockTempRec.EXPECT().AddInfoElement(elemList[i+len(IANAInfoElements)], nil).Return(tempBytes, nil) + } + mockTempRec.EXPECT().GetRecord().Return(templateRecord) + // Passing 0 for sentBytes as it is not used anywhere in the test. In reality, this IPFIX message size + // for template record of above elements. + mockIPFIXExpProc.EXPECT().AddRecordAndSendMsg(ipfixentities.Template, templateRecord).Return(0, nil) + + _, err := flowExp.sendTemplateRecord(mockTempRec) + if err != nil { + t.Errorf("Error in sending templated record: %v", err) + } + + assert.Equal(t, len(IANAInfoElements)+len(AntreaInfoElements), len(flowExp.elementsList), "flowExp.elementsList and template record should have same number of elements") +} + +// TestFlowExporter_sendDataRecord tests essentially if element names in the switch-case matches globals +// IANAInfoElements and AntreaInfoElements. +func TestFlowExporter_sendDataRecord(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Values in the connection are not important. Initializing with 0s. + flow1 := flowexporter.Connection{ + StartTime: time.Time{}, + StopTime: time.Time{}, + OriginalPackets: 0, + OriginalBytes: 0, + ReversePackets: 0, + ReverseBytes: 0, + TupleOrig: flowexporter.Tuple{ + SourceAddress: nil, + DestinationAddress: nil, + Protocol: 0, + SourcePort: 0, + DestinationPort: 0, + }, + TupleReply: flowexporter.Tuple{ + SourceAddress: nil, + DestinationAddress: nil, + Protocol: 0, + SourcePort: 0, + DestinationPort: 0, + }, + SourcePodNamespace: "", + SourcePodName: "", + DestinationPodNamespace: "", + DestinationPodName: "", + } + record1 := flowexporter.FlowRecord{ + Conn: &flow1, + PrevPackets: 0, + PrevBytes: 0, + PrevReversePackets: 0, + PrevReverseBytes: 0, + } + // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) + // Need only element name and other are dummys + elemList := make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(AntreaInfoElements)) + for i, ie := range IANAInfoElements { + elemList[i] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) + } + for i, ie := range AntreaInfoElements { + elemList[i+len(IANAInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) + } + mockIPFIXExpProc := ipfixtest.NewMockIPFIXExportingProcess(ctrl) + mockDataRec := ipfixtest.NewMockIPFIXRecord(ctrl) + flowExp := &flowExporter{ + nil, + mockIPFIXExpProc, + elemList, + 256, + } + // Expect calls required + var dataRecord ipfixentities.Record + var dataBuff bytes.Buffer + tempBytes := uint16(0) + for i, ie := range flowExp.elementsList { + // Could not come up with a way to exclude if else conditions as different IEs have different data types. + if i == 0 || i == 1 { + // For time elements + mockDataRec.EXPECT().AddInfoElement(ie, record1.Conn.StartTime.Unix()).Return(tempBytes, nil) + } else if i == 2 || i == 3 { + // For IP addresses + mockDataRec.EXPECT().AddInfoElement(ie, nil).Return(tempBytes, nil) + } else if i == 4 || i == 5 { + // For transport ports + mockDataRec.EXPECT().AddInfoElement(ie, uint16(0)).Return(tempBytes, nil) + } else if i == 6 { + // For proto identifier + mockDataRec.EXPECT().AddInfoElement(ie, uint8(0)).Return(tempBytes, nil) + } else if i >= 7 && i < 15 { + // For packets and octets + mockDataRec.EXPECT().AddInfoElement(ie, uint64(0)).Return(tempBytes, nil) + } else { + // For string elements + mockDataRec.EXPECT().AddInfoElement(ie, "").Return(tempBytes, nil) + } + } + mockDataRec.EXPECT().GetFieldCount().Return(uint16(len(flowExp.elementsList))) + mockDataRec.EXPECT().GetBuffer().Return(&dataBuff) + mockDataRec.EXPECT().GetRecord().Return(dataRecord) + mockIPFIXExpProc.EXPECT().AddRecordAndSendMsg(ipfixentities.Data, dataRecord).Return(0, nil) + + err := flowExp.sendDataRecord(mockDataRec, record1) + if err != nil { + t.Errorf("Error in sending data record: %v", err) + } +} diff --git a/pkg/agent/flowexporter/flowrecords/flowrecords.go b/pkg/agent/flowexporter/flowrecords/flowrecords.go new file mode 100644 index 00000000000..b124fd2a7cc --- /dev/null +++ b/pkg/agent/flowexporter/flowrecords/flowrecords.go @@ -0,0 +1,79 @@ +package flowrecords + +import ( + "fmt" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" + + "k8s.io/klog" + + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections" +) + +var _ FlowRecords = new(flowRecords) + +type FlowRecords interface { + BuildFlowRecords() error + IterateFlowRecordsWithSendCB(callback flowexporter.FlowRecordSend, templateID uint16) error +} + +type flowRecords struct { + // synchronization is not required as there is no concurrency involving this object. + // Add lock when this is consumed by more than one entity concurrently. + recordsMap map[flowexporter.ConnectionKey]flowexporter.FlowRecord + connStoreBuilder connections.ConnectionStore +} + +func NewFlowRecords(connStore connections.ConnectionStore) *flowRecords { + return &flowRecords{ + make(map[flowexporter.ConnectionKey]flowexporter.FlowRecord), + connStore, + } +} + +func (fr *flowRecords) BuildFlowRecords() error { + err := fr.connStoreBuilder.IterateCxnMapWithCB(fr.addOrUpdateFlowRecord) + if err != nil { + return fmt.Errorf("error in iterating cxn map: %v", err) + } + klog.V(2).Infof("Flow records that are built: %d", len(fr.recordsMap)) + return nil +} + +func (fr *flowRecords) IterateFlowRecordsWithSendCB(sendCallback flowexporter.FlowRecordSend, templateID uint16) error { + for k, v := range fr.recordsMap { + dataRec := ipfix.NewIPFIXDataRecord(templateID) + err := sendCallback(dataRec, v) + if err != nil { + klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) + return err + } + // Update the flow record after it is sent successfully + v.PrevPackets = v.Conn.OriginalPackets + v.PrevBytes = v.Conn.OriginalBytes + v.PrevReversePackets = v.Conn.ReversePackets + v.PrevReverseBytes = v.Conn.ReverseBytes + fr.recordsMap[k] = v + klog.V(2).Infof("Flow record sent successfully") + } + + return nil +} + +func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { + record, exists := fr.recordsMap[key] + if !exists { + record = flowexporter.FlowRecord{ + &conn, + 0, + 0, + 0, + 0, + } + } else { + record.Conn = &conn + } + fr.recordsMap[key] = record + klog.V(2).Infof("Flow record added or updated: %v", record) + return nil +} diff --git a/pkg/agent/flowexporter/ipfix/ipfixprocess.go b/pkg/agent/flowexporter/ipfix/ipfixprocess.go new file mode 100644 index 00000000000..90093725894 --- /dev/null +++ b/pkg/agent/flowexporter/ipfix/ipfixprocess.go @@ -0,0 +1,82 @@ +package ipfix + +import ( + "fmt" + "net" + + ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" + ipfixexport "github.com/srikartati/go-ipfixlib/pkg/exporter" + ipfixregistry "github.com/srikartati/go-ipfixlib/pkg/registry" +) + +var _ IPFIXExportingProcess = new(ipfixExportingProcess) + +type IPFIXExportingProcess interface { + LoadRegistries() + GetIANARegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) + GetAntreaRegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) + AddTemplate() uint16 + AddRecordAndSendMsg(setType ipfixentities.ContentType, record ipfixentities.Record) (int, error) + CloseConnToCollector() error +} + +type ipfixExportingProcess struct { + *ipfixexport.ExportingProcess + ianaReg ipfixregistry.Registry + antreaReg ipfixregistry.Registry +} + +func NewIPFIXExportingProcess(collector net.Addr, obsID uint32) (*ipfixExportingProcess, error) { + expProcess, err := ipfixexport.InitExportingProcess(collector, obsID) + if err != nil { + return nil, fmt.Errorf("error while initializing IPFIX exporting process: %v", err) + } + + return &ipfixExportingProcess{ + ExportingProcess: expProcess, + }, nil +} + +func (exp *ipfixExportingProcess) AddRecordAndSendMsg(setType ipfixentities.ContentType, record ipfixentities.Record) (int, error) { + sentBytes, err := exp.ExportingProcess.AddRecordAndSendMsg(setType, record) + return sentBytes, err +} + +func (exp *ipfixExportingProcess) CloseConnToCollector() error { + err := exp.ExportingProcess.CloseConnToCollector() + return err +} + +func (exp *ipfixExportingProcess) LoadRegistries() { + exp.ianaReg = ipfixregistry.NewIanaRegistry() + exp.ianaReg.LoadRegistry() + exp.antreaReg = ipfixregistry.NewAntreaRegistry() + exp.antreaReg.LoadRegistry() + return +} + +func (exp *ipfixExportingProcess) GetIANARegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) { + var ie *ipfixentities.InfoElement + var err error + if !isReverse { + ie, err = exp.ianaReg.GetInfoElement(name) + } else { + ie, err = exp.ianaReg.GetReverseInfoElement(name) + } + return ie, err +} + +func (exp *ipfixExportingProcess) GetAntreaRegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) { + var ie *ipfixentities.InfoElement + var err error + if !isReverse { + ie, err = exp.antreaReg.GetInfoElement(name) + } else { + ie, err = exp.antreaReg.GetReverseInfoElement(name) + } + return ie, err +} + +func (exp *ipfixExportingProcess) AddTemplate() uint16 { + return exp.ExportingProcess.AddTemplate() +} diff --git a/pkg/agent/flowexporter/ipfix/ipfixrecord.go b/pkg/agent/flowexporter/ipfix/ipfixrecord.go new file mode 100644 index 00000000000..ebb6708db1d --- /dev/null +++ b/pkg/agent/flowexporter/ipfix/ipfixrecord.go @@ -0,0 +1,78 @@ +package ipfix + +import ( + "bytes" + ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" +) + +var _ IPFIXRecord = new(ipfixDataRecord) +var _ IPFIXRecord = new(ipfixTemplateRecord) + +type IPFIXRecord interface { + GetRecord() ipfixentities.Record + PrepareRecord() (uint16, error) + AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) + GetBuffer() *bytes.Buffer + GetFieldCount() uint16 +} + +type ipfixDataRecord struct { + dataRecord ipfixentities.Record +} + +type ipfixTemplateRecord struct { + templateRecord ipfixentities.Record +} + +func NewIPFIXDataRecord(tempID uint16) *ipfixDataRecord { + dr := ipfixentities.NewDataRecord(tempID) + return &ipfixDataRecord{dataRecord: dr} +} +func NewIPFIXTemplateRecord(elementCount uint16, tempID uint16) *ipfixTemplateRecord { + tr := ipfixentities.NewTemplateRecord(elementCount, tempID) + return &ipfixTemplateRecord{templateRecord: tr} +} + +func (dr *ipfixDataRecord) GetRecord() ipfixentities.Record { + return dr.dataRecord +} + +func (dr *ipfixDataRecord) PrepareRecord() (uint16, error) { + addedBytes, err := dr.dataRecord.PrepareRecord() + return addedBytes, err +} + +func (dr *ipfixDataRecord) AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) { + addedBytes, err := dr.dataRecord.AddInfoElement(element, val) + return addedBytes, err +} + +func (dr *ipfixDataRecord) GetBuffer() *bytes.Buffer { + return dr.dataRecord.GetBuffer() +} + +func (dr *ipfixDataRecord) GetFieldCount() uint16 { + return dr.dataRecord.GetFieldCount() +} + +func (tr *ipfixTemplateRecord) GetRecord() ipfixentities.Record { + return tr.templateRecord +} + +func (tr *ipfixTemplateRecord) PrepareRecord() (uint16, error) { + addedBytes, err := tr.templateRecord.PrepareRecord() + return addedBytes, err +} + +func (tr *ipfixTemplateRecord) AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) { + addedBytes, err := tr.templateRecord.AddInfoElement(element, val) + return addedBytes, err +} + +func (tr *ipfixTemplateRecord) GetBuffer() *bytes.Buffer { + return tr.templateRecord.GetBuffer() +} + +func (tr *ipfixTemplateRecord) GetFieldCount() uint16 { + return tr.templateRecord.GetFieldCount() +} diff --git a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go new file mode 100644 index 00000000000..a966c4b1b3f --- /dev/null +++ b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go @@ -0,0 +1,230 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix (interfaces: IPFIXExportingProcess,IPFIXRecord) + +// Package testing is a generated GoMock package. +package testing + +import ( + bytes "bytes" + gomock "github.com/golang/mock/gomock" + entities "github.com/srikartati/go-ipfixlib/pkg/entities" + reflect "reflect" +) + +// MockIPFIXExportingProcess is a mock of IPFIXExportingProcess interface +type MockIPFIXExportingProcess struct { + ctrl *gomock.Controller + recorder *MockIPFIXExportingProcessMockRecorder +} + +// MockIPFIXExportingProcessMockRecorder is the mock recorder for MockIPFIXExportingProcess +type MockIPFIXExportingProcessMockRecorder struct { + mock *MockIPFIXExportingProcess +} + +// NewMockIPFIXExportingProcess creates a new mock instance +func NewMockIPFIXExportingProcess(ctrl *gomock.Controller) *MockIPFIXExportingProcess { + mock := &MockIPFIXExportingProcess{ctrl: ctrl} + mock.recorder = &MockIPFIXExportingProcessMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockIPFIXExportingProcess) EXPECT() *MockIPFIXExportingProcessMockRecorder { + return m.recorder +} + +// AddRecordAndSendMsg mocks base method +func (m *MockIPFIXExportingProcess) AddRecordAndSendMsg(arg0 entities.ContentType, arg1 entities.Record) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddRecordAndSendMsg", arg0, arg1) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddRecordAndSendMsg indicates an expected call of AddRecordAndSendMsg +func (mr *MockIPFIXExportingProcessMockRecorder) AddRecordAndSendMsg(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRecordAndSendMsg", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).AddRecordAndSendMsg), arg0, arg1) +} + +// AddTemplate mocks base method +func (m *MockIPFIXExportingProcess) AddTemplate() uint16 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddTemplate") + ret0, _ := ret[0].(uint16) + return ret0 +} + +// AddTemplate indicates an expected call of AddTemplate +func (mr *MockIPFIXExportingProcessMockRecorder) AddTemplate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTemplate", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).AddTemplate)) +} + +// CloseConnToCollector mocks base method +func (m *MockIPFIXExportingProcess) CloseConnToCollector() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloseConnToCollector") + ret0, _ := ret[0].(error) + return ret0 +} + +// CloseConnToCollector indicates an expected call of CloseConnToCollector +func (mr *MockIPFIXExportingProcessMockRecorder) CloseConnToCollector() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseConnToCollector", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).CloseConnToCollector)) +} + +// GetAntreaRegistryInfoElement mocks base method +func (m *MockIPFIXExportingProcess) GetAntreaRegistryInfoElement(arg0 string, arg1 bool) (*entities.InfoElement, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAntreaRegistryInfoElement", arg0, arg1) + ret0, _ := ret[0].(*entities.InfoElement) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAntreaRegistryInfoElement indicates an expected call of GetAntreaRegistryInfoElement +func (mr *MockIPFIXExportingProcessMockRecorder) GetAntreaRegistryInfoElement(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAntreaRegistryInfoElement", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).GetAntreaRegistryInfoElement), arg0, arg1) +} + +// GetIANARegistryInfoElement mocks base method +func (m *MockIPFIXExportingProcess) GetIANARegistryInfoElement(arg0 string, arg1 bool) (*entities.InfoElement, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIANARegistryInfoElement", arg0, arg1) + ret0, _ := ret[0].(*entities.InfoElement) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIANARegistryInfoElement indicates an expected call of GetIANARegistryInfoElement +func (mr *MockIPFIXExportingProcessMockRecorder) GetIANARegistryInfoElement(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIANARegistryInfoElement", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).GetIANARegistryInfoElement), arg0, arg1) +} + +// LoadRegistries mocks base method +func (m *MockIPFIXExportingProcess) LoadRegistries() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "LoadRegistries") +} + +// LoadRegistries indicates an expected call of LoadRegistries +func (mr *MockIPFIXExportingProcessMockRecorder) LoadRegistries() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRegistries", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).LoadRegistries)) +} + +// MockIPFIXRecord is a mock of IPFIXRecord interface +type MockIPFIXRecord struct { + ctrl *gomock.Controller + recorder *MockIPFIXRecordMockRecorder +} + +// MockIPFIXRecordMockRecorder is the mock recorder for MockIPFIXRecord +type MockIPFIXRecordMockRecorder struct { + mock *MockIPFIXRecord +} + +// NewMockIPFIXRecord creates a new mock instance +func NewMockIPFIXRecord(ctrl *gomock.Controller) *MockIPFIXRecord { + mock := &MockIPFIXRecord{ctrl: ctrl} + mock.recorder = &MockIPFIXRecordMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockIPFIXRecord) EXPECT() *MockIPFIXRecordMockRecorder { + return m.recorder +} + +// AddInfoElement mocks base method +func (m *MockIPFIXRecord) AddInfoElement(arg0 *entities.InfoElement, arg1 interface{}) (uint16, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddInfoElement", arg0, arg1) + ret0, _ := ret[0].(uint16) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddInfoElement indicates an expected call of AddInfoElement +func (mr *MockIPFIXRecordMockRecorder) AddInfoElement(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddInfoElement", reflect.TypeOf((*MockIPFIXRecord)(nil).AddInfoElement), arg0, arg1) +} + +// GetBuffer mocks base method +func (m *MockIPFIXRecord) GetBuffer() *bytes.Buffer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBuffer") + ret0, _ := ret[0].(*bytes.Buffer) + return ret0 +} + +// GetBuffer indicates an expected call of GetBuffer +func (mr *MockIPFIXRecordMockRecorder) GetBuffer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBuffer", reflect.TypeOf((*MockIPFIXRecord)(nil).GetBuffer)) +} + +// GetFieldCount mocks base method +func (m *MockIPFIXRecord) GetFieldCount() uint16 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFieldCount") + ret0, _ := ret[0].(uint16) + return ret0 +} + +// GetFieldCount indicates an expected call of GetFieldCount +func (mr *MockIPFIXRecordMockRecorder) GetFieldCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFieldCount", reflect.TypeOf((*MockIPFIXRecord)(nil).GetFieldCount)) +} + +// GetRecord mocks base method +func (m *MockIPFIXRecord) GetRecord() entities.Record { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRecord") + ret0, _ := ret[0].(entities.Record) + return ret0 +} + +// GetRecord indicates an expected call of GetRecord +func (mr *MockIPFIXRecordMockRecorder) GetRecord() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRecord", reflect.TypeOf((*MockIPFIXRecord)(nil).GetRecord)) +} + +// PrepareRecord mocks base method +func (m *MockIPFIXRecord) PrepareRecord() (uint16, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PrepareRecord") + ret0, _ := ret[0].(uint16) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PrepareRecord indicates an expected call of PrepareRecord +func (mr *MockIPFIXRecordMockRecorder) PrepareRecord() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareRecord", reflect.TypeOf((*MockIPFIXRecord)(nil).PrepareRecord)) +} diff --git a/pkg/agent/flowexporter/types.go b/pkg/agent/flowexporter/types.go index ef981c40dbf..b9b34f4be19 100644 --- a/pkg/agent/flowexporter/types.go +++ b/pkg/agent/flowexporter/types.go @@ -15,16 +15,21 @@ package flowexporter import ( + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" "net" "time" ) const ( - PollInterval = 5 * time.Second + PollInterval = 5 * time.Second + FlowExportInterval = 120 * time.Second ) type ConnectionKey [5]string +type FlowRecordUpdate func(key ConnectionKey, cxn Connection) error +type FlowRecordSend func(dataRecord ipfix.IPFIXRecord, record FlowRecord) error + type Tuple struct { SourceAddress net.IP DestinationAddress net.IP @@ -52,3 +57,11 @@ type Connection struct { DestinationPodNamespace string DestinationPodName string } + +type FlowRecord struct { + Conn *Connection + PrevPackets uint64 + PrevBytes uint64 + PrevReversePackets uint64 + PrevReverseBytes uint64 +} diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index 282059f6bb5..0e9b5e08257 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -441,6 +441,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 628ef8f3f42534709e5bde856dff88b2c5113d6c Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Wed, 17 Jun 2020 11:22:53 -0700 Subject: [PATCH 02/15] Support UDP transport to send IPFIX flow records --- build/yamls/base/conf/antrea-agent.conf | 5 +-- cmd/antrea-agent/config.go | 4 +-- cmd/antrea-agent/options.go | 35 ++++++++++++++++++--- pkg/agent/flowexporter/exporter/exporter.go | 6 ++-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 0e6110f90e9..d4a2dd3f247 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -61,6 +61,7 @@ featureGates: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false -# Provide flow collector address as string with format IP:port. This also enables flow exporter that sends IPFIX -# flow records of conntrack flows on OVS bridge. +# Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX +# flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. +# Defaults to "". #flowCollectorAddr: "" diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 36865938623..f644cbe6ebd 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -86,8 +86,8 @@ type AgentConfig struct { // Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener // Defaults to false. EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"` - // Provide flow collector address as string with format IP:port. This also enables flow exporter that sends IPFIX - // flow records of conntrack flows on OVS bridge. + // Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + // flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. // Defaults to "". FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 700914c5db4..3c7112a2487 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "net" + "strings" "github.com/spf13/pflag" "gopkg.in/yaml.v2" @@ -101,14 +102,38 @@ func (o *Options) validate(args []string) error { if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { return fmt.Errorf("exporting flows is not supported for OVS datapath type %s", o.config.OVSDatapathType) } else { + // Check if it is TCP or UDP + strSlice := strings.Split(o.config.FlowCollectorAddr, ":") + var proto string + if len(strSlice) == 2 { + // No separator "." and proto is given + proto = "tcp" + } else if len(strSlice) > 2 { + if strSlice[2] == "udp" { + proto = "udp" + } else { + // All other cases default proto is tcp + proto = "tcp" + } + } else { + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + } // Convert the string input in net.Addr format - _, _, err := net.SplitHostPort(o.config.FlowCollectorAddr) + hostPortAddr := strSlice[0]+":"+strSlice[1] + _, _, err := net.SplitHostPort(hostPortAddr) if err != nil { - return fmt.Errorf("IPFIX flow collector is given in invalid format. Error: %v", err) + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) } - o.flowCollector, err = net.ResolveTCPAddr("tcp", o.config.FlowCollectorAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector server over TCP proto is not resolved. Error: %v", err) + if proto == "udp" { + o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved. Error: %v", err) + } + } else { + o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved. Error: %v", err) + } } } } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index bbeb0934c35..79cbe523c36 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -124,12 +124,14 @@ func (exp *flowExporter) Run(stopCh <-chan struct{}) { err := exp.flowRecords.BuildFlowRecords() if err != nil { klog.Errorf("Error when building flow records: %v", err) - return + exp.process.CloseConnToCollector() + break } err = exp.sendFlowRecords() if err != nil { klog.Errorf("Error when sending flow records: %v", err) - return + exp.process.CloseConnToCollector() + break } } } From 2cc62b4635a0b894d2b1cfb3074e269caec87fe7 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Tue, 23 Jun 2020 15:51:44 -0700 Subject: [PATCH 03/15] Flow exporter feature Added e2e test that can run on both vagrant and kind clusters. ipfix collector is a light weight collector. Cleaned up and refactored code. Addressed review comments. Changed go.mod to pick latest go-ipfixlib --- build/images/ipfixcollector/Dockerfile | 20 ++ build/images/ipfixcollector/README.md | 18 ++ build/yamls/base/conf/antrea-agent.conf | 7 + cmd/antrea-agent/agent.go | 14 +- cmd/antrea-agent/config.go | 6 + cmd/antrea-agent/options.go | 88 +++--- go.mod | 2 +- go.sum | 4 +- .../flowexporter/connections/connections.go | 25 +- .../flowexporter/connections/conntrack.go | 42 +++ .../connections/conntrack_linux.go | 259 ++++++++++++++---- .../connections/conntrack_test.go | 121 ++++---- .../connections/conntrack_windows.go | 35 +-- .../flowexporter/connections/interface.go | 7 + .../connections/testing/mock_connections.go | 33 ++- pkg/agent/flowexporter/exporter/exporter.go | 50 ++-- .../flowexporter/exporter/exporter_test.go | 16 ++ .../flowexporter/flowrecords/flowrecords.go | 14 + pkg/agent/flowexporter/ipfix/ipfixprocess.go | 14 + pkg/agent/flowexporter/ipfix/ipfixrecord.go | 14 + pkg/agent/flowexporter/types.go | 6 +- pkg/agent/flowexporter/utils.go | 4 +- pkg/ovs/ovsctl/appctl.go | 11 +- pkg/ovs/ovsctl/interface.go | 3 + pkg/ovs/ovsctl/testing/mock_ovsctl.go | 20 ++ pkg/util/ip/ip.go | 22 +- plugins/octant/go.sum | 2 +- test/e2e/bandwidth_test.go | 8 +- test/e2e/fixtures.go | 40 +++ test/e2e/flowexporter_test.go | 135 +++++++++ test/e2e/framework.go | 63 ++++- test/e2e/providers/exec/docker.go | 8 +- 32 files changed, 851 insertions(+), 260 deletions(-) create mode 100644 build/images/ipfixcollector/Dockerfile create mode 100644 build/images/ipfixcollector/README.md create mode 100644 pkg/agent/flowexporter/connections/conntrack.go create mode 100644 test/e2e/flowexporter_test.go diff --git a/build/images/ipfixcollector/Dockerfile b/build/images/ipfixcollector/Dockerfile new file mode 100644 index 00000000000..93f387d90a0 --- /dev/null +++ b/build/images/ipfixcollector/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:18.04 + +LABEL maintainer="Antrea " +LABEL description="A Docker image based on Ubuntu 18.04 which contains simple IPFIX collector to run flow exporter tests" + +WORKDIR /ipfix + +ADD https://svwh.dl.sourceforge.net/project/libipfix/libipfix/libipfix-impd4e_110224.tgz . + +RUN apt-get update && \ + apt-get install -y --no-install-recommends libc6-dev build-essential libpcap0.8-dev && \ + tar -xvf libipfix-* && rm libipfix-* && \ + cd libipfix_* && ./configure && make && make install && ldconfig && \ + cp collector/ipfix_collector /usr/local/bin && \ + cd .. && \ + rm -rf libipfix_* && \ + apt-get remove -y build-essential && \ + rm -rf /var/cache/apt/* /var/lib/apt/lists/* + +ENTRYPOINT "ipfix_collector" \ No newline at end of file diff --git a/build/images/ipfixcollector/README.md b/build/images/ipfixcollector/README.md new file mode 100644 index 00000000000..0108b1a55be --- /dev/null +++ b/build/images/ipfixcollector/README.md @@ -0,0 +1,18 @@ +# images/ipfixcollector + +This Docker image is a very lightweight image based on Ubuntu 18.04 which +includes ipfix collector based on libipfix, a C library. +In this image, IPFIX collector listening on tcp:4739 port. + +libipfix package is downloaded from https://svwh.dl.sourceforge.net/project/libipfix/libipfix/libipfix-impd4e_110224.tgz + +New version of the image can be built and pushed to Dockerhub using following instructions: + +```bash +cd build/images/ipfixcollector +docker build -t antrea/ipfixcollector:latest . +docker push antrea/ipfixcollector:latest +``` + +The `docker push` command will fail if you do not have permission to push to the +`antrea` Dockerhub repository. diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index d4a2dd3f247..ce0ea29ba26 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -65,3 +65,10 @@ featureGates: # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. # Defaults to "". #flowCollectorAddr: "" + +# Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections +# in conntrack module and exports IPFIX flow records that are built from connection store. +# Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. +# Any value in range (PollInterval(s), 600s] for export interval is acceptable. +# Defaults to "5s:60s". Follow the time units of duration. +#pollAndExportInterval: "" diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 5cd94724644..4549947e143 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -240,13 +240,17 @@ func run(o *Options) error { // Initialize flow exporter; start go routines to poll conntrack flows and export IPFIX flow records if features.DefaultFeatureGate.Enabled(features.FlowExporter) { if o.flowCollector != nil { - ctDumper := connections.NewConnTrackDumper(nodeConfig, serviceCIDRNet, connections.NewConnTrackInterfacer()) - connStore := connections.NewConnectionStore(ctDumper, ifaceStore) + var connTrackDumper connections.ConnTrackDumper + if o.config.OVSDatapathType == ovsconfig.OVSDatapathSystem { + connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackSystem(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) + } else if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { + connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackNetdev(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) + } + connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval) flowRecords := flowrecords.NewFlowRecords(connStore) - flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords) + flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords, o.exportInterval) if err != nil { - // Antrea agent do not exit, if flow exporter cannot be initialized. - // Currently, only logging the error. + // If flow exporter cannot be initialized, then Antrea agent does not exit; only error is logged. klog.Errorf("error when initializing flow exporter: %v", err) } else { go connStore.Run(stopCh) diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index f644cbe6ebd..1e0e546739a 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -90,4 +90,10 @@ type AgentConfig struct { // flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. // Defaults to "". FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` + // Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + // in conntrack module and exports IPFIX flow records that are built from connection store. + // Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. + // Any value in range (PollInterval(s), 600s] for export interval is acceptable. + // Defaults to "5s:60s". Follow the time units of duration. + PollAndExportInterval string `yaml:"pollAndExportInterval,omitempty"` } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 3c7112a2487..535778d801d 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "net" "strings" + "time" "github.com/spf13/pflag" "gopkg.in/yaml.v2" @@ -45,6 +46,10 @@ type Options struct { config *AgentConfig // IPFIX flow collector flowCollector net.Addr + // Flow exporter polling interval + pollingInterval time.Duration + // Flow exporter export interval + exportInterval time.Duration } func newOptions() *Options { @@ -98,42 +103,56 @@ func (o *Options) validate(args []string) error { if encapMode.SupportsNoEncap() && o.config.EnableIPSecTunnel { return fmt.Errorf("IPSec tunnel may only be enabled on %s mode", config.TrafficEncapModeEncap) } - if o.config.FlowCollectorAddr != "" && features.DefaultFeatureGate.Enabled(features.FlowExporter) { - if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { - return fmt.Errorf("exporting flows is not supported for OVS datapath type %s", o.config.OVSDatapathType) - } else { - // Check if it is TCP or UDP - strSlice := strings.Split(o.config.FlowCollectorAddr, ":") - var proto string - if len(strSlice) == 2 { - // No separator "." and proto is given - proto = "tcp" - } else if len(strSlice) > 2 { - if strSlice[2] == "udp" { - proto = "udp" - } else { - // All other cases default proto is tcp - proto = "tcp" - } + if o.config.FlowCollectorAddr != "" && features.DefaultFeatureGate.Enabled(features.FlowExporter) { + // Check if it is TCP or UDP + strSlice := strings.Split(o.config.FlowCollectorAddr, ":") + var proto string + if len(strSlice) == 2 { + // No separator "." and proto is given + proto = "tcp" + } else if len(strSlice) > 2 { + if strSlice[2] == "udp" { + proto = "udp" } else { - return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + // All other cases default proto is tcp + proto = "tcp" } - // Convert the string input in net.Addr format - hostPortAddr := strSlice[0]+":"+strSlice[1] - _, _, err := net.SplitHostPort(hostPortAddr) + } else { + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + } + // Convert the string input in net.Addr format + hostPortAddr := strSlice[0] + ":" + strSlice[1] + _, _, err := net.SplitHostPort(hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + } + if proto == "udp" { + o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) if err != nil { - return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved: %v", err) } - if proto == "udp" { - o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved. Error: %v", err) - } - } else { - o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved. Error: %v", err) - } + } else { + o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved: %v", err) + } + } + + if o.config.PollAndExportInterval != "" { + intervalSlice := strings.Split(o.config.PollAndExportInterval, ":") + if len(intervalSlice) != 2 { + return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.PollAndExportInterval) + } + o.pollingInterval, err = time.ParseDuration(intervalSlice[0]) + if err != nil { + return fmt.Errorf("poll interval is not provided in right format: %v", err) + } + o.exportInterval, err = time.ParseDuration(intervalSlice[1]) + if err != nil { + return fmt.Errorf("export interval is not provided in right format: %v", err) + } + if o.pollingInterval > o.exportInterval { + return fmt.Errorf("poll interval should be less than or equal to export interval") } } } @@ -185,4 +204,9 @@ func (o *Options) setDefaults() { if o.config.APIPort == 0 { o.config.APIPort = apis.AntreaAgentAPIPort } + + if o.config.FlowCollectorAddr != "" && o.config.PollAndExportInterval == "" { + o.pollingInterval = 5 * time.Second + o.exportInterval = 60 * time.Second + } } diff --git a/go.mod b/go.mod index f28e3c273ee..d8979d6ced8 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836 + github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect github.com/stretchr/testify v1.5.1 github.com/ti-mo/conntrack v0.3.0 diff --git a/go.sum b/go.sum index a229f7d5b41..89ab336bcd0 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836 h1:DHd3ZLldmrmJK3xYUhfXsW0pqicrro9QNo2FxyoIMB4= -github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= +github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c h1:/yq/4iWgtsFZPe685ytq2OWtDNKWPT2MpDDUdP1oYdM= +github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 h1:XklXvOrWxWCDX2n4vdEQWkjuIP820XD6C4kF0O0FzH4= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 0ffd82a45f5..bd2edff2d74 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -34,17 +34,19 @@ type ConnectionStore interface { } type connectionStore struct { - connections map[flowexporter.ConnectionKey]flowexporter.Connection // Add 5-tuple as string array - connDumper ConnTrackDumper - ifaceStore interfacestore.InterfaceStore - mutex sync.Mutex + connections map[flowexporter.ConnectionKey]flowexporter.Connection // Add 5-tuple as string array + connDumper ConnTrackDumper + ifaceStore interfacestore.InterfaceStore + pollInterval time.Duration + mutex sync.Mutex } -func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore) *connectionStore { +func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, interval time.Duration) *connectionStore { return &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: ctDumper, - ifaceStore: ifaceStore, + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: ctDumper, + ifaceStore: ifaceStore, + pollInterval: interval, } } @@ -53,7 +55,7 @@ func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.Inte func (cs *connectionStore) Run(stopCh <-chan struct{}) { klog.Infof("Starting conntrack polling") - ticker := time.NewTicker(flowexporter.PollInterval) + ticker := time.NewTicker(cs.pollInterval) defer ticker.Stop() for { select { @@ -97,6 +99,7 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { if !srcFound && !dstFound { klog.Warningf("Cannot map any of the IP %s or %s to a local Pod", conn.TupleOrig.SourceAddress.String(), conn.TupleReply.SourceAddress.String()) } + // sourceIP/destinationIP are mapped only to local pods and not remote pods. if srcFound && sIface.Type == interfacestore.ContainerInterface { conn.SourcePodName = sIface.ContainerInterfaceConfig.PodName conn.SourcePodNamespace = sIface.ContainerInterfaceConfig.PodNamespace @@ -126,10 +129,9 @@ func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowR cs.mutex.Unlock() err := updateCallback(k, v) if err != nil { - klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) + klog.Errorf("Update callback failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) return err } - klog.V(2).Infof("Flow record added or updated") cs.mutex.Lock() } return nil @@ -142,7 +144,6 @@ func (cs *connectionStore) poll() (int, error) { filteredConns, err := cs.connDumper.DumpFlows(openflow.CtZone) if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) return 0, err } // Update only the Connection store. IPFIX records are generated based on Connection store. diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go new file mode 100644 index 00000000000..a0132d06bf3 --- /dev/null +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -0,0 +1,42 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connections + +import ( + "net" + + "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" +) + +var _ ConnTrackDumper = new(connTrackDumper) + +type connTrackDumper struct { + connTrack ConnTrackInterfacer + nodeConfig *config.NodeConfig + serviceCIDR *net.IPNet + datapathType string + ovsctlClient ovsctl.OVSCtlClient +} + +func NewConnTrackDumper(connTrack ConnTrackInterfacer, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, dpType string, ovsctlClient ovsctl.OVSCtlClient) *connTrackDumper { + return &connTrackDumper{ + connTrack, + nodeConfig, + serviceCIDR, + dpType, + ovsctlClient, + } +} diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index c183ca28662..11fc0b38e6e 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -17,61 +17,83 @@ package connections import ( + "fmt" "net" + "os" + "strconv" + "strings" "github.com/ti-mo/conntrack" "k8s.io/klog/v2" - "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" "github.com/vmware-tanzu/antrea/pkg/agent/util/sysctl" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" + "github.com/vmware-tanzu/antrea/pkg/util/ip" ) -var _ ConnTrackDumper = new(connTrackDumper) - -type connTrackDumper struct { - nodeConfig *config.NodeConfig - serviceCIDR *net.IPNet - connTrack ConnTrackInterfacer -} - -func NewConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, conntrack ConnTrackInterfacer) *connTrackDumper { - return &connTrackDumper{ - nodeConfig, - serviceCIDR, - conntrack, - } -} - // DumpFlows opens netlink connection and dumps all the flows in Antrea ZoneID // of conntrack table, i.e., corresponding to Antrea OVS bridge. func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - // Get netlink Connection to netfilter - err := ctdump.connTrack.Dial() - if err != nil { - klog.Errorf("Error when getting netlink conn: %v", err) - return nil, err +if ctdump.datapathType == ovsconfig.OVSDatapathSystem { + // Get connection to netlink socket + err := ctdump.connTrack.GetConnTrack(nil) + if err != nil { + klog.Errorf("Error when getting netlink conn: %v", err) + return nil, err + } + } else if ctdump.datapathType == ovsconfig.OVSDatapathNetdev { + // Set ovsCtlClient to dump conntrack flows + err := ctdump.connTrack.GetConnTrack(ctdump.ovsctlClient) + if err != nil { + klog.Errorf("Error when getting ovsclient: %v", err) + return nil, err + } } // ZoneID filter is not supported currently in tl-mo/conntrack library. // Link to issue: https://github.com/ti-mo/conntrack/issues/23 // Dump all flows in the conntrack table for now. - conns, err := ctdump.connTrack.DumpFilter(conntrack.Filter{}) - if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) - return nil, err + var conns []*flowexporter.Connection + if ctdump.datapathType == ovsconfig.OVSDatapathSystem { + conns, err = ctdump.connTrack.DumpFilter(conntrack.Filter{}) + if err != nil { + klog.Errorf("Error when dumping flows from conntrack: %v", err) + return nil, err + } + } else if ctdump.datapathType == ovsconfig.OVSDatapathNetdev { + // This is supported for kind clusters. Ovs-appctl access in kind clusters is unstable currently. + // This will be used once the issue with Ovs-appctl is fixed on kind cluster nodes. + conns, err = ctdump.connTrack.DumpFilter(uint16(openflow.CtZone)) + if err != nil { + klog.Errorf("Error when dumping flows from conntrack: %v", err) + return nil, err + } } - filteredConns := make([]*flowexporter.Connection, 0, len(conns)) - for _, conn := range conns { - if conn.Zone != openflow.CtZone { + for i := 0; i < len(conns); i++ { + if conns[i].Zone != openflow.CtZone { + // Delete the element from the slice + conns[i] = conns[len(conns)-1] + conns[len(conns)-1] = nil + conns = conns[:len(conns)-1] + // Decrement i to iterate over swapped element + i = i - 1 continue } - srcIP := conn.TupleOrig.IP.SourceAddress - dstIP := conn.TupleReply.IP.SourceAddress + srcIP := conns[i].TupleOrig.SourceAddress + dstIP := conns[i].TupleReply.SourceAddress + // Only get Pod-to-Pod flows. Pod-to-ExternalService flows are ignored for now. if srcIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) || dstIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) { + // Delete the element from the slice + conns[i] = conns[len(conns)-1] + conns[len(conns)-1] = nil + conns = conns[:len(conns)-1] + // Decrement i to iterate over swapped element + i = i - 1 continue } @@ -84,25 +106,23 @@ func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Con // Conntrack flows will be different for Pod-to-Service flows w/ Antrea-proxy. This implementation will be simpler, when the // Antrea proxy is supported. if ctdump.serviceCIDR.Contains(srcIP) || ctdump.serviceCIDR.Contains(dstIP) { + // Delete element from the slice + conns[i] = conns[len(conns)-1] + conns[len(conns)-1] = nil + conns = conns[:len(conns)-1] + // Decrement i to iterate over swapped element + i = i - 1 continue } - filteredConns = append(filteredConns, createAntreaConn(&conn)) } + klog.V(2).Infof("No. of flow exporter considered flows in Antrea zoneID: %d", len(conns)) - klog.V(2).Infof("Finished poll cycle -- total flows: %d flows in Antrea zoneID: %d", len(conns), len(filteredConns)) - - return filteredConns, nil + return conns, nil } // connTrackSystem implements ConnTrackInterfacer var _ ConnTrackInterfacer = new(connTrackSystem) - -// ConnTrackInterfacer is an interface created to consume the required functions from the third party -// conntrack library. This is helpful in writing unit tests. -type ConnTrackInterfacer interface { - Dial() error - DumpFilter(filter conntrack.Filter) ([]conntrack.Flow, error) -} +var _ ConnTrackInterfacer = new(connTrackNetdev) type connTrackSystem struct { netlinkConn *conntrack.Conn @@ -121,24 +141,167 @@ func NewConnTrackInterfacer() *connTrackSystem { return &connTrackSystem{} } -func (c *connTrackSystem) Dial() error { - // Get conntrack in current namespace +type connTrackNetdev struct { + ovsCtl ovsctl.OVSCtlClient +} + +func NewConnTrackNetdev() *connTrackNetdev { + return &connTrackNetdev{} +} + +func (ctnl *connTrackSystem) GetConnTrack(config interface{}) error { + if config != nil { + return fmt.Errorf("this function does not expect any netlink config") + } + // Get netlink client in current namespace conn, err := conntrack.Dial(nil) if err != nil { klog.Errorf("Error when dialing conntrack: %v", err) return err } - c.netlinkConn = conn + ctnl.netlinkConn = conn return nil } -func (c *connTrackSystem) DumpFilter(filter conntrack.Filter) ([]conntrack.Flow, error) { - conns, err := c.netlinkConn.DumpFilter(filter) +func (ctnl *connTrackSystem) DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) { + netlinkFilter, ok := filter.(conntrack.Filter) + if !ok { + return nil, fmt.Errorf("error: filter should be of type conntrack.Filter") + } + conns, err := ctnl.netlinkConn.DumpFilter(netlinkFilter) if err != nil { klog.Errorf("Error when dumping flows from conntrack: %v", err) return nil, err } - return conns, nil + antreaConns := make([]*flowexporter.Connection, len(conns)) + for i, conn := range conns { + antreaConns[i] = createAntreaConn(&conn) + } + klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + return antreaConns, nil +} + +func (ctnd *connTrackNetdev) GetConnTrack(config interface{}) error { + client, ok := config.(ovsctl.OVSCtlClient) + if !ok { + return fmt.Errorf("config should be ovsCtlClient of type OVSCtlClient") + } + ctnd.ovsCtl = client + return nil +} + +func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) { + zoneFilter, ok := filter.(uint16) + if !ok { + return nil, fmt.Errorf("filter should be of type uint16") + } + + // Dump conntrack using ovs-appctl dpctl/dump-conntrack + cmdOutput, execErr := ctnd.ovsCtl.RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s") + if execErr != nil { + return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) + } + + // Parse the output to get the flows + antreaConns := make([]*flowexporter.Connection, 0) + outputFlow := strings.Split(string(cmdOutput), "\n") + var err error + for _, flow := range outputFlow { + conn := flowexporter.Connection{} + flowSlice := strings.Split(flow, ",") + isReply := false + inZone := true + for _, fs := range flowSlice { + // Indicator to populate reply or reverse fields + if strings.Contains(fs, "reply") { + isReply = true + } + if !strings.Contains(fs, "=") { + // Proto identifier + conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) + if err != nil { + klog.Errorf("Unknown protocol to convert to ID: %s", fs) + continue + } + conn.TupleReply.Protocol = conn.TupleOrig.Protocol + } else if strings.Contains(fs, "src") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "dst") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "sport") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.SourcePort = uint16(val) + } else { + conn.TupleReply.SourcePort = uint16(val) + } + } else if strings.Contains(fs, "dport") { + // dport field could be the last tuple field in ovs-dpctl output format. + fs = strings.TrimSuffix(fs, ")") + + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.DestinationPort = uint16(val) + } else { + conn.TupleReply.DestinationPort = uint16(val) + } + } else if strings.Contains(fs, "zone") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) + continue + } + if zoneFilter != uint16(val) { + inZone = false + break + } else { + conn.Zone = uint16(val) + } + } else if strings.Contains(fs, "timeout") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) + continue + } + conn.Timeout = uint32(val) + } else if strings.Contains(fs, "id") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) + continue + } + conn.ID = uint32(val) + } + } + if inZone { + antreaConns = append(antreaConns, &conn) + } + } + klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + return antreaConns, nil } func createAntreaConn(conn *conntrack.Flow) *flowexporter.Connection { diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index e88a99e46d7..5e4389cd0cf 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -25,109 +25,88 @@ import ( "github.com/ti-mo/conntrack" "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" connectionstest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections/testing" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" + ovsctltest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl/testing" ) var ( - tuple3 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{1, 2, 3, 4}, - DestinationAddress: net.IP{4, 3, 2, 1}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 65280, - DestinationPort: 255, - }, + tuple3 = flowexporter.Tuple{ + SourceAddress: net.IP{1, 2, 3, 4}, + DestinationAddress: net.IP{4, 3, 2, 1}, + Protocol: 6, + SourcePort: 65280, + DestinationPort: 255, } - revTuple3 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{4, 3, 2, 1}, - DestinationAddress: net.IP{1, 2, 3, 4}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 255, - DestinationPort: 65280, - }, + revTuple3 = flowexporter.Tuple{ + SourceAddress: net.IP{4, 3, 2, 1}, + DestinationAddress: net.IP{1, 2, 3, 4}, + Protocol: 6, + SourcePort: 255, + DestinationPort: 65280, } - tuple4 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{5, 6, 7, 8}, - DestinationAddress: net.IP{8, 7, 6, 5}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 60001, - DestinationPort: 200, - }, + tuple4 = flowexporter.Tuple{ + SourceAddress: net.IP{5, 6, 7, 8}, + DestinationAddress: net.IP{8, 7, 6, 5}, + Protocol: 6, + SourcePort: 60001, + DestinationPort: 200, } - revTuple4 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{8, 7, 6, 5}, - DestinationAddress: net.IP{5, 6, 7, 8}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 200, - DestinationPort: 60001, - }, + revTuple4 = flowexporter.Tuple{ + SourceAddress: net.IP{8, 7, 6, 5}, + DestinationAddress: net.IP{5, 6, 7, 8}, + Protocol: 6, + SourcePort: 200, + DestinationPort: 60001, } - tuple5 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{1, 2, 3, 4}, - DestinationAddress: net.IP{100, 50, 25, 5}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 60001, - DestinationPort: 200, - }, + tuple5 = flowexporter.Tuple{ + SourceAddress: net.IP{1, 2, 3, 4}, + DestinationAddress: net.IP{100, 50, 25, 5}, + Protocol: 6, + SourcePort: 60001, + DestinationPort: 200, } - revTuple5 = conntrack.Tuple{ - IP: conntrack.IPTuple{ - SourceAddress: net.IP{100, 50, 25, 5}, - DestinationAddress: net.IP{1, 2, 3, 4}, - }, - Proto: conntrack.ProtoTuple{ - Protocol: 6, - SourcePort: 200, - DestinationPort: 60001, - }, + revTuple5 = flowexporter.Tuple{ + SourceAddress: net.IP{100, 50, 25, 5}, + DestinationAddress: net.IP{1, 2, 3, 4}, + Protocol: 6, + SourcePort: 200, + DestinationPort: 60001, } ) -func TestConnTrack_DumpFilter(t *testing.T) { +func TestConnTrack_DumpFlows(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Create flows to test - antreaFlow := conntrack.Flow{ + antreaFlow := &flowexporter.Connection{ TupleOrig: tuple3, TupleReply: revTuple3, Zone: openflow.CtZone, } - antreaServiceFlow := conntrack.Flow{ + antreaServiceFlow := &flowexporter.Connection{ TupleOrig: tuple5, TupleReply: revTuple5, Zone: openflow.CtZone, } - antreaGWFlow := conntrack.Flow{ + antreaGWFlow := &flowexporter.Connection{ TupleOrig: tuple4, TupleReply: revTuple4, Zone: openflow.CtZone, } - nonAntreaFlow := conntrack.Flow{ + nonAntreaFlow := &flowexporter.Connection{ TupleOrig: tuple4, TupleReply: revTuple4, Zone: 100, } - testFlows := []conntrack.Flow{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} + testFlows := []*flowexporter.Connection{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} - // Create mock ConnTrackInterfacer interface + // Create mock interfaces mockCTInterfacer := connectionstest.NewMockConnTrackInterfacer(ctrl) - + mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) // Create nodeConfig and gateWayConfig // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP gwConfig := &config.GatewayConfig{ @@ -142,11 +121,11 @@ func TestConnTrack_DumpFilter(t *testing.T) { Mask: net.IPMask{255, 255, 255, 0}, } // set expects for mocks - mockCTInterfacer.EXPECT().Dial().Return(nil) + mockCTInterfacer.EXPECT().GetConnTrack(nil).Return(nil) mockCTInterfacer.EXPECT().DumpFilter(conntrack.Filter{}).Return(testFlows, nil) - connTrack := NewConnTrackDumper(nodeConfig, serviceCIDR, mockCTInterfacer) - conns, err := connTrack.DumpFlows(openflow.CtZone) + connDumper := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathSystem, mockOVSCtlClient) + conns, err := connDumper.DumpFlows(openflow.CtZone) if err != nil { t.Errorf("Dump flows function returned error: %v", err) } diff --git a/pkg/agent/flowexporter/connections/conntrack_windows.go b/pkg/agent/flowexporter/connections/conntrack_windows.go index 6a045c55c22..c4d1f81bd47 100644 --- a/pkg/agent/flowexporter/connections/conntrack_windows.go +++ b/pkg/agent/flowexporter/connections/conntrack_windows.go @@ -17,39 +17,18 @@ package connections import ( - "net" - - "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ) -var _ ConnTrackDumper = new(connTrackDumper) - -type connTrackDumper struct { - nodeConfig *config.NodeConfig - serviceCIDR *net.IPNet - connTrack ConnTrackInterfacer +func (cp *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { + return nil, nil } -func NewConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, conntrack ConnTrackInterfacer) *connTrackDumper { - return &connTrackDumper{ - nodeConfig, - serviceCIDR, - conntrack, - } +// TODO: Implement ConnTrackInterfacer when flow exporter is supported for windows. +func NewConnTrackSystem() ConnTrackInterfacer { + return nil } -// TODO: These will be defined when polling from ovs-dpctl dump conntrack is supported for windows. -var _ ConnTrackInterfacer = new(connTrackSystem) - -type ConnTrackInterfacer interface{} - -type connTrackSystem struct{} - -func NewConnTrackInterfacer() *connTrackSystem { - return &connTrackSystem{} -} - -func (cp *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - return nil, nil +func NewConnTrackNetdev() ConnTrackInterfacer { + return nil } diff --git a/pkg/agent/flowexporter/connections/interface.go b/pkg/agent/flowexporter/connections/interface.go index df74cc2cea9..d510be271d3 100644 --- a/pkg/agent/flowexporter/connections/interface.go +++ b/pkg/agent/flowexporter/connections/interface.go @@ -23,3 +23,10 @@ import ( type ConnTrackDumper interface { DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) } + +// ConnTrackInterfacer is an interface created to consume the required dump functions from either the third party +// conntrack library or internal packages depending on OVS datapath type or OS. +type ConnTrackInterfacer interface { + GetConnTrack(config interface{}) error // suggest a different name for config if it is not appropriate + DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) +} diff --git a/pkg/agent/flowexporter/connections/testing/mock_connections.go b/pkg/agent/flowexporter/connections/testing/mock_connections.go index 0ccd5ebf092..6a26b3cc296 100644 --- a/pkg/agent/flowexporter/connections/testing/mock_connections.go +++ b/pkg/agent/flowexporter/connections/testing/mock_connections.go @@ -21,7 +21,6 @@ package testing import ( gomock "github.com/golang/mock/gomock" - conntrack "github.com/ti-mo/conntrack" flowexporter "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" reflect "reflect" ) @@ -87,25 +86,11 @@ func (m *MockConnTrackInterfacer) EXPECT() *MockConnTrackInterfacerMockRecorder return m.recorder } -// Dial mocks base method -func (m *MockConnTrackInterfacer) Dial() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Dial") - ret0, _ := ret[0].(error) - return ret0 -} - -// Dial indicates an expected call of Dial -func (mr *MockConnTrackInterfacerMockRecorder) Dial() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Dial", reflect.TypeOf((*MockConnTrackInterfacer)(nil).Dial)) -} - // DumpFilter mocks base method -func (m *MockConnTrackInterfacer) DumpFilter(arg0 conntrack.Filter) ([]conntrack.Flow, error) { +func (m *MockConnTrackInterfacer) DumpFilter(arg0 interface{}) ([]*flowexporter.Connection, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DumpFilter", arg0) - ret0, _ := ret[0].([]conntrack.Flow) + ret0, _ := ret[0].([]*flowexporter.Connection) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -115,3 +100,17 @@ func (mr *MockConnTrackInterfacerMockRecorder) DumpFilter(arg0 interface{}) *gom mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpFilter", reflect.TypeOf((*MockConnTrackInterfacer)(nil).DumpFilter), arg0) } + +// GetConnTrack mocks base method +func (m *MockConnTrackInterfacer) GetConnTrack(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConnTrack", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetConnTrack indicates an expected call of GetConnTrack +func (mr *MockConnTrackInterfacerMockRecorder) GetConnTrack(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConnTrack", reflect.TypeOf((*MockConnTrackInterfacer)(nil).GetConnTrack), arg0) +} diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 79cbe523c36..5bb19b6cb7e 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -1,11 +1,25 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package exporter import ( "fmt" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" + "github.com/vmware-tanzu/antrea/pkg/util/env" "hash/fnv" "net" - "os" "strings" "time" "unicode" @@ -50,29 +64,15 @@ type FlowExporter interface { } type flowExporter struct { - flowRecords flowrecords.FlowRecords - process ipfix.IPFIXExportingProcess - elementsList []*ipfixentities.InfoElement - templateID uint16 -} - -func getNodeName() (string, error) { - const nodeNameEnvKey = "NODE_NAME" - nodeName := os.Getenv(nodeNameEnvKey) - if nodeName != "" { - return nodeName, nil - } - klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey) - var err error - nodeName, err = os.Hostname() - if err != nil { - return "", fmt.Errorf("failed to get local hostname: %v", err) - } - return nodeName, nil + flowRecords flowrecords.FlowRecords + process ipfix.IPFIXExportingProcess + elementsList []*ipfixentities.InfoElement + exportInterval time.Duration + templateID uint16 } func genObservationID() (uint32, error) { - name, err := getNodeName() + name, err := env.GetNodeName() if err != nil { return 0, err } @@ -81,7 +81,7 @@ func genObservationID() (uint32, error) { return h.Sum32(), nil } -func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords) (*flowExporter, error) { +func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expInterval time.Duration) (*flowExporter, error) { // Create IPFIX exporting expProcess and initialize registries and other related entities obsID, err := genObservationID() if err != nil { @@ -98,6 +98,7 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords) (*flo records, expProcess, nil, + expInterval, 0, } @@ -115,12 +116,15 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords) (*flo func (exp *flowExporter) Run(stopCh <-chan struct{}) { klog.Infof("Start exporting IPFIX flow records") + ticker := time.NewTicker(exp.exportInterval) + defer ticker.Stop() + for { select { case <-stopCh: exp.process.CloseConnToCollector() break - case <-time.After(flowexporter.FlowExportInterval): + case <-ticker.C: err := exp.flowRecords.BuildFlowRecords() if err != nil { klog.Errorf("Error when building flow records: %v", err) diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 6a88b6c791a..ebc7a5cda95 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package exporter import ( @@ -25,6 +39,7 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { nil, mockIPFIXExpProc, nil, + 60 * time.Second, 256, } // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) @@ -131,6 +146,7 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { nil, mockIPFIXExpProc, elemList, + 60 * time.Second, 256, } // Expect calls required diff --git a/pkg/agent/flowexporter/flowrecords/flowrecords.go b/pkg/agent/flowexporter/flowrecords/flowrecords.go index b124fd2a7cc..3fd89b02423 100644 --- a/pkg/agent/flowexporter/flowrecords/flowrecords.go +++ b/pkg/agent/flowexporter/flowrecords/flowrecords.go @@ -1,3 +1,17 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package flowrecords import ( diff --git a/pkg/agent/flowexporter/ipfix/ipfixprocess.go b/pkg/agent/flowexporter/ipfix/ipfixprocess.go index 90093725894..6e3ee65172a 100644 --- a/pkg/agent/flowexporter/ipfix/ipfixprocess.go +++ b/pkg/agent/flowexporter/ipfix/ipfixprocess.go @@ -1,3 +1,17 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ipfix import ( diff --git a/pkg/agent/flowexporter/ipfix/ipfixrecord.go b/pkg/agent/flowexporter/ipfix/ipfixrecord.go index ebb6708db1d..536135b9564 100644 --- a/pkg/agent/flowexporter/ipfix/ipfixrecord.go +++ b/pkg/agent/flowexporter/ipfix/ipfixrecord.go @@ -1,3 +1,17 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ipfix import ( diff --git a/pkg/agent/flowexporter/types.go b/pkg/agent/flowexporter/types.go index b9b34f4be19..e44930c337b 100644 --- a/pkg/agent/flowexporter/types.go +++ b/pkg/agent/flowexporter/types.go @@ -20,11 +20,6 @@ import ( "time" ) -const ( - PollInterval = 5 * time.Second - FlowExportInterval = 120 * time.Second -) - type ConnectionKey [5]string type FlowRecordUpdate func(key ConnectionKey, cxn Connection) error @@ -48,6 +43,7 @@ type Connection struct { StopTime time.Time Zone uint16 StatusFlag uint32 + // TODO: Have a separate field for protocol. No need to keep it in Tuple. TupleOrig, TupleReply Tuple OriginalPackets, OriginalBytes uint64 ReversePackets, ReverseBytes uint64 diff --git a/pkg/agent/flowexporter/utils.go b/pkg/agent/flowexporter/utils.go index 59f9a55f29a..f74e97e6589 100644 --- a/pkg/agent/flowexporter/utils.go +++ b/pkg/agent/flowexporter/utils.go @@ -14,7 +14,9 @@ package flowexporter -import "strconv" +import ( + "strconv" +) // NewConnectionKey creates 5-tuple of flow as connection key func NewConnectionKey(conn *Connection) ConnectionKey { diff --git a/pkg/ovs/ovsctl/appctl.go b/pkg/ovs/ovsctl/appctl.go index 1a81eb3ab7f..e50125193f5 100644 --- a/pkg/ovs/ovsctl/appctl.go +++ b/pkg/ovs/ovsctl/appctl.go @@ -121,7 +121,7 @@ func (c *ovsCtlClient) Trace(req *TracingRequest) (string, error) { } func (c *ovsCtlClient) runTracing(flow string) (string, error) { - out, execErr := c.runAppctlCmd("ofproto/trace", flow) + out, execErr := c.RunAppctlCmd("ofproto/trace", true, flow) if execErr != nil { return "", execErr } @@ -130,11 +130,16 @@ func (c *ovsCtlClient) runTracing(flow string) (string, error) { return string(out), nil } -func (c *ovsCtlClient) runAppctlCmd(cmd string, args ...string) ([]byte, *ExecError) { +func (c *ovsCtlClient) RunAppctlCmd(cmd string, needsBridge bool, args ...string) ([]byte, *ExecError) { // Use the control UNIX domain socket to connect to ovs-vswitchd, as Agent can // run in a different PID namespace from ovs-vswitchd, and so might not be able // to reach ovs-vswitchd using the PID. - cmdStr := fmt.Sprintf("ovs-appctl -t %s %s %s", ovsVSwitchdUDS, cmd, c.bridge) + var cmdStr string + if needsBridge { + cmdStr = fmt.Sprintf("ovs-appctl -t %s %s %s", ovsVSwitchdUDS, cmd, c.bridge) + } else { + cmdStr = fmt.Sprintf("ovs-appctl -t %s %s", ovsVSwitchdUDS, cmd) + } cmdStr = cmdStr + " " + strings.Join(args, " ") out, err := getOVSCommand(cmdStr).CombinedOutput() if err != nil { diff --git a/pkg/ovs/ovsctl/interface.go b/pkg/ovs/ovsctl/interface.go index b285c7fc6af..f77d9e8258b 100644 --- a/pkg/ovs/ovsctl/interface.go +++ b/pkg/ovs/ovsctl/interface.go @@ -47,6 +47,9 @@ type OVSCtlClient interface { SetPortNoFlood(ofport int) error // Trace executes "ovs-appctl ofproto/trace" to perform OVS packet tracing. Trace(req *TracingRequest) (string, error) + // RunAppctlCmd executes "ovs-appctl" command and returns the outputs. + // Some commands are bridge specific and some are not. Passing a bool to distinguish that. + RunAppctlCmd(cmd string, needsBridge bool, args ...string) ([]byte, *ExecError) } type BadRequestError string diff --git a/pkg/ovs/ovsctl/testing/mock_ovsctl.go b/pkg/ovs/ovsctl/testing/mock_ovsctl.go index 20b7a941975..5ad8b9fe7b2 100644 --- a/pkg/ovs/ovsctl/testing/mock_ovsctl.go +++ b/pkg/ovs/ovsctl/testing/mock_ovsctl.go @@ -116,6 +116,26 @@ func (mr *MockOVSCtlClientMockRecorder) DumpTableFlows(arg0 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpTableFlows", reflect.TypeOf((*MockOVSCtlClient)(nil).DumpTableFlows), arg0) } +// RunAppctlCmd mocks base method +func (m *MockOVSCtlClient) RunAppctlCmd(arg0 string, arg1 bool, arg2 ...string) ([]byte, *ovsctl.ExecError) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RunAppctlCmd", varargs...) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(*ovsctl.ExecError) + return ret0, ret1 +} + +// RunAppctlCmd indicates an expected call of RunAppctlCmd +func (mr *MockOVSCtlClientMockRecorder) RunAppctlCmd(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunAppctlCmd", reflect.TypeOf((*MockOVSCtlClient)(nil).RunAppctlCmd), varargs...) +} + // RunOfctlCmd mocks base method func (m *MockOVSCtlClient) RunOfctlCmd(arg0 string, arg1 ...string) ([]byte, error) { m.ctrl.T.Helper() diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go index 2af91343451..9afbbd7c85a 100644 --- a/pkg/util/ip/ip.go +++ b/pkg/util/ip/ip.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "sort" + "strings" "github.com/vmware-tanzu/antrea/pkg/apis/networking/v1beta1" ) @@ -28,6 +29,15 @@ const ( v6BitLen = 8 * net.IPv6len ) +// Following map is for converting protocol name (string) to protocol identifier +var protocols = map[string]uint8{ + "icmp": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "ipv6-icmp": 58, +} + // This function takes in one allow CIDR and multiple except CIDRs and gives diff CIDRs // in allowCIDR eliminating except CIDRs. It currently supports only IPv4. except CIDR input // can be changed. @@ -141,8 +151,18 @@ func IPNetToNetIPNet(ipNet *v1beta1.IPNet) *net.IPNet { return &net.IPNet{IP: ip, Mask: net.CIDRMask(int(ipNet.PrefixLength), bits)} } -// Function to transform net.IPNet to Antrea IPNet +// NetIPNetToIPNet transforms net.IPNet to Antrea IPNet func NetIPNetToIPNet(ipNet *net.IPNet) *v1beta1.IPNet { prefix, _ := ipNet.Mask.Size() return &v1beta1.IPNet{IP: v1beta1.IPAddress(ipNet.IP), PrefixLength: int32(prefix)} } + +// LookupProtocolMap return protocol identifier given protocol name +func LookupProtocolMap(name string) (uint8, error) { + lowerCaseStr := strings.ToLower(name) + proto, found := protocols[lowerCaseStr] + if !found { + return 0, fmt.Errorf("unknown IP protocol specified: %s", name) + } + return proto, nil +} diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index 0e9b5e08257..32fd7cc0274 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -441,7 +441,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/srikartati/go-ipfixlib v0.0.0-20200615234147-74c918af6836/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= +github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/test/e2e/bandwidth_test.go b/test/e2e/bandwidth_test.go index 5fd5a3d486e..959e704066a 100644 --- a/test/e2e/bandwidth_test.go +++ b/test/e2e/bandwidth_test.go @@ -32,13 +32,13 @@ func TestBenchmarkBandwidthIntraNode(t *testing.T) { t.Fatalf("Error when setting up test: %v", err) } defer teardownTest(t, data) - if err := data.createPodOnNode("perftest-a", masterNodeName(), perftoolImage, nil, nil, nil, nil); err != nil { + if err := data.createPodOnNode("perftest-a", masterNodeName(), perftoolImage, nil, nil, nil, nil, false); err != nil { t.Fatalf("Error when creating the perftest client Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-a", testNamespace); err != nil { t.Fatalf("Error when waiting for the perftest client Pod: %v", err) } - if err := data.createPodOnNode("perftest-b", masterNodeName(), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}); err != nil { + if err := data.createPodOnNode("perftest-b", masterNodeName(), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false); err != nil { t.Fatalf("Error when creating the perftest server Pod: %v", err) } podBIP, err := data.podWaitForIP(defaultTimeout, "perftest-b", testNamespace) @@ -64,13 +64,13 @@ func benchmarkBandwidthService(t *testing.T, endpointNode, clientNode string) { if err != nil { t.Fatalf("Error when creating perftest service: %v", err) } - if err := data.createPodOnNode("perftest-a", clientNode, perftoolImage, nil, nil, nil, nil); err != nil { + if err := data.createPodOnNode("perftest-a", clientNode, perftoolImage, nil, nil, nil, nil, false); err != nil { t.Fatalf("Error when creating the perftest client Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-a", testNamespace); err != nil { t.Fatalf("Error when waiting for the perftest client Pod: %v", err) } - if err := data.createPodOnNode("perftest-b", endpointNode, perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}); err != nil { + if err := data.createPodOnNode("perftest-b", endpointNode, perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false); err != nil { t.Fatalf("Error when creating the perftest server Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-b", testNamespace); err != nil { diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index f0ef98514f8..46bf5faf336 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -23,6 +23,11 @@ import ( "time" ) +const ( + ipfixCollectorImage = "antrea/ipfix-collector:06252020.1" + ipfixCollectorPort = "4739" +) + func skipIfNotBenchmarkTest(tb testing.TB) { if !testOptions.withBench { tb.Skipf("Skipping benchmark test: %s", tb.Name()) @@ -75,6 +80,41 @@ func setupTest(tb testing.TB) (*TestData, error) { return data, nil } +func setupTestWithIPFIXCollector(tb testing.TB) (*TestData, error) { + data := &TestData{} + tb.Logf("Creating K8s clientset") + // TODO: it is probably not needed to re-create the clientset in each test, maybe we could + // just keep it in clusterInfo? + if err := data.createClient(); err != nil { + return nil, err + } + tb.Logf("Creating '%s' K8s Namespace", testNamespace) + if err := data.createTestNamespace(); err != nil { + return nil, err + } + // Create pod using ipfix collector image + if err := data.createPodOnNode("ipfix-collector", masterNodeName(), ipfixCollectorImage, nil, nil, nil, nil, true); err != nil { + tb.Fatalf("Error when creating the ipfix collector Pod: %v", err) + } + ipfixCollIP, err := data.podWaitForIP(defaultTimeout, "ipfix-collector", testNamespace) + if err != nil { + tb.Fatalf("Error when waiting to get ipfix collector Pod IP: %v", err) + } + tb.Logf("Applying Antrea YAML with ipfix collector address") + if err := data.deployAntreaFlowExporter(ipfixCollIP + ":" + ipfixCollectorPort + ":tcp"); err != nil { + return data, err + } + tb.Logf("Waiting for all Antrea DaemonSet Pods") + if err := data.waitForAntreaDaemonSetPods(defaultTimeout); err != nil { + return data, err + } + tb.Logf("Checking CoreDNS deployment") + if err := data.checkCoreDNSPods(defaultTimeout); err != nil { + return data, err + } + return data, nil +} + func logsDirForTest(testName string) string { // a filepath-friendly timestamp format. const timeFormat = "Jan02-15-04-05" diff --git a/test/e2e/flowexporter_test.go b/test/e2e/flowexporter_test.go new file mode 100644 index 00000000000..33c2f86dd62 --- /dev/null +++ b/test/e2e/flowexporter_test.go @@ -0,0 +1,135 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/hex" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" +) + +// TestFlowExporter runs flow exporter to export flow records for flows. +// Flows are deployed between Pods on same node. +func TestFlowExporter(t *testing.T) { + // Should I add skipBenchmark as this runs iperf? + data, err := setupTestWithIPFIXCollector(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + if err := data.createPodOnNode("perftest-a", masterNodeName(), perftoolImage, nil, nil, nil, nil, false); err != nil { + t.Fatalf("Error when creating the perftest client Pod: %v", err) + } + podAIP, err := data.podWaitForIP(defaultTimeout, "perftest-a", testNamespace) + if err != nil { + t.Fatalf("Error when waiting for the perftest client Pod: %v", err) + } + if err := data.createPodOnNode("perftest-b", masterNodeName(), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false); err != nil { + t.Fatalf("Error when creating the perftest server Pod: %v", err) + } + podBIP, err := data.podWaitForIP(defaultTimeout, "perftest-b", testNamespace) + if err != nil { + t.Fatalf("Error when getting the perftest server Pod's IP: %v", err) + } + stdout, _, err := data.runCommandFromPod(testNamespace, "perftest-a", "perftool", []string{"bash", "-c", fmt.Sprintf("iperf3 -c %s|grep sender|awk '{print $7,$8}'", podBIP)}) + if err != nil { + t.Fatalf("Error when running iperf3 client: %v", err) + } + bandwidth := strings.TrimSpace(stdout) + + // Adding some delay to make sure all the data records corresponding to iperf flow are received. + time.Sleep(500 * time.Millisecond) + + rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs ipfix-collector -n antrea-test")) + if err != nil || rc != 0 { + t.Fatalf("error when getting logs %v, rc: %v", err, rc) + } + + // Parse through IPFIX collector output + re := regexp.MustCompile("(?m)^.*" + "#" + ".*$[\r\n]+") + collectorOutput = re.ReplaceAllString(collectorOutput, "") + collectorOutput = strings.TrimSpace(collectorOutput) + recordSlices := strings.Split(collectorOutput, "IPFIX-HDR:") + // Delete the first element from recordSlices + recordSlices[0] = recordSlices[len(recordSlices)-1] + recordSlices[len(recordSlices)-1] = "" + recordSlices = recordSlices[:len(recordSlices)-1] + // Iterate over recordSlices and build some results to test with expected results + templateRecords := 0 + dataRecordsIntraNode := 0 + for _, record := range recordSlices { + if strings.Contains(record, "TEMPLATE RECORD") { + templateRecords = templateRecords + 1 + } + + if strings.Contains(record, podAIP) && strings.Contains(record, podBIP) { + dataRecordsIntraNode = dataRecordsIntraNode + 1 + // Check if records have both Pod name and Pod namespace or not + if !strings.Contains(record, hex.EncodeToString([]byte("perftest-a"))) { + t.Fatalf("Records with podAIP does not have pod name") + } + if !strings.Contains(record, hex.EncodeToString([]byte("perftest-b"))) { + t.Fatalf("Records with podBIP does not have pod name") + } + if !strings.Contains(record, hex.EncodeToString([]byte(testNamespace))) { + t.Fatalf("Records with podAIP and podBIP does not have pod namespace") + } + // Check the bandwidth using octetDeltaCount in data records sent in second ipfix interval + if strings.Contains(record, "seqno=2") || strings.Contains(record, "seqno=3") { + // One of them has no bytes ignore that + if !strings.Contains(record, "octetDeltaCount: 0") { + //split the record in lines to compute bandwidth + splitLines := strings.Split(record, "\n") + for _, line := range splitLines { + if strings.Contains(line, "octetDeltaCount") { + lineSlice := strings.Split(line, ":") + deltaBytes, err := strconv.ParseFloat(strings.TrimSpace(lineSlice[1]), 64) + if err != nil { + t.Fatalf("Error in converting octetDeltaCount to int type") + } + // compute the bandwidth using 5s as interval + recBandwidth := (deltaBytes * 8.0) / (5.0 * math.Pow10(9)) + // bandwidth from iperf output + bwSlice := strings.Split(bandwidth, " ") + iperfBandwidth, err := strconv.ParseFloat(bwSlice[0], 64) + if err != nil { + t.Fatalf("Error in converting iperf bandwidth to float64 type") + } + // Check if at least the first digit is equal, i.e., 42 Gb/s and 48 Gb/s are considered equal + // we cannot guarantee both will be exactly same. Logging both values to give visibility. + t.Logf("Iperf bandwidth: %v", iperfBandwidth) + t.Logf("IPFIX record bandwidth: %v", recBandwidth) + assert.Equal(t, int(recBandwidth/10), int(float64(iperfBandwidth)/10), "Iperf bandwidth and IPFIX record bandwidth should be similar") + break + } + } + } + } + } + } + assert.Equal(t, templateRecords, clusterInfo.numNodes, "Each agent should send out template record") + // Single iperf resulting in two connections with separate ports. Suspecting second flow to be control flow to exchange + // stats info. As 5s is export interval and iperf traffic runs for 10s, we expect 4 records. + assert.Equal(t, dataRecordsIntraNode, 4, "Iperf flow should have expected number of flow records") +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 943617e3af0..967daccf3b7 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -47,7 +47,7 @@ import ( ) const ( - defaultTimeout time.Duration = 90 * time.Second + defaultTimeout = 90 * time.Second // antreaNamespace is the K8s Namespace in which all Antrea resources are running. antreaNamespace string = "kube-system" @@ -288,6 +288,7 @@ func (data *TestData) deployAntreaCommon(yamlFile string, extraOptions string) e if err != nil || rc != 0 { return fmt.Errorf("error when waiting for Antrea rollout to complete") } + return nil } @@ -301,6 +302,57 @@ func (data *TestData) deployAntreaIPSec() error { return data.deployAntreaCommon(antreaIPSecYML, "") } +// deployAntreaFlowExporter deploys Antrea with flow exporter config params enabled. +func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { + // This is to add ipfixCollector address and pollAndExportInterval config params to antrea agent configmap + cmd := fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowCollectorAddr: \"\"|flowCollectorAddr: \"%s\"|g' %s", ipfixCollector, antreaYML) + rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + // pollAndExportInterval is added as harcoded value "1s:5s" + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#pollAndExportInterval: \"\"|pollAndExportInterval: \"1s:5s\"|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + // Turn on FlowExporter feature in featureGates + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#featureGates:|featureGates:\\n FlowExporter: true|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + + // Delete and re-deploy antrea for config map settings to take effect. + // Question: Can end-to-end tests run in parallel? Is there an issue deleting Antrea daemon set? + // TODO: Remove this when configmap can be changed runtime + if err := data.deleteAntrea(defaultTimeout); err != nil { + return err + } + if err := data.deployAntreaCommon(antreaYML, ""); err != nil { + return err + } + + // Change the yaml file back for other tests + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowCollectorAddr: \"%s\"|#flowCollectorAddr: \"\"|g' %s", ipfixCollector, antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|pollAndExportInterval: \"1s:5s\"|#pollAndExportInterval: \"\"|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|featureGates:\\n FlowExporter: true|#featureGates:|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + + return nil +} + // waitForAntreaDaemonSetPods waits for the K8s apiserver to report that all the Antrea Pods are // available, i.e. all the Nodes have one or more of the Antrea daemon Pod running and available. func (data *TestData) waitForAntreaDaemonSetPods(timeout time.Duration) error { @@ -467,7 +519,7 @@ func getImageName(uri string) string { // createPodOnNode creates a pod in the test namespace with a container whose type is decided by imageName. // Pod will be scheduled on the specified Node (if nodeName is not empty). -func (data *TestData) createPodOnNode(name string, nodeName string, image string, command []string, args []string, env []corev1.EnvVar, ports []corev1.ContainerPort) error { +func (data *TestData) createPodOnNode(name string, nodeName string, image string, command []string, args []string, env []corev1.EnvVar, ports []corev1.ContainerPort, hostNetwork bool) error { // image could be a fully qualified URI which can't be used as container name and label value, // extract the image name from it. imageName := getImageName(image) @@ -484,6 +536,7 @@ func (data *TestData) createPodOnNode(name string, nodeName string, image string }, }, RestartPolicy: corev1.RestartPolicyNever, + HostNetwork: hostNetwork, } if nodeName != "" { podSpec.NodeSelector = map[string]string{ @@ -519,7 +572,7 @@ func (data *TestData) createPodOnNode(name string, nodeName string, image string // Pod will be scheduled on the specified Node (if nodeName is not empty). func (data *TestData) createBusyboxPodOnNode(name string, nodeName string) error { sleepDuration := 3600 // seconds - return data.createPodOnNode(name, nodeName, "busybox", []string{"sleep", strconv.Itoa(sleepDuration)}, nil, nil, nil) + return data.createPodOnNode(name, nodeName, "busybox", []string{"sleep", strconv.Itoa(sleepDuration)}, nil, nil, nil, false) } // createBusyboxPod creates a Pod in the test namespace with a single busybox container. @@ -536,7 +589,7 @@ func (data *TestData) createNginxPodOnNode(name string, nodeName string) error { ContainerPort: 80, Protocol: corev1.ProtocolTCP, }, - }) + }, false) } // createNginxPod creates a Pod in the test namespace with a single nginx container. @@ -555,7 +608,7 @@ func (data *TestData) createServerPod(name string, portName string, portNum int, // If hostPort is to be set, it must match the container port number. port.HostPort = int32(portNum) } - return data.createPodOnNode(name, "", image, nil, []string{cmd}, []corev1.EnvVar{env}, []corev1.ContainerPort{port}) + return data.createPodOnNode(name, "", image, nil, []string{cmd}, []corev1.EnvVar{env}, []corev1.ContainerPort{port}, false) } // deletePod deletes a Pod in the test namespace. diff --git a/test/e2e/providers/exec/docker.go b/test/e2e/providers/exec/docker.go index e2659a6609b..a5d0bd63f14 100644 --- a/test/e2e/providers/exec/docker.go +++ b/test/e2e/providers/exec/docker.go @@ -32,7 +32,13 @@ func RunDockerExecCommand(container string, cmd string, workdir string) ( ) { args := make([]string, 0) args = append(args, "exec", "-w", workdir, "-t", container) - args = append(args, strings.Fields(cmd)...) + if strings.Contains(cmd, "/bin/sh") { + // Just split in to "/bin/sh" "-c" and "actual_cmd" + // This is useful for passing piped commands in to exec + args = append(args, strings.SplitN(cmd, " ", 3)...) + } else { + args = append(args, strings.Fields(cmd)...) + } dockerCmd := exec.Command("docker", args...) stdoutPipe, err := dockerCmd.StdoutPipe() if err != nil { From 9240e7e83050d040a50212783347ae84499cf808 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Wed, 1 Jul 2020 16:38:11 -0700 Subject: [PATCH 04/15] Fixed memory lead issue and addressed new comments Regenerated manifest files too. --- build/images/ipfixcollector/README.md | 3 +- build/yamls/antrea-eks.yml | 12 ++ build/yamls/antrea-gke.yml | 12 ++ build/yamls/antrea-ipsec.yml | 12 ++ build/yamls/antrea.yml | 12 ++ cmd/antrea-agent/main.go | 4 + cmd/antrea-agent/options.go | 112 +++++++++--------- .../flowexporter/connections/connections.go | 14 ++- .../connections/conntrack_linux.go | 8 +- pkg/agent/flowexporter/exporter/exporter.go | 52 ++++---- .../{flowrecords.go => flow_records.go} | 7 +- .../{ipfixprocess.go => ipfix_process.go} | 0 .../ipfix/{ipfixrecord.go => ipfix_record.go} | 0 13 files changed, 160 insertions(+), 88 deletions(-) rename pkg/agent/flowexporter/flowrecords/{flowrecords.go => flow_records.go} (90%) rename pkg/agent/flowexporter/ipfix/{ipfixprocess.go => ipfix_process.go} (100%) rename pkg/agent/flowexporter/ipfix/{ipfixrecord.go => ipfix_record.go} (100%) diff --git a/build/images/ipfixcollector/README.md b/build/images/ipfixcollector/README.md index 0108b1a55be..9f5ed99a73b 100644 --- a/build/images/ipfixcollector/README.md +++ b/build/images/ipfixcollector/README.md @@ -1,7 +1,6 @@ # images/ipfixcollector -This Docker image is a very lightweight image based on Ubuntu 18.04 which -includes ipfix collector based on libipfix, a C library. +This Docker image is based on Ubuntu 18.04 which includes ipfix collector based on libipfix, a C library. In this image, IPFIX collector listening on tcp:4739 port. libipfix package is downloaded from https://svwh.dl.sourceforge.net/project/libipfix/libipfix/libipfix-impd4e_110224.tgz diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 767080c872c..ca4de457e3b 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -722,6 +722,18 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + + # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. + # Defaults to "". + #flowCollectorAddr: "" + + # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + # in conntrack module and exports IPFIX flow records that are built from connection store. + # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. + # Any value in range (PollInterval(s), 600s] for export interval is acceptable. + # Defaults to "5s:60s". Follow the time units of duration. + #pollAndExportInterval: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 96a35f750e4..b42370e2fae 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -722,6 +722,18 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + + # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. + # Defaults to "". + #flowCollectorAddr: "" + + # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + # in conntrack module and exports IPFIX flow records that are built from connection store. + # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. + # Any value in range (PollInterval(s), 600s] for export interval is acceptable. + # Defaults to "5s:60s". Follow the time units of duration. + #pollAndExportInterval: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 61f2e833a57..1dc1d9d6485 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -722,6 +722,18 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + + # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. + # Defaults to "". + #flowCollectorAddr: "" + + # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + # in conntrack module and exports IPFIX flow records that are built from connection store. + # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. + # Any value in range (PollInterval(s), 600s] for export interval is acceptable. + # Defaults to "5s:60s". Follow the time units of duration. + #pollAndExportInterval: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 09100f1f0a5..a15409aa95e 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -722,6 +722,18 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + + # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. + # Defaults to "". + #flowCollectorAddr: "" + + # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + # in conntrack module and exports IPFIX flow records that are built from connection store. + # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. + # Any value in range (PollInterval(s), 600s] for export interval is acceptable. + # Defaults to "5s:60s". Follow the time units of duration. + #pollAndExportInterval: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", diff --git a/cmd/antrea-agent/main.go b/cmd/antrea-agent/main.go index 9e641d9225c..2b9c68a5c2f 100644 --- a/cmd/antrea-agent/main.go +++ b/cmd/antrea-agent/main.go @@ -62,6 +62,10 @@ func newAgentCommand() *cobra.Command { if err := opts.validate(args); err != nil { klog.Fatalf("Failed to validate: %v", err) } + // Not passing args again as it is already validated and not used in flow exporter config + if err := opts.validateFlowExporterConfig(); err != nil { + klog.Fatalf("Failed to validate flow exporter config: %v", err) + } if err := run(opts); err != nil { klog.Fatalf("Error running agent: %v", err) } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 535778d801d..72500fd96a1 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -103,59 +103,6 @@ func (o *Options) validate(args []string) error { if encapMode.SupportsNoEncap() && o.config.EnableIPSecTunnel { return fmt.Errorf("IPSec tunnel may only be enabled on %s mode", config.TrafficEncapModeEncap) } - if o.config.FlowCollectorAddr != "" && features.DefaultFeatureGate.Enabled(features.FlowExporter) { - // Check if it is TCP or UDP - strSlice := strings.Split(o.config.FlowCollectorAddr, ":") - var proto string - if len(strSlice) == 2 { - // No separator "." and proto is given - proto = "tcp" - } else if len(strSlice) > 2 { - if strSlice[2] == "udp" { - proto = "udp" - } else { - // All other cases default proto is tcp - proto = "tcp" - } - } else { - return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) - } - // Convert the string input in net.Addr format - hostPortAddr := strSlice[0] + ":" + strSlice[1] - _, _, err := net.SplitHostPort(hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) - } - if proto == "udp" { - o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved: %v", err) - } - } else { - o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved: %v", err) - } - } - - if o.config.PollAndExportInterval != "" { - intervalSlice := strings.Split(o.config.PollAndExportInterval, ":") - if len(intervalSlice) != 2 { - return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.PollAndExportInterval) - } - o.pollingInterval, err = time.ParseDuration(intervalSlice[0]) - if err != nil { - return fmt.Errorf("poll interval is not provided in right format: %v", err) - } - o.exportInterval, err = time.ParseDuration(intervalSlice[1]) - if err != nil { - return fmt.Errorf("export interval is not provided in right format: %v", err) - } - if o.pollingInterval > o.exportInterval { - return fmt.Errorf("poll interval should be less than or equal to export interval") - } - } - } return nil } @@ -210,3 +157,62 @@ func (o *Options) setDefaults() { o.exportInterval = 60 * time.Second } } + +func (o *Options) validateFlowExporterConfig() error { + if features.DefaultFeatureGate.Enabled(features.FlowExporter) { + if o.config.FlowCollectorAddr != "" { + // Check if it is TCP or UDP + strSlice := strings.Split(o.config.FlowCollectorAddr, ":") + var proto string + if len(strSlice) == 2 { + // No separator "." and proto is given + proto = "tcp" + } else if len(strSlice) > 2 { + if strSlice[2] == "udp" { + proto = "udp" + } else { + // All other cases default proto is tcp + proto = "tcp" + } + } else { + return fmt.Errorf("IPFIX flow collector is given in invalid format") + } + // Convert the string input in net.Addr format + hostPortAddr := strSlice[0] + ":" + strSlice[1] + _, _, err := net.SplitHostPort(hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + } + if proto == "udp" { + o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved: %v", err) + } + } else { + o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved: %v", err) + } + } + + if o.config.PollAndExportInterval != "" { + intervalSlice := strings.Split(o.config.PollAndExportInterval, ":") + if len(intervalSlice) != 2 { + return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.PollAndExportInterval) + } + o.pollingInterval, err = time.ParseDuration(intervalSlice[0]) + if err != nil { + return fmt.Errorf("poll interval is not provided in right format: %v", err) + } + o.exportInterval, err = time.ParseDuration(intervalSlice[1]) + if err != nil { + return fmt.Errorf("export interval is not provided in right format: %v", err) + } + if o.pollingInterval > o.exportInterval { + return fmt.Errorf("poll interval should be less than or equal to export interval") + } + } + } + } + return nil +} diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index bd2edff2d74..8ea5fd6ed42 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -91,7 +91,7 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn.ReversePackets = conn.ReversePackets // Reassign the flow to update the map cs.connections[connKey] = *existingConn - klog.V(2).Infof("Antrea flow updated: %v", existingConn) + klog.V(4).Infof("Antrea flow updated: %v", existingConn) } else { var srcFound, dstFound bool sIface, srcFound := cs.ifaceStore.GetInterfaceByIP(conn.TupleOrig.SourceAddress.String()) @@ -108,7 +108,7 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodName = dIface.ContainerInterfaceConfig.PodName conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } - klog.V(2).Infof("New Antrea flow added: %v", conn) + klog.V(4).Infof("New Antrea flow added: %v", conn) // Add new antrea connection to connection store cs.connections[connKey] = *conn } @@ -127,6 +127,10 @@ func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowR for k, v := range cs.connections { cs.mutex.Unlock() + // Releasing lock as there are no concurrent deletes. There can be concurrent add or modify. We may not consider the + // newly added or modified field and send old connection data collected in last poll cycle. + // This has to be changed when flushing of connection data logic changes. + // Followed this go documentation (point#3 on iteration over map): https://golang.org/ref/spec#For_statements_with_range_clause err := updateCallback(k, v) if err != nil { klog.Errorf("Update callback failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) @@ -150,14 +154,16 @@ func (cs *connectionStore) poll() (int, error) { for _, conn := range filteredConns { cs.addOrUpdateConn(conn) } + connsLen := len(filteredConns) + filteredConns = nil + klog.V(2).Infof("Conntrack polling successful") - return len(filteredConns), nil + return connsLen, nil } // FlushConnectionStore after each IPFIX export of flow records. // Timed out conntrack connections will not be sent as IPFIX flow records. -// TODO: Enhance/optimize this logic. func (cs *connectionStore) FlushConnectionStore() { klog.Infof("Flushing connection map") diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 11fc0b38e6e..86da6a1ccfc 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -19,7 +19,6 @@ package connections import ( "fmt" "net" - "os" "strconv" "strings" @@ -57,6 +56,7 @@ if ctdump.datapathType == ovsconfig.OVSDatapathSystem { // Link to issue: https://github.com/ti-mo/conntrack/issues/23 // Dump all flows in the conntrack table for now. var conns []*flowexporter.Connection + var err error if ctdump.datapathType == ovsconfig.OVSDatapathSystem { conns, err = ctdump.connTrack.DumpFilter(conntrack.Filter{}) if err != nil { @@ -128,7 +128,7 @@ type connTrackSystem struct { netlinkConn *conntrack.Conn } -func NewConnTrackInterfacer() *connTrackSystem { +func NewConnTrackSystem() *connTrackSystem { // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of connections. // Do not handle error and continue with creation of interfacer object as we can still dump flows with no stats. // If log says permission error, please ensure net.netfilter.nf_conntrack_acct to be set to 1. @@ -177,7 +177,11 @@ func (ctnl *connTrackSystem) DumpFilter(filter interface{}) ([]*flowexporter.Con for i, conn := range conns { antreaConns[i] = createAntreaConn(&conn) } + conns = nil + klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + + ctnl.netlinkConn.Close() return antreaConns, nil } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 5bb19b6cb7e..6cbb3151402 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -44,6 +44,11 @@ var ( "octetTotalCount", "packetDeltaCount", "octetDeltaCount", + } + // Substring "reverse" is an indication to get reverse element of go-ipfix library. + // Specifically using GetReverseInfoElement, which is part of implementations of GetIANARegistryInfoElement and + // GetAntreaRegistryInfoElement. + IANAReverseInfoElements = []string{ "reverse_PacketTotalCount", "reverse_OctetTotalCount", "reverse_PacketDeltaCount", @@ -151,7 +156,7 @@ func (exp *flowExporter) sendFlowRecords() error { func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, error) { // Initialize this every time new template is added - exp.elementsList = make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(AntreaInfoElements)) + exp.elementsList = make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)) // Add template header _, err := templateRec.PrepareRecord() if err != nil { @@ -159,37 +164,37 @@ func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, } for i, ie := range IANAInfoElements { - var element *ipfixentities.InfoElement - var err error - if !strings.Contains(ie, "reverse") { - element, err = exp.process.GetIANARegistryInfoElement(ie, false) - if err != nil { - return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) - } - } else { - split := strings.Split(ie, "_") - runeStr := []rune(split[1]) - runeStr[0] = unicode.ToLower(runeStr[0]) - element, err = exp.process.GetIANARegistryInfoElement(string(runeStr), true) - if err != nil { - return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) - } - } - _, err = templateRec.AddInfoElement(element, nil) + element, err := exp.process.GetIANARegistryInfoElement(ie, false) if err != nil { - // Add error interface to IPFIX library in future to avoid overloading of fmt.Errorf. + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + if _, err = templateRec.AddInfoElement(element, nil); err != nil { return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) } exp.elementsList[i] = element } - + for i, ie := range IANAReverseInfoElements { + split := strings.Split(ie, "_") + runeStr := []rune(split[1]) + runeStr[0] = unicode.ToLower(runeStr[0]) + element, err := exp.process.GetIANARegistryInfoElement(string(runeStr), true) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + if _, err = templateRec.AddInfoElement(element, nil); err != nil { + return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) + } + exp.elementsList[i+len(IANAInfoElements)] = element + } for i, ie := range AntreaInfoElements { element, err := exp.process.GetAntreaRegistryInfoElement(ie, false) if err != nil { return 0, fmt.Errorf("information element %s is not present in Antrea registry", ie) } - templateRec.AddInfoElement(element, nil) - exp.elementsList[i+len(IANAInfoElements)] = element + if _, err := templateRec.AddInfoElement(element, nil); err != nil { + return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) + } + exp.elementsList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)] = element } sentBytes, err := exp.process.AddRecordAndSendMsg(ipfixentities.Template, templateRec.GetRecord()) @@ -264,13 +269,12 @@ func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowex return fmt.Errorf("error while adding info element: %s to data record: %v", ie.Name, err) } } - klog.V(2).Infof("Flow data record created. Number of fields: %d, Bytes added: %d", dataRec.GetFieldCount(), dataRec.GetBuffer().Len()) sentBytes, err := exp.process.AddRecordAndSendMsg(ipfixentities.Data, dataRec.GetRecord()) if err != nil { return fmt.Errorf("error in IPFIX exporting process when sending data record: %v", err) } - klog.V(2).Infof("Flow record sent successfully. Bytes sent: %d", sentBytes) + klog.V(4).Infof("Flow record created and sent. Bytes sent: %d", sentBytes) return nil } diff --git a/pkg/agent/flowexporter/flowrecords/flowrecords.go b/pkg/agent/flowexporter/flowrecords/flow_records.go similarity index 90% rename from pkg/agent/flowexporter/flowrecords/flowrecords.go rename to pkg/agent/flowexporter/flowrecords/flow_records.go index 3fd89b02423..889037fcf08 100644 --- a/pkg/agent/flowexporter/flowrecords/flowrecords.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -50,7 +50,7 @@ func (fr *flowRecords) BuildFlowRecords() error { if err != nil { return fmt.Errorf("error in iterating cxn map: %v", err) } - klog.V(2).Infof("Flow records that are built: %d", len(fr.recordsMap)) + klog.V(2).Infof("No. of flow records built: %d", len(fr.recordsMap)) return nil } @@ -68,8 +68,10 @@ func (fr *flowRecords) IterateFlowRecordsWithSendCB(sendCallback flowexporter.Fl v.PrevReversePackets = v.Conn.ReversePackets v.PrevReverseBytes = v.Conn.ReverseBytes fr.recordsMap[k] = v - klog.V(2).Infof("Flow record sent successfully") } + // Flush connection map once all flow records are sent. + // TODO: Optimize this logic by flushing individual connections based on their timeout values. + fr.connStoreBuilder.FlushConnectionStore() return nil } @@ -88,6 +90,5 @@ func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, con record.Conn = &conn } fr.recordsMap[key] = record - klog.V(2).Infof("Flow record added or updated: %v", record) return nil } diff --git a/pkg/agent/flowexporter/ipfix/ipfixprocess.go b/pkg/agent/flowexporter/ipfix/ipfix_process.go similarity index 100% rename from pkg/agent/flowexporter/ipfix/ipfixprocess.go rename to pkg/agent/flowexporter/ipfix/ipfix_process.go diff --git a/pkg/agent/flowexporter/ipfix/ipfixrecord.go b/pkg/agent/flowexporter/ipfix/ipfix_record.go similarity index 100% rename from pkg/agent/flowexporter/ipfix/ipfixrecord.go rename to pkg/agent/flowexporter/ipfix/ipfix_record.go From 7495f1e356e2e60b6935072bf3af091dbf77fac9 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Tue, 7 Jul 2020 16:24:36 -0700 Subject: [PATCH 05/15] Resolving conflicts with ToT and address template refresh issue Changed synchronization logic for poll(ConnectionStore.Run) and export(FlowExporter.Run) go routines. Fixed e2e tests. --- cmd/antrea-agent/agent.go | 7 +- cmd/antrea-agent/options.go | 6 ++ go.mod | 2 +- go.sum | 4 +- .../flowexporter/connections/connections.go | 65 ++++++++++--------- .../connections/conntrack_linux.go | 2 +- pkg/agent/flowexporter/exporter/exporter.go | 53 +++++++++++---- .../flowexporter/exporter/exporter_test.go | 44 ++++++------- .../flowexporter/flowrecords/flow_records.go | 33 ++++++---- pkg/agent/flowexporter/ipfix/ipfix_process.go | 18 ++--- pkg/agent/flowexporter/ipfix/ipfix_record.go | 9 +++ .../flowexporter/ipfix/testing/mock_ipfix.go | 48 +++++++++----- pkg/agent/flowexporter/types.go | 7 +- plugins/octant/go.sum | 2 +- test/e2e/flowexporter_test.go | 4 +- test/e2e/framework.go | 9 +-- test/e2e/proxy_test.go | 2 +- 17 files changed, 188 insertions(+), 127 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 4549947e143..1d6bc924917 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -246,15 +246,16 @@ func run(o *Options) error { } else if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackNetdev(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) } - connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval) + connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval, o.exportInterval) flowRecords := flowrecords.NewFlowRecords(connStore) flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords, o.exportInterval) if err != nil { // If flow exporter cannot be initialized, then Antrea agent does not exit; only error is logged. klog.Errorf("error when initializing flow exporter: %v", err) } else { - go connStore.Run(stopCh) - go flowExporter.Run(stopCh) + pollDone := make(chan bool, 1) + go connStore.Run(stopCh, pollDone) + go flowExporter.Run(stopCh, pollDone) } } } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 72500fd96a1..2a84d9e6dfe 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -208,9 +208,15 @@ func (o *Options) validateFlowExporterConfig() error { if err != nil { return fmt.Errorf("export interval is not provided in right format: %v", err) } + if o.pollingInterval < time.Second { + return fmt.Errorf("poll interval should be minimum of one second") + } if o.pollingInterval > o.exportInterval { return fmt.Errorf("poll interval should be less than or equal to export interval") } + if o.exportInterval%o.pollingInterval != 0 { + return fmt.Errorf("export interval should be a multiple of poll interval") + } } } } diff --git a/go.mod b/go.mod index d8979d6ced8..35017dc8414 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c + github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect github.com/stretchr/testify v1.5.1 github.com/ti-mo/conntrack v0.3.0 diff --git a/go.sum b/go.sum index 89ab336bcd0..93b6fcb524f 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c h1:/yq/4iWgtsFZPe685ytq2OWtDNKWPT2MpDDUdP1oYdM= -github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= +github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c h1:q4GI08OoU/6n9GHsSIg8QEqNo9HAVfyR7IbAOqj1nDA= +github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 h1:XklXvOrWxWCDX2n4vdEQWkjuIP820XD6C4kF0O0FzH4= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 8ea5fd6ed42..76d14120592 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -15,7 +15,6 @@ package connections import ( - "sync" "time" "k8s.io/klog/v2" @@ -28,46 +27,64 @@ import ( var _ ConnectionStore = new(connectionStore) type ConnectionStore interface { - Run(stopCh <-chan struct{}) + Run(stopCh <-chan struct{}, pollDone chan bool) IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error FlushConnectionStore() } type connectionStore struct { - connections map[flowexporter.ConnectionKey]flowexporter.Connection // Add 5-tuple as string array - connDumper ConnTrackDumper - ifaceStore interfacestore.InterfaceStore - pollInterval time.Duration - mutex sync.Mutex + // pollDone channel is used for synchronization of poll(ConnectionStore.Run) and export(FlowExporter.Run) go routines. + // Therefore, there is no requirement of lock to make connections map thread safe. + connections map[flowexporter.ConnectionKey]flowexporter.Connection // Add 5-tuple as string array + connDumper ConnTrackDumper + ifaceStore interfacestore.InterfaceStore + pollInterval time.Duration + exportInterval time.Duration } -func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, interval time.Duration) *connectionStore { +func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration, exportInterval time.Duration) *connectionStore { return &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: ctDumper, - ifaceStore: ifaceStore, - pollInterval: interval, + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: ctDumper, + ifaceStore: ifaceStore, + pollInterval: pollInterval, + exportInterval: exportInterval, } } // Run polls the connTrackDumper module periodically to get connections. These connections are used // to build connection store. -func (cs *connectionStore) Run(stopCh <-chan struct{}) { +func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { klog.Infof("Starting conntrack polling") - ticker := time.NewTicker(cs.pollInterval) - defer ticker.Stop() + pollTicker := time.NewTicker(cs.pollInterval) + defer pollTicker.Stop() + exportTrigger := uint(cs.exportInterval / cs.pollInterval) + exportCounter := uint(0) + for { select { case <-stopCh: break - case <-ticker.C: + case <-pollTicker.C: + if exportCounter == 0 { + exportCounter = exportTrigger + // Flush connection map once all flow records are sent. + // TODO: Optimize this logic by flushing individual connections based on the individual timeout values. + cs.FlushConnectionStore() + } _, err := cs.poll() if err != nil { // Not failing here as errors can be transient and could be resolved in future poll cycles. // TODO: Come up with a backoff/retry mechanism by increasing poll interval and adding retry timeout klog.Errorf("Error during conntrack poll cycle: %v", err) } + exportCounter = exportCounter - 1 + if exportCounter == 0 { + // We need synchronization between ConnectionStore.Run and FlowExporter.Run go routines. pollDone channel provides that synchronization. + // More details in exporter/exporter.go. + pollDone <- true + } } } } @@ -79,8 +96,6 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn, exists := cs.getConnByKey(connKey) - cs.mutex.Lock() - defer cs.mutex.Unlock() if exists { // Update the necessary fields that are used in generating flow records. // Can same 5-tuple flow get deleted and added to conntrack table? If so use ID. @@ -115,28 +130,18 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { } func (cs *connectionStore) getConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { - cs.mutex.Lock() - defer cs.mutex.Unlock() conn, found := cs.connections[flowTuple] return &conn, found } func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error { - cs.mutex.Lock() - defer cs.mutex.Unlock() - for k, v := range cs.connections { - cs.mutex.Unlock() - // Releasing lock as there are no concurrent deletes. There can be concurrent add or modify. We may not consider the - // newly added or modified field and send old connection data collected in last poll cycle. - // This has to be changed when flushing of connection data logic changes. - // Followed this go documentation (point#3 on iteration over map): https://golang.org/ref/spec#For_statements_with_range_clause + klog.V(4).Infof("After: iterating flow with key: %v, conn: %v", k, v) err := updateCallback(k, v) if err != nil { klog.Errorf("Update callback failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) return err } - cs.mutex.Lock() } return nil } @@ -167,8 +172,6 @@ func (cs *connectionStore) poll() (int, error) { func (cs *connectionStore) FlushConnectionStore() { klog.Infof("Flushing connection map") - cs.mutex.Lock() - defer cs.mutex.Unlock() for conn := range cs.connections { delete(cs.connections, conn) } diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 86da6a1ccfc..0c0e6f8ebc2 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -36,7 +36,7 @@ import ( // DumpFlows opens netlink connection and dumps all the flows in Antrea ZoneID // of conntrack table, i.e., corresponding to Antrea OVS bridge. func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { -if ctdump.datapathType == ovsconfig.OVSDatapathSystem { + if ctdump.datapathType == ovsconfig.OVSDatapathSystem { // Get connection to netlink socket err := ctdump.connTrack.GetConnTrack(nil) if err != nil { diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 6cbb3151402..cd70cc89e33 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -57,15 +57,17 @@ var ( AntreaInfoElements = []string{ "sourcePodName", "sourcePodNamespace", + "sourceNodeName", "destinationPodName", "destinationPodNamespace", + "destinationNodeName", } ) var _ FlowExporter = new(flowExporter) type FlowExporter interface { - Run(stopCh <-chan struct{}) + Run(stopCh <-chan struct{}, pollDone <-chan bool) } type flowExporter struct { @@ -93,7 +95,14 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn return nil, fmt.Errorf("cannot generate obsID for IPFIX ipfixexport: %v", err) } - expProcess, err := ipfix.NewIPFIXExportingProcess(collector, obsID) + var expProcess ipfix.IPFIXExportingProcess + if collector.Network() == "tcp" { + // TCP transport do not need any tempRefTimeout, so sending 0. + expProcess, err = ipfix.NewIPFIXExportingProcess(collector, obsID, 0) + } else { + // For UDP transport, hardcoding tempRefTimeout value as 1800s. + expProcess, err = ipfix.NewIPFIXExportingProcess(collector, obsID, 1800) + } if err != nil { return nil, fmt.Errorf("error while initializing IPFIX exporting expProcess: %v", err) } @@ -107,8 +116,8 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn 0, } - flowExp.templateID = flowExp.process.AddTemplate() - templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(AntreaInfoElements)), flowExp.templateID) + flowExp.templateID = flowExp.process.NewTemplateID() + templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), flowExp.templateID) sentBytes, err := flowExp.sendTemplateRecord(templateRec) if err != nil { @@ -119,7 +128,7 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn return flowExp, nil } -func (exp *flowExporter) Run(stopCh <-chan struct{}) { +func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan bool) { klog.Infof("Start exporting IPFIX flow records") ticker := time.NewTicker(exp.exportInterval) defer ticker.Stop() @@ -130,6 +139,11 @@ func (exp *flowExporter) Run(stopCh <-chan struct{}) { exp.process.CloseConnToCollector() break case <-ticker.C: + // Waiting for pollDone channel is necessary because IPFIX collector computes throughput based on + // flow records received interval, therefore we need to wait for poll go routine (ConnectionStore.Run). + // pollDone provides this synchronization. Note that export interval is multiple of poll interval, and poll + // interval is at least one second long. + <-pollDone err := exp.flowRecords.BuildFlowRecords() if err != nil { klog.Errorf("Error when building flow records: %v", err) @@ -155,15 +169,13 @@ func (exp *flowExporter) sendFlowRecords() error { } func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, error) { - // Initialize this every time new template is added - exp.elementsList = make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)) // Add template header _, err := templateRec.PrepareRecord() if err != nil { return 0, fmt.Errorf("error when writing template header: %v", err) } - for i, ie := range IANAInfoElements { + for _, ie := range IANAInfoElements { element, err := exp.process.GetIANARegistryInfoElement(ie, false) if err != nil { return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) @@ -171,9 +183,8 @@ func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, if _, err = templateRec.AddInfoElement(element, nil); err != nil { return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) } - exp.elementsList[i] = element } - for i, ie := range IANAReverseInfoElements { + for _, ie := range IANAReverseInfoElements { split := strings.Split(ie, "_") runeStr := []rune(split[1]) runeStr[0] = unicode.ToLower(runeStr[0]) @@ -184,9 +195,8 @@ func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, if _, err = templateRec.AddInfoElement(element, nil); err != nil { return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) } - exp.elementsList[i+len(IANAInfoElements)] = element } - for i, ie := range AntreaInfoElements { + for _, ie := range AntreaInfoElements { element, err := exp.process.GetAntreaRegistryInfoElement(ie, false) if err != nil { return 0, fmt.Errorf("information element %s is not present in Antrea registry", ie) @@ -194,7 +204,6 @@ func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, if _, err := templateRec.AddInfoElement(element, nil); err != nil { return 0, fmt.Errorf("error when adding %s to template: %v", element.Name, err) } - exp.elementsList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)] = element } sentBytes, err := exp.process.AddRecordAndSendMsg(ipfixentities.Template, templateRec.GetRecord()) @@ -202,10 +211,14 @@ func (exp *flowExporter) sendTemplateRecord(templateRec ipfix.IPFIXRecord) (int, return 0, fmt.Errorf("error in IPFIX exporting process when sending template record: %v", err) } + // Get all elements from template record. + exp.elementsList = templateRec.GetTemplateElements() + return sentBytes, nil } func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowexporter.FlowRecord) error { + nodeName, _ := env.GetNodeName() // Iterate over all infoElements in the list for _, ie := range exp.elementsList { var err error @@ -260,10 +273,24 @@ func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowex _, err = dataRec.AddInfoElement(ie, record.Conn.SourcePodNamespace) case "sourcePodName": _, err = dataRec.AddInfoElement(ie, record.Conn.SourcePodName) + case "sourceNodeName": + // Add nodeName for only local pods whose pod names are resolved. + if record.Conn.SourcePodName != "" { + _, err = dataRec.AddInfoElement(ie, nodeName) + } else { + _, err = dataRec.AddInfoElement(ie, "") + } case "destinationPodNamespace": _, err = dataRec.AddInfoElement(ie, record.Conn.DestinationPodNamespace) case "destinationPodName": _, err = dataRec.AddInfoElement(ie, record.Conn.DestinationPodName) + case "destinationNodeName": + // Add nodeName for only local pods whose pod names are resolved. + if record.Conn.DestinationPodName != "" { + _, err = dataRec.AddInfoElement(ie, nodeName) + } else { + _, err = dataRec.AddInfoElement(ie, "") + } } if err != nil { return fmt.Errorf("error while adding info element: %s to data record: %v", ie.Name, err) diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index ebc7a5cda95..71cf10a2e0b 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -15,7 +15,6 @@ package exporter import ( - "bytes" "strings" "testing" "time" @@ -48,6 +47,9 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { for _, ie := range IANAInfoElements { elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) } + for _, ie := range IANAReverseInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 29305, 0)) + } for _, ie := range AntreaInfoElements { elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) } @@ -57,28 +59,22 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { mockTempRec.EXPECT().PrepareRecord().Return(tempBytes, nil) for i, ie := range IANAInfoElements { - if !strings.Contains(ie, "reverse") { - mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(ie, false).Return(elemList[i], nil) - } else { - split := strings.Split(ie, "_") - runeStr := []rune(split[1]) - runeStr[0] = unicode.ToLower(runeStr[0]) - mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(string(runeStr), true).Return(elemList[i], nil) - } + mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(ie, false).Return(elemList[i], nil) mockTempRec.EXPECT().AddInfoElement(elemList[i], nil).Return(tempBytes, nil) } - for i, ie := range AntreaInfoElements { - if !strings.Contains(ie, "reverse") { - mockIPFIXExpProc.EXPECT().GetAntreaRegistryInfoElement(ie, false).Return(elemList[i+len(IANAInfoElements)], nil) - } else { - split := strings.Split(ie, "_") - runeStr := []rune(split[1]) - runeStr[0] = unicode.ToLower(runeStr[0]) - mockIPFIXExpProc.EXPECT().GetAntreaRegistryInfoElement(string(runeStr), true).Return(elemList[i+len(IANAInfoElements)], nil) - } + for i, ie := range IANAReverseInfoElements { + split := strings.Split(ie, "_") + runeStr := []rune(split[1]) + runeStr[0] = unicode.ToLower(runeStr[0]) + mockIPFIXExpProc.EXPECT().GetIANARegistryInfoElement(string(runeStr), true).Return(elemList[i+len(IANAInfoElements)], nil) mockTempRec.EXPECT().AddInfoElement(elemList[i+len(IANAInfoElements)], nil).Return(tempBytes, nil) } + for i, ie := range AntreaInfoElements { + mockIPFIXExpProc.EXPECT().GetAntreaRegistryInfoElement(ie, false).Return(elemList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)], nil) + mockTempRec.EXPECT().AddInfoElement(elemList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)], nil).Return(tempBytes, nil) + } mockTempRec.EXPECT().GetRecord().Return(templateRecord) + mockTempRec.EXPECT().GetTemplateElements().Return(elemList) // Passing 0 for sentBytes as it is not used anywhere in the test. In reality, this IPFIX message size // for template record of above elements. mockIPFIXExpProc.EXPECT().AddRecordAndSendMsg(ipfixentities.Template, templateRecord).Return(0, nil) @@ -88,7 +84,7 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { t.Errorf("Error in sending templated record: %v", err) } - assert.Equal(t, len(IANAInfoElements)+len(AntreaInfoElements), len(flowExp.elementsList), "flowExp.elementsList and template record should have same number of elements") + assert.Equal(t, len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements), len(flowExp.elementsList), flowExp.elementsList, "flowExp.elementsList and template record should have same number of elements") } // TestFlowExporter_sendDataRecord tests essentially if element names in the switch-case matches globals @@ -133,12 +129,15 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { } // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) // Need only element name and other are dummys - elemList := make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(AntreaInfoElements)) + elemList := make([]*ipfixentities.InfoElement, len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)) for i, ie := range IANAInfoElements { elemList[i] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) } + for i, ie := range IANAReverseInfoElements { + elemList[i+len(IANAInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 29305, 0) + } for i, ie := range AntreaInfoElements { - elemList[i+len(IANAInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) + elemList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) } mockIPFIXExpProc := ipfixtest.NewMockIPFIXExportingProcess(ctrl) mockDataRec := ipfixtest.NewMockIPFIXRecord(ctrl) @@ -151,7 +150,6 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { } // Expect calls required var dataRecord ipfixentities.Record - var dataBuff bytes.Buffer tempBytes := uint16(0) for i, ie := range flowExp.elementsList { // Could not come up with a way to exclude if else conditions as different IEs have different data types. @@ -175,8 +173,6 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { mockDataRec.EXPECT().AddInfoElement(ie, "").Return(tempBytes, nil) } } - mockDataRec.EXPECT().GetFieldCount().Return(uint16(len(flowExp.elementsList))) - mockDataRec.EXPECT().GetBuffer().Return(&dataBuff) mockDataRec.EXPECT().GetRecord().Return(dataRecord) mockIPFIXExpProc.EXPECT().AddRecordAndSendMsg(ipfixentities.Data, dataRecord).Return(0, nil) diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index 889037fcf08..8c050afafd3 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -56,22 +56,25 @@ func (fr *flowRecords) BuildFlowRecords() error { func (fr *flowRecords) IterateFlowRecordsWithSendCB(sendCallback flowexporter.FlowRecordSend, templateID uint16) error { for k, v := range fr.recordsMap { - dataRec := ipfix.NewIPFIXDataRecord(templateID) - err := sendCallback(dataRec, v) - if err != nil { - klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) - return err + if v.IsActive == true { + dataRec := ipfix.NewIPFIXDataRecord(templateID) + err := sendCallback(dataRec, v) + if err != nil { + klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) + return err + } + // Update the flow record after it is sent successfully + v.PrevPackets = v.Conn.OriginalPackets + v.PrevBytes = v.Conn.OriginalBytes + v.PrevReversePackets = v.Conn.ReversePackets + v.PrevReverseBytes = v.Conn.ReverseBytes + v.IsActive = false + fr.recordsMap[k] = v + } else { + // Delete flow record as the corresponding connection is not present in conntrack table. + delete(fr.recordsMap, k) } - // Update the flow record after it is sent successfully - v.PrevPackets = v.Conn.OriginalPackets - v.PrevBytes = v.Conn.OriginalBytes - v.PrevReversePackets = v.Conn.ReversePackets - v.PrevReverseBytes = v.Conn.ReverseBytes - fr.recordsMap[k] = v } - // Flush connection map once all flow records are sent. - // TODO: Optimize this logic by flushing individual connections based on their timeout values. - fr.connStoreBuilder.FlushConnectionStore() return nil } @@ -85,9 +88,11 @@ func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, con 0, 0, 0, + true, } } else { record.Conn = &conn + record.IsActive = true } fr.recordsMap[key] = record return nil diff --git a/pkg/agent/flowexporter/ipfix/ipfix_process.go b/pkg/agent/flowexporter/ipfix/ipfix_process.go index 6e3ee65172a..9641d4cdab7 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_process.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_process.go @@ -29,9 +29,9 @@ type IPFIXExportingProcess interface { LoadRegistries() GetIANARegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) GetAntreaRegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) - AddTemplate() uint16 + NewTemplateID() uint16 AddRecordAndSendMsg(setType ipfixentities.ContentType, record ipfixentities.Record) (int, error) - CloseConnToCollector() error + CloseConnToCollector() } type ipfixExportingProcess struct { @@ -40,8 +40,8 @@ type ipfixExportingProcess struct { antreaReg ipfixregistry.Registry } -func NewIPFIXExportingProcess(collector net.Addr, obsID uint32) (*ipfixExportingProcess, error) { - expProcess, err := ipfixexport.InitExportingProcess(collector, obsID) +func NewIPFIXExportingProcess(collector net.Addr, obsID uint32, tempRefTimeout uint32) (*ipfixExportingProcess, error) { + expProcess, err := ipfixexport.InitExportingProcess(collector, obsID, tempRefTimeout) if err != nil { return nil, fmt.Errorf("error while initializing IPFIX exporting process: %v", err) } @@ -56,9 +56,9 @@ func (exp *ipfixExportingProcess) AddRecordAndSendMsg(setType ipfixentities.Cont return sentBytes, err } -func (exp *ipfixExportingProcess) CloseConnToCollector() error { - err := exp.ExportingProcess.CloseConnToCollector() - return err +func (exp *ipfixExportingProcess) CloseConnToCollector() { + exp.ExportingProcess.CloseConnToCollector() + return } func (exp *ipfixExportingProcess) LoadRegistries() { @@ -91,6 +91,6 @@ func (exp *ipfixExportingProcess) GetAntreaRegistryInfoElement(name string, isRe return ie, err } -func (exp *ipfixExportingProcess) AddTemplate() uint16 { - return exp.ExportingProcess.AddTemplate() +func (exp *ipfixExportingProcess) NewTemplateID() uint16 { + return exp.ExportingProcess.NewTemplateID() } diff --git a/pkg/agent/flowexporter/ipfix/ipfix_record.go b/pkg/agent/flowexporter/ipfix/ipfix_record.go index 536135b9564..5f6145947d7 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_record.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_record.go @@ -27,6 +27,7 @@ type IPFIXRecord interface { PrepareRecord() (uint16, error) AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) GetBuffer() *bytes.Buffer + GetTemplateElements() []*ipfixentities.InfoElement GetFieldCount() uint16 } @@ -69,6 +70,10 @@ func (dr *ipfixDataRecord) GetFieldCount() uint16 { return dr.dataRecord.GetFieldCount() } +func (dr *ipfixDataRecord) GetTemplateElements() []*ipfixentities.InfoElement { + return nil +} + func (tr *ipfixTemplateRecord) GetRecord() ipfixentities.Record { return tr.templateRecord } @@ -90,3 +95,7 @@ func (tr *ipfixTemplateRecord) GetBuffer() *bytes.Buffer { func (tr *ipfixTemplateRecord) GetFieldCount() uint16 { return tr.templateRecord.GetFieldCount() } + +func (tr *ipfixTemplateRecord) GetTemplateElements() []*ipfixentities.InfoElement { + return tr.templateRecord.GetTemplateElements() +} diff --git a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go index a966c4b1b3f..1d441c02872 100644 --- a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go +++ b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go @@ -64,26 +64,10 @@ func (mr *MockIPFIXExportingProcessMockRecorder) AddRecordAndSendMsg(arg0, arg1 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRecordAndSendMsg", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).AddRecordAndSendMsg), arg0, arg1) } -// AddTemplate mocks base method -func (m *MockIPFIXExportingProcess) AddTemplate() uint16 { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddTemplate") - ret0, _ := ret[0].(uint16) - return ret0 -} - -// AddTemplate indicates an expected call of AddTemplate -func (mr *MockIPFIXExportingProcessMockRecorder) AddTemplate() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTemplate", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).AddTemplate)) -} - // CloseConnToCollector mocks base method -func (m *MockIPFIXExportingProcess) CloseConnToCollector() error { +func (m *MockIPFIXExportingProcess) CloseConnToCollector() { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloseConnToCollector") - ret0, _ := ret[0].(error) - return ret0 + m.ctrl.Call(m, "CloseConnToCollector") } // CloseConnToCollector indicates an expected call of CloseConnToCollector @@ -134,6 +118,20 @@ func (mr *MockIPFIXExportingProcessMockRecorder) LoadRegistries() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRegistries", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).LoadRegistries)) } +// NewTemplateID mocks base method +func (m *MockIPFIXExportingProcess) NewTemplateID() uint16 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewTemplateID") + ret0, _ := ret[0].(uint16) + return ret0 +} + +// NewTemplateID indicates an expected call of NewTemplateID +func (mr *MockIPFIXExportingProcessMockRecorder) NewTemplateID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewTemplateID", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).NewTemplateID)) +} + // MockIPFIXRecord is a mock of IPFIXRecord interface type MockIPFIXRecord struct { ctrl *gomock.Controller @@ -214,6 +212,20 @@ func (mr *MockIPFIXRecordMockRecorder) GetRecord() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRecord", reflect.TypeOf((*MockIPFIXRecord)(nil).GetRecord)) } +// GetTemplateElements mocks base method +func (m *MockIPFIXRecord) GetTemplateElements() []*entities.InfoElement { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTemplateElements") + ret0, _ := ret[0].([]*entities.InfoElement) + return ret0 +} + +// GetTemplateElements indicates an expected call of GetTemplateElements +func (mr *MockIPFIXRecordMockRecorder) GetTemplateElements() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateElements", reflect.TypeOf((*MockIPFIXRecord)(nil).GetTemplateElements)) +} + // PrepareRecord mocks base method func (m *MockIPFIXRecord) PrepareRecord() (uint16, error) { m.ctrl.T.Helper() diff --git a/pkg/agent/flowexporter/types.go b/pkg/agent/flowexporter/types.go index e44930c337b..be5e2f6c810 100644 --- a/pkg/agent/flowexporter/types.go +++ b/pkg/agent/flowexporter/types.go @@ -40,9 +40,9 @@ type Connection struct { StartTime time.Time // For invalid and closed connections: StopTime is the time when connection was updated last. // For established connections: StopTime is latest time when it was polled. - StopTime time.Time - Zone uint16 - StatusFlag uint32 + StopTime time.Time + Zone uint16 + StatusFlag uint32 // TODO: Have a separate field for protocol. No need to keep it in Tuple. TupleOrig, TupleReply Tuple OriginalPackets, OriginalBytes uint64 @@ -60,4 +60,5 @@ type FlowRecord struct { PrevBytes uint64 PrevReversePackets uint64 PrevReverseBytes uint64 + IsActive bool } diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index 32fd7cc0274..0a22b12486c 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -441,7 +441,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/srikartati/go-ipfixlib v0.0.0-20200624191537-df05a1e72f7c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= +github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/test/e2e/flowexporter_test.go b/test/e2e/flowexporter_test.go index 33c2f86dd62..1ebee0cb0b9 100644 --- a/test/e2e/flowexporter_test.go +++ b/test/e2e/flowexporter_test.go @@ -59,7 +59,7 @@ func TestFlowExporter(t *testing.T) { bandwidth := strings.TrimSpace(stdout) // Adding some delay to make sure all the data records corresponding to iperf flow are received. - time.Sleep(500 * time.Millisecond) + time.Sleep(250 * time.Millisecond) rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs ipfix-collector -n antrea-test")) if err != nil || rc != 0 { @@ -131,5 +131,5 @@ func TestFlowExporter(t *testing.T) { assert.Equal(t, templateRecords, clusterInfo.numNodes, "Each agent should send out template record") // Single iperf resulting in two connections with separate ports. Suspecting second flow to be control flow to exchange // stats info. As 5s is export interval and iperf traffic runs for 10s, we expect 4 records. - assert.Equal(t, dataRecordsIntraNode, 4, "Iperf flow should have expected number of flow records") + assert.GreaterOrEqual(t, dataRecordsIntraNode, 4, "Iperf flow should have expected number of flow records") } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 967daccf3b7..6dd420590e5 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -242,7 +242,7 @@ func (data *TestData) createTestNamespace() error { // deleteNamespace deletes the provided namespace and waits for deletion to actually complete. func (data *TestData) deleteNamespace(namespace string, timeout time.Duration) error { var gracePeriodSeconds int64 = 0 - var propagationPolicy metav1.DeletionPropagation = metav1.DeletePropagationForeground + var propagationPolicy = metav1.DeletePropagationForeground deleteOptions := metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, PropagationPolicy: &propagationPolicy, @@ -304,6 +304,7 @@ func (data *TestData) deployAntreaIPSec() error { // deployAntreaFlowExporter deploys Antrea with flow exporter config params enabled. func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { + // May be better to change this from configmap rather than directly changing antrea manifest? // This is to add ipfixCollector address and pollAndExportInterval config params to antrea agent configmap cmd := fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowCollectorAddr: \"\"|flowCollectorAddr: \"%s\"|g' %s", ipfixCollector, antreaYML) rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), cmd) @@ -317,7 +318,7 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } // Turn on FlowExporter feature in featureGates - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#featureGates:|featureGates:\\n FlowExporter: true|g' %s", antreaYML) + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|# FlowExporter: false| FlowExporter: true|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) @@ -344,7 +345,7 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|featureGates:\\n FlowExporter: true|#featureGates:|g' %s", antreaYML) + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's| FlowExporter: true|# FlowExporter: false|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) @@ -482,7 +483,7 @@ func (data *TestData) deleteAntrea(timeout time.Duration) error { var gracePeriodSeconds int64 = 5 // Foreground deletion policy ensures that by the time the DaemonSet is deleted, there are // no Antrea Pods left. - var propagationPolicy metav1.DeletionPropagation = metav1.DeletePropagationForeground + var propagationPolicy = metav1.DeletePropagationForeground deleteOptions := metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, PropagationPolicy: &propagationPolicy, diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index aade8cf6cd8..f2ee814ee11 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -100,7 +100,7 @@ func TestProxyHairpin(t *testing.T) { skipIfProxyDisabled(t, data) nodeName := nodeName(1) - err = data.createPodOnNode("busybox", nodeName, "busybox", []string{"nc", "-lk", "-p", "80"}, nil, nil, []corev1.ContainerPort{{ContainerPort: 80, Protocol: corev1.ProtocolTCP}}) + err = data.createPodOnNode("busybox", nodeName, "busybox", []string{"nc", "-lk", "-p", "80"}, nil, nil, []corev1.ContainerPort{{ContainerPort: 80, Protocol: corev1.ProtocolTCP}}, false) require.NoError(t, err) require.NoError(t, data.podWaitForRunning(defaultTimeout, "busybox", testNamespace)) svc, err := data.createService("busybox", 80, 80, map[string]string{"antrea-e2e": "busybox"}, false, corev1.ServiceTypeClusterIP) From 1fea8a425c88ec1724b219ebbbb361f344c6d55a Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Tue, 14 Jul 2020 11:50:46 -0700 Subject: [PATCH 06/15] Fixed starting record issue First record should send 0 delta bytes/packets otherwise already established flow will show incorrect throughput (Mb/s or PPS) --- pkg/agent/flowexporter/exporter/exporter.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index cd70cc89e33..cb90804a82f 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -242,13 +242,19 @@ func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowex case "octetTotalCount": _, err = dataRec.AddInfoElement(ie, record.Conn.OriginalBytes) case "packetDeltaCount": - deltaPkts := int(record.Conn.OriginalPackets) - int(record.PrevPackets) + deltaPkts := 0 + if record.PrevPackets != 0 { + deltaPkts = int(record.Conn.OriginalPackets) - int(record.PrevPackets) + } if deltaPkts < 0 { klog.Warningf("Delta packets is not expected to be negative: %d", deltaPkts) } _, err = dataRec.AddInfoElement(ie, uint64(deltaPkts)) case "octetDeltaCount": - deltaBytes := int(record.Conn.OriginalBytes) - int(record.PrevBytes) + deltaBytes := 0 + if record.PrevBytes != 0 { + deltaBytes = int(record.Conn.OriginalBytes) - int(record.PrevBytes) + } if deltaBytes < 0 { klog.Warningf("Delta bytes is not expected to be negative: %d", deltaBytes) } @@ -258,13 +264,19 @@ func (exp *flowExporter) sendDataRecord(dataRec ipfix.IPFIXRecord, record flowex case "reverse_OctetTotalCount": _, err = dataRec.AddInfoElement(ie, record.Conn.ReverseBytes) case "reverse_PacketDeltaCount": - deltaPkts := int(record.Conn.ReversePackets) - int(record.PrevReversePackets) + deltaPkts := 0 + if record.PrevReversePackets != 0 { + deltaPkts = int(record.Conn.ReversePackets) - int(record.PrevReversePackets) + } if deltaPkts < 0 { klog.Warningf("Delta packets is not expected to be negative: %d", deltaPkts) } _, err = dataRec.AddInfoElement(ie, uint64(deltaPkts)) case "reverse_OctetDeltaCount": - deltaBytes := int(record.Conn.ReverseBytes) - int(record.PrevReverseBytes) + deltaBytes := 0 + if record.PrevReverseBytes != 0 { + deltaBytes = int(record.Conn.ReverseBytes) - int(record.PrevReverseBytes) + } if deltaBytes < 0 { klog.Warningf("Delta bytes is not expected to be negative: %d", deltaBytes) } From d3fafaefd7dd8368689f8c1fec11a52e0570c160 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Wed, 15 Jul 2020 16:23:49 -0700 Subject: [PATCH 07/15] Replaced personal ipfix library to vmware/go-ipfix library --- go.mod | 2 +- go.sum | 4 ++-- pkg/agent/flowexporter/exporter/exporter.go | 2 +- pkg/agent/flowexporter/exporter/exporter_test.go | 2 +- pkg/agent/flowexporter/ipfix/ipfix_process.go | 6 +++--- pkg/agent/flowexporter/ipfix/ipfix_record.go | 2 +- pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go | 2 +- plugins/octant/go.sum | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 35017dc8414..b800d8d41aa 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,11 @@ require ( github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect github.com/stretchr/testify v1.5.1 github.com/ti-mo/conntrack v0.3.0 github.com/vishvananda/netlink v1.1.0 + github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index 93b6fcb524f..837ceb65b15 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c h1:q4GI08OoU/6n9GHsSIg8QEqNo9HAVfyR7IbAOqj1nDA= -github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 h1:XklXvOrWxWCDX2n4vdEQWkjuIP820XD6C4kF0O0FzH4= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= @@ -386,6 +384,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f h1:XyyczLRk8+6YqYXE8v20XjbVtK415KR114IrjX9THpQ= +github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e h1:NM4NTe6Z+mF5IYlYAiEdRlY8XcMY4P6VlYqgsBhpojQ= github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index cb90804a82f..1bb1def25d0 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -24,7 +24,7 @@ import ( "time" "unicode" - ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 71cf10a2e0b..d3419aa7869 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -23,9 +23,9 @@ import ( "github.com/golang/mock/gomock" "gotest.tools/assert" - ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ipfixtest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix/testing" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" ) func TestFlowExporter_sendTemplateRecord(t *testing.T) { diff --git a/pkg/agent/flowexporter/ipfix/ipfix_process.go b/pkg/agent/flowexporter/ipfix/ipfix_process.go index 9641d4cdab7..56036e99077 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_process.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_process.go @@ -18,9 +18,9 @@ import ( "fmt" "net" - ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" - ipfixexport "github.com/srikartati/go-ipfixlib/pkg/exporter" - ipfixregistry "github.com/srikartati/go-ipfixlib/pkg/registry" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" + ipfixexport "github.com/vmware/go-ipfix/pkg/exporter" + ipfixregistry "github.com/vmware/go-ipfix/pkg/registry" ) var _ IPFIXExportingProcess = new(ipfixExportingProcess) diff --git a/pkg/agent/flowexporter/ipfix/ipfix_record.go b/pkg/agent/flowexporter/ipfix/ipfix_record.go index 5f6145947d7..31d32633073 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_record.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_record.go @@ -16,7 +16,7 @@ package ipfix import ( "bytes" - ipfixentities "github.com/srikartati/go-ipfixlib/pkg/entities" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" ) var _ IPFIXRecord = new(ipfixDataRecord) diff --git a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go index 1d441c02872..609961e2e1c 100644 --- a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go +++ b/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go @@ -22,7 +22,7 @@ package testing import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - entities "github.com/srikartati/go-ipfixlib/pkg/entities" + entities "github.com/vmware/go-ipfix/pkg/entities" reflect "reflect" ) diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index 0a22b12486c..382954524dc 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -441,7 +441,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/srikartati/go-ipfixlib v0.0.0-20200701221601-953047e9896c/go.mod h1:kMk7mBXI7S5sFxbQSx+FOBbNogjsF8GNqCkYvM7LHLY= github.com/streamrail/concurrent-map v0.0.0-20160803124810-238fe79560e1/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6/go.mod h1:yqDD2twFAqxvvH5gtpwwgLsj5L1kbNwtoPoDOwBzXcs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -470,6 +469,7 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware-tanzu/octant v0.13.1 h1:hz4JDnAA7xDkFjF4VEbt5SrSRrG26FCxKXXBGapf6Nc= github.com/vmware-tanzu/octant v0.13.1/go.mod h1:4q+wrV4tmUwAdMjvYOujSTtZbE4+zm0n5mb7FjvN0I0= +github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= github.com/wenyingd/ofnet v0.0.0-20200601065543-2c7a62482f16/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= From 6abc0f831402784d48925bc5a6ae10d7d7ae1675 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Sun, 19 Jul 2020 23:17:11 -0700 Subject: [PATCH 08/15] Added integration test to increase test coverage --- .../flowexporter/connections/connections.go | 9 +- .../connections/connections_test.go | 2 +- pkg/agent/flowexporter/exporter/exporter.go | 19 +- .../flowexporter/flowrecords/flow_records.go | 6 + test/integration/agent/flowexporter_test.go | 172 ++++++++++++++++++ 5 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 test/integration/agent/flowexporter_test.go diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 76d14120592..feb12bb21fa 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -29,6 +29,7 @@ var _ ConnectionStore = new(connectionStore) type ConnectionStore interface { Run(stopCh <-chan struct{}, pollDone chan bool) IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error + GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) FlushConnectionStore() } @@ -73,7 +74,7 @@ func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { // TODO: Optimize this logic by flushing individual connections based on the individual timeout values. cs.FlushConnectionStore() } - _, err := cs.poll() + _, err := cs.Poll() if err != nil { // Not failing here as errors can be transient and could be resolved in future poll cycles. // TODO: Come up with a backoff/retry mechanism by increasing poll interval and adding retry timeout @@ -94,7 +95,7 @@ func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { connKey := flowexporter.NewConnectionKey(conn) - existingConn, exists := cs.getConnByKey(connKey) + existingConn, exists := cs.GetConnByKey(connKey) if exists { // Update the necessary fields that are used in generating flow records. @@ -129,7 +130,7 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { } } -func (cs *connectionStore) getConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { +func (cs *connectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { conn, found := cs.connections[flowTuple] return &conn, found } @@ -148,7 +149,7 @@ func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowR // poll returns number of filtered connections after poll cycle // TODO: Optimize polling cycle--Only poll invalid/close connection during every poll. Poll established right before export -func (cs *connectionStore) poll() (int, error) { +func (cs *connectionStore) Poll() (int, error) { klog.V(2).Infof("Polling conntrack") filteredConns, err := cs.connDumper.DumpFlows(openflow.CtZone) diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index 0b481ea1d61..bf15ab22b32 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -135,7 +135,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { iStore.EXPECT().GetInterfaceByIP(test.flow.TupleReply.SourceAddress.String()).Return(interfaceFlow2, true) } connStore.addOrUpdateConn(&test.flow) - actualConn, _ := connStore.getConnByKey(flowTuple) + actualConn, _ := connStore.GetConnByKey(flowTuple) assert.Equal(t, expConn, *actualConn, "Connections should be equal") } } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 1bb1def25d0..682e5207b3d 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -88,6 +88,16 @@ func genObservationID() (uint32, error) { return h.Sum32(), nil } +func NewFlowExporter(records flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, expInterval time.Duration, tempID uint16) *flowExporter { + return &flowExporter{ + records, + expProcess, + elemList, + expInterval, + tempID, + } +} + func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expInterval time.Duration) (*flowExporter, error) { // Create IPFIX exporting expProcess and initialize registries and other related entities obsID, err := genObservationID() @@ -108,15 +118,8 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn } expProcess.LoadRegistries() - flowExp := &flowExporter{ - records, - expProcess, - nil, - expInterval, - 0, - } + flowExp := NewFlowExporter(records, expProcess, nil, expInterval, expProcess.NewTemplateID()) - flowExp.templateID = flowExp.process.NewTemplateID() templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), flowExp.templateID) sentBytes, err := flowExp.sendTemplateRecord(templateRec) diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index 8c050afafd3..e66a5b41a15 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -28,6 +28,7 @@ var _ FlowRecords = new(flowRecords) type FlowRecords interface { BuildFlowRecords() error + GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) IterateFlowRecordsWithSendCB(callback flowexporter.FlowRecordSend, templateID uint16) error } @@ -54,6 +55,11 @@ func (fr *flowRecords) BuildFlowRecords() error { return nil } +func (fr *flowRecords) GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) { + record, found := fr.recordsMap[connKey] + return &record, found +} + func (fr *flowRecords) IterateFlowRecordsWithSendCB(sendCallback flowexporter.FlowRecordSend, templateID uint16) error { for k, v := range fr.recordsMap { if v.IsActive == true { diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go new file mode 100644 index 00000000000..b4cc9103739 --- /dev/null +++ b/test/integration/agent/flowexporter_test.go @@ -0,0 +1,172 @@ +package agent + +import ( + "fmt" + "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" + "net" + "testing" + "time" + + mock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections" + connectionstest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections/testing" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" + interfacestoretest "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore/testing" + "github.com/vmware-tanzu/antrea/pkg/agent/openflow" +) + +func makeTuple(srcIP *net.IP, dstIP *net.IP, protoID uint8, srcPort uint16, dstPort uint16) (*flowexporter.Tuple, *flowexporter.Tuple) { + tuple := &flowexporter.Tuple{ + SourceAddress: *srcIP, + DestinationAddress: *dstIP, + Protocol: protoID, + SourcePort: srcPort, + DestinationPort: dstPort, + } + revTuple := &flowexporter.Tuple{ + SourceAddress: *dstIP, + DestinationAddress: *srcIP, + Protocol: protoID, + SourcePort: dstPort, + DestinationPort: srcPort, + } + return tuple, revTuple +} + +func createConnsForTest() ([]*flowexporter.Connection, []*flowexporter.ConnectionKey) { + // Reference for flow timestamp + refTime := time.Now() + + testConns := make([]*flowexporter.Connection, 2) + testConnKeys := make([]*flowexporter.ConnectionKey, 2) + // Flow-1 + tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) + testConn1 := &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 50)), + StopTime: refTime, + OriginalPackets: 0xffff, + OriginalBytes: 0xbaaaaa0000000000, + ReversePackets: 0xff, + ReverseBytes: 0xbaaa, + TupleOrig: *tuple1, + TupleReply: *revTuple1, + } + testConnKey1 := flowexporter.NewConnectionKey(testConn1) + testConns[0] = testConn1 + testConnKeys[0] = &testConnKey1 + // Flow-2 + tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) + testConn2 := &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 20)), + StopTime: refTime, + OriginalPackets: 0xbb, + OriginalBytes: 0xcbbb, + ReversePackets: 0xbbbb, + ReverseBytes: 0xcbbbb0000000000, + TupleOrig: *tuple2, + TupleReply: *revTuple2, + } + testConnKey2 := flowexporter.NewConnectionKey(testConn2) + testConns[1] = testConn2 + testConnKeys[1] = &testConnKey2 + + return testConns, testConnKeys +} + +func prepareInterfaceConfigs(contID, podName, podNS, ifName string, ip *net.IP) *interfacestore.InterfaceConfig { + podConfig := &interfacestore.ContainerInterfaceConfig{ + ContainerID: contID, + PodName: podName, + PodNamespace: podNS, + } + iface := &interfacestore.InterfaceConfig{ + InterfaceName: ifName, + IP: *ip, + ContainerInterfaceConfig: podConfig, + } + return iface +} + +func testBuildFlowRecords(t *testing.T, flowRecords flowrecords.FlowRecords, conns []*flowexporter.Connection, connKeys []*flowexporter.ConnectionKey) { + err := flowRecords.BuildFlowRecords() + require.Nil(t, err, fmt.Sprintf("Failed to build flow records from connection store: %v", err)) + // Check if records in flow records are built as expected or not + for i, expRecConn := range conns { + actualRec, found := flowRecords.GetFlowRecordByConnKey(*connKeys[i]) + assert.Equal(t, found, true, "testConn should be part of flow records") + assert.Equal(t, actualRec.Conn, expRecConn, "testConn and connection in connection store should be equal") + } +} + +// TestConnectionStoreAndFlowRecords covers two scenarios: (i.) Add connections to connection store through connectionStore.Poll +// execution and build flow records. (ii.) Flush the connections and check records are sti:w +func TestConnectionStoreAndFlowRecords(t *testing.T) { + // Test setup + ctrl := mock.NewController(t) + defer ctrl.Finish() + + // Create ConnectionStore, FlowRecords and associated mocks + connDumperMock := connectionstest.NewMockConnTrackDumper(ctrl) + ifStoreMock := interfacestoretest.NewMockInterfaceStore(ctrl) + // Hardcoded poll and export intervals; they are not used + connStore := connections.NewConnectionStore(connDumperMock, ifStoreMock, time.Second, time.Second) + flowRecords := flowrecords.NewFlowRecords(connStore) + // Prepare connections and interface config for test + testConns, testConnKeys := createConnsForTest() + testIfConfigs := make([]*interfacestore.InterfaceConfig, 2) + testIfConfigs[0] = prepareInterfaceConfigs("1", "pod1", "ns1", "interface1", &testConns[0].TupleOrig.SourceAddress) + testIfConfigs[1] = prepareInterfaceConfigs("2", "pod2", "ns2", "interface2", &testConns[1].TupleOrig.DestinationAddress) + + // Expect calls for connStore.poll and other callees + connDumperMock.EXPECT().DumpFlows(uint16(openflow.CtZone)).Return(testConns, nil) + for i, testConn := range testConns { + if i == 0 { + ifStoreMock.EXPECT().GetInterfaceByIP(testConn.TupleOrig.SourceAddress.String()).Return(testIfConfigs[i], true) + ifStoreMock.EXPECT().GetInterfaceByIP(testConn.TupleOrig.DestinationAddress.String()).Return(nil, false) + } else { + ifStoreMock.EXPECT().GetInterfaceByIP(testConn.TupleOrig.SourceAddress.String()).Return(nil, false) + ifStoreMock.EXPECT().GetInterfaceByIP(testConn.TupleOrig.DestinationAddress.String()).Return(testIfConfigs[i], true) + } + } + // Execute connStore.Poll + connsLen, err := connStore.Poll() + require.Nil(t, err, fmt.Sprintf("Failed to add connections to connection store: %v", err)) + assert.Equal(t, connsLen, len(testConns), "expected connections should be equal to number of testConns") + + // Check if connections in connectionStore are same as testConns or not + for i, expConn := range testConns { + if i == 0 { + expConn.SourcePodName = testIfConfigs[i].PodName + expConn.SourcePodNamespace = testIfConfigs[i].PodNamespace + } else { + expConn.DestinationPodName = testIfConfigs[i].PodName + expConn.DestinationPodNamespace = testIfConfigs[i].PodNamespace + } + actualConn, found := connStore.GetConnByKey(*testConnKeys[i]) + assert.Equal(t, found, true, "testConn should be present in connection store") + assert.Equal(t, expConn, actualConn, "testConn and connection in connection store should be equal") + } + + // Test for build flow records + testBuildFlowRecords(t, flowRecords, testConns, testConnKeys) + + // Delete the connections from connection store and check + connStore.FlushConnectionStore() + // Check the resulting connectionStore; connections should not be present in ConnectionStore + for i := 0; i < len(testConns); i++ { + _, found := connStore.GetConnByKey(*testConnKeys[i]) + assert.Equal(t, found, false, "testConn should not be part of connection store") + } + err = flowRecords.BuildFlowRecords() + require.Nil(t, err, fmt.Sprintf("Failed to build flow records from connection store: %v", err)) + // Make sure that records corresponding to testConns are not gone in flow records. + for i := 0; i < len(testConns); i++ { + _, found := flowRecords.GetFlowRecordByConnKey(*testConnKeys[i]) + assert.Equal(t, found, true, "testConn should not be part of flow records") + } + +} From 52bb040d023464ac72e2a38fb117191db581cf84 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Tue, 21 Jul 2020 08:37:23 -0700 Subject: [PATCH 09/15] Address comments from 7/17 --- build/images/ipfixcollector/Dockerfile | 2 +- build/images/ipfixcollector/README.md | 2 +- build/yamls/base/conf/antrea-agent.conf | 11 ++- cmd/antrea-agent/agent.go | 7 +- cmd/antrea-agent/config.go | 11 ++- cmd/antrea-agent/options.go | 20 ++-- .../flowexporter/connections/connections.go | 96 +++++++++++-------- .../connections/connections_test.go | 3 + .../connections/conntrack_linux.go | 1 + pkg/agent/flowexporter/exporter/exporter.go | 36 +++++-- .../flowexporter/exporter/exporter_test.go | 2 + .../flowexporter/flowrecords/flow_records.go | 67 +++++++------ pkg/agent/flowexporter/ipfix/ipfix_record.go | 1 + pkg/agent/flowexporter/types.go | 10 +- test/e2e/framework.go | 4 +- test/integration/agent/flowexporter_test.go | 17 +--- 16 files changed, 161 insertions(+), 129 deletions(-) diff --git a/build/images/ipfixcollector/Dockerfile b/build/images/ipfixcollector/Dockerfile index 93f387d90a0..81545ed5b10 100644 --- a/build/images/ipfixcollector/Dockerfile +++ b/build/images/ipfixcollector/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 LABEL maintainer="Antrea " -LABEL description="A Docker image based on Ubuntu 18.04 which contains simple IPFIX collector to run flow exporter tests" +LABEL description="A Docker image based on Ubuntu 18.04 which contains a simple IPFIX collector to run flow exporter tests" WORKDIR /ipfix diff --git a/build/images/ipfixcollector/README.md b/build/images/ipfixcollector/README.md index 9f5ed99a73b..5eeadc15b60 100644 --- a/build/images/ipfixcollector/README.md +++ b/build/images/ipfixcollector/README.md @@ -1,6 +1,6 @@ # images/ipfixcollector -This Docker image is based on Ubuntu 18.04 which includes ipfix collector based on libipfix, a C library. +This Docker image is based on Ubuntu 18.04 which includes an IPFIX collector based on [libipfix](http://libipfix.sourceforge.net/), a C library. In this image, IPFIX collector listening on tcp:4739 port. libipfix package is downloaded from https://svwh.dl.sourceforge.net/project/libipfix/libipfix/libipfix-impd4e_110224.tgz diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index ce0ea29ba26..e063b366ff2 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -61,14 +61,15 @@ featureGates: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false -# Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX +# Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables flow exporter that sends IPFIX # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. # Defaults to "". #flowCollectorAddr: "" -# Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections +# Provide flow exporter poll and export intervals in format "0s:0s". This determines how often flow exporter polls connections # in conntrack module and exports IPFIX flow records that are built from connection store. -# Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. -# Any value in range (PollInterval(s), 600s] for export interval is acceptable. +# Flow export interval should be a multiple of flow poll interval. +# Flow poll interval value should be in range [1s, ExportInterval(s)). +# Flow export interval value should be in range (PollInterval(s), 600s]. # Defaults to "5s:60s". Follow the time units of duration. -#pollAndExportInterval: "" +#flowPollAndFlowExportIntervals: "" diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 1d6bc924917..2909a3ec07b 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -246,14 +246,15 @@ func run(o *Options) error { } else if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackNetdev(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) } - connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval, o.exportInterval) + connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval) flowRecords := flowrecords.NewFlowRecords(connStore) - flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords, o.exportInterval) + flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords, o.exportInterval, o.pollingInterval) if err != nil { // If flow exporter cannot be initialized, then Antrea agent does not exit; only error is logged. klog.Errorf("error when initializing flow exporter: %v", err) } else { - pollDone := make(chan bool, 1) + // pollDone helps in synchronizing connStore.Run and flowExporter.Run go routines. + pollDone := make(chan bool) go connStore.Run(stopCh, pollDone) go flowExporter.Run(stopCh, pollDone) } diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 1e0e546739a..7d5e809b576 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -86,14 +86,15 @@ type AgentConfig struct { // Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener // Defaults to false. EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"` - // Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX + // Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables flow exporter that sends IPFIX // flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. // Defaults to "". FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` - // Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections + // Provide flow exporter poll and export intervals in format "0s:0s". This determines how often flow exporter polls connections // in conntrack module and exports IPFIX flow records that are built from connection store. - // Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. - // Any value in range (PollInterval(s), 600s] for export interval is acceptable. + // Flow export interval should be a multiple of flow poll interval. + // Flow poll interval value should be in range [1s, ExportInterval(s)). + // Flow export interval value should be in range (PollInterval(s), 600s]. // Defaults to "5s:60s". Follow the time units of duration. - PollAndExportInterval string `yaml:"pollAndExportInterval,omitempty"` + FlowPollAndFlowExportIntervals string `yaml:"flowPollAndFlowExportIntervals,omitempty"` } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 2a84d9e6dfe..9978cbb5f0e 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -152,7 +152,7 @@ func (o *Options) setDefaults() { o.config.APIPort = apis.AntreaAgentAPIPort } - if o.config.FlowCollectorAddr != "" && o.config.PollAndExportInterval == "" { + if o.config.FlowCollectorAddr != "" && o.config.FlowPollAndFlowExportIntervals == "" { o.pollingInterval = 5 * time.Second o.exportInterval = 60 * time.Second } @@ -165,15 +165,13 @@ func (o *Options) validateFlowExporterConfig() error { strSlice := strings.Split(o.config.FlowCollectorAddr, ":") var proto string if len(strSlice) == 2 { - // No separator "." and proto is given + // If no separator ":" and proto is given, then default to TCP. proto = "tcp" } else if len(strSlice) > 2 { - if strSlice[2] == "udp" { - proto = "udp" - } else { - // All other cases default proto is tcp - proto = "tcp" + if (strSlice[2] != "udp") && (strSlice[2] != "tcp") { + return fmt.Errorf("IPFIX flow collector over %s proto is not supported", strSlice[2]) } + proto = strSlice[2] } else { return fmt.Errorf("IPFIX flow collector is given in invalid format") } @@ -191,14 +189,14 @@ func (o *Options) validateFlowExporterConfig() error { } else { o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) if err != nil { - return fmt.Errorf("IPFIX flow collector server TCP proto is not resolved: %v", err) + return fmt.Errorf("IPFIX flow collector over TCP proto is not resolved: %v", err) } } - if o.config.PollAndExportInterval != "" { - intervalSlice := strings.Split(o.config.PollAndExportInterval, ":") + if o.config.FlowPollAndFlowExportIntervals != "" { + intervalSlice := strings.Split(o.config.FlowPollAndFlowExportIntervals, ":") if len(intervalSlice) != 2 { - return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.PollAndExportInterval) + return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.FlowPollAndFlowExportIntervals) } o.pollingInterval, err = time.ParseDuration(intervalSlice[0]) if err != nil { diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index feb12bb21fa..a53f36d203a 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -15,6 +15,8 @@ package connections import ( + "fmt" + "sync" "time" "k8s.io/klog/v2" @@ -27,29 +29,32 @@ import ( var _ ConnectionStore = new(connectionStore) type ConnectionStore interface { + // Run enables to poll conntrack connections periodically at given flowPollInterval Run(stopCh <-chan struct{}, pollDone chan bool) - IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error - GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) - FlushConnectionStore() + // Poll calls into conntrackDumper interface + Poll() (int, error) + // ForAllConnectionsDo execute the callback for each connection in connection map + ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error + // GetConnByKey gets the connection in connection map given the connection key + GetConnByKey(connKey flowexporter.ConnectionKey) (*flowexporter.Connection, bool) + // DeleteConnectionByKey deletes the connection in connection map given the connection key + DeleteConnectionByKey(connKey flowexporter.ConnectionKey) error } type connectionStore struct { - // pollDone channel is used for synchronization of poll(ConnectionStore.Run) and export(FlowExporter.Run) go routines. - // Therefore, there is no requirement of lock to make connections map thread safe. - connections map[flowexporter.ConnectionKey]flowexporter.Connection // Add 5-tuple as string array - connDumper ConnTrackDumper - ifaceStore interfacestore.InterfaceStore - pollInterval time.Duration - exportInterval time.Duration + connections map[flowexporter.ConnectionKey]flowexporter.Connection + mutex sync.Mutex + connDumper ConnTrackDumper + ifaceStore interfacestore.InterfaceStore + pollInterval time.Duration } -func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration, exportInterval time.Duration) *connectionStore { +func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration) *connectionStore { return &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: ctDumper, - ifaceStore: ifaceStore, - pollInterval: pollInterval, - exportInterval: exportInterval, + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: ctDumper, + ifaceStore: ifaceStore, + pollInterval: pollInterval, } } @@ -60,32 +65,23 @@ func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { pollTicker := time.NewTicker(cs.pollInterval) defer pollTicker.Stop() - exportTrigger := uint(cs.exportInterval / cs.pollInterval) - exportCounter := uint(0) for { select { case <-stopCh: break case <-pollTicker.C: - if exportCounter == 0 { - exportCounter = exportTrigger - // Flush connection map once all flow records are sent. - // TODO: Optimize this logic by flushing individual connections based on the individual timeout values. - cs.FlushConnectionStore() - } _, err := cs.Poll() if err != nil { // Not failing here as errors can be transient and could be resolved in future poll cycles. // TODO: Come up with a backoff/retry mechanism by increasing poll interval and adding retry timeout klog.Errorf("Error during conntrack poll cycle: %v", err) } - exportCounter = exportCounter - 1 - if exportCounter == 0 { - // We need synchronization between ConnectionStore.Run and FlowExporter.Run go routines. pollDone channel provides that synchronization. - // More details in exporter/exporter.go. - pollDone <- true - } + // We need synchronization between ConnectionStore.Run and FlowExporter.Run go routines. + // ConnectionStore.Run (connection poll) should be done to start FlowExporter.Run (connection export); pollDone signals helps enabling this. + // FlowExporter.Run should be done to start ConnectionStore.Run; mutex on connection map object makes sure of this synchronization guarantee. + pollDone <- true + } } } @@ -97,6 +93,8 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn, exists := cs.GetConnByKey(connKey) + cs.mutex.Lock() + defer cs.mutex.Unlock() if exists { // Update the necessary fields that are used in generating flow records. // Can same 5-tuple flow get deleted and added to conntrack table? If so use ID. @@ -105,6 +103,7 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn.OriginalPackets = conn.OriginalPackets existingConn.ReverseBytes = conn.ReverseBytes existingConn.ReversePackets = conn.ReversePackets + existingConn.IsActive = true // Reassign the flow to update the map cs.connections[connKey] = *existingConn klog.V(4).Infof("Antrea flow updated: %v", existingConn) @@ -131,16 +130,19 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { } func (cs *connectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { + cs.mutex.Lock() + defer cs.mutex.Unlock() conn, found := cs.connections[flowTuple] return &conn, found } -func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowRecordUpdate) error { +func (cs *connectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error { + cs.mutex.Lock() + defer cs.mutex.Unlock() for k, v := range cs.connections { - klog.V(4).Infof("After: iterating flow with key: %v, conn: %v", k, v) - err := updateCallback(k, v) + err := callback(k, v) if err != nil { - klog.Errorf("Update callback failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) + klog.Errorf("Callback execution failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) return err } } @@ -152,6 +154,15 @@ func (cs *connectionStore) IterateCxnMapWithCB(updateCallback flowexporter.FlowR func (cs *connectionStore) Poll() (int, error) { klog.V(2).Infof("Polling conntrack") + // Reset all connections in connection map before dumping flows in conntrack module. + resetConn := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { + conn.IsActive = false + cs.connections[key] = conn + return nil + } + // We do not expect any error as resetConn is not returning any error + cs.ForAllConnectionsDo(resetConn) + filteredConns, err := cs.connDumper.DumpFlows(openflow.CtZone) if err != nil { return 0, err @@ -168,12 +179,15 @@ func (cs *connectionStore) Poll() (int, error) { return connsLen, nil } -// FlushConnectionStore after each IPFIX export of flow records. -// Timed out conntrack connections will not be sent as IPFIX flow records. -func (cs *connectionStore) FlushConnectionStore() { - klog.Infof("Flushing connection map") - - for conn := range cs.connections { - delete(cs.connections, conn) +// DeleteConnectionByKey after each IPFIX export of flow records. +func (cs *connectionStore) DeleteConnectionByKey(connKey flowexporter.ConnectionKey) error { + _, exists := cs.GetConnByKey(connKey) + if !exists { + return fmt.Errorf("connection with key %v doesn't exist in map", connKey) } + cs.mutex.Lock() + defer cs.mutex.Unlock() + delete(cs.connections, connKey) + + return nil } diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index bf15ab22b32..4808ed77664 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -62,6 +62,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { ReverseBytes: 0xbaaa, TupleOrig: *tuple1, TupleReply: *revTuple1, + IsActive: true, } // Flow-2, which is not in connectionStore tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) @@ -74,6 +75,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { ReverseBytes: 0xcbbbb0000000000, TupleOrig: *tuple2, TupleReply: *revTuple2, + IsActive: true, } // Create copy of old conntrack flow for testing purposes. // This flow is already in connection store. @@ -90,6 +92,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { SourcePodName: "pod1", DestinationPodNamespace: "", DestinationPodName: "", + IsActive: true, } podConfigFlow2 := &interfacestore.ContainerInterfaceConfig{ ContainerID: "2", diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 0c0e6f8ebc2..2dea3a66e6f 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -330,6 +330,7 @@ func createAntreaConn(conn *conntrack.Flow) *flowexporter.Connection { conn.Timeout, conn.Timestamp.Start, conn.Timestamp.Stop, + true, conn.Zone, uint32(conn.Status.Value), tupleOrig, diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 682e5207b3d..1dd444c2820 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -67,6 +67,7 @@ var ( var _ FlowExporter = new(flowExporter) type FlowExporter interface { + // Run enables to export flow records periodically at given export interval Run(stopCh <-chan struct{}, pollDone <-chan bool) } @@ -75,6 +76,7 @@ type flowExporter struct { process ipfix.IPFIXExportingProcess elementsList []*ipfixentities.InfoElement exportInterval time.Duration + pollInterval time.Duration templateID uint16 } @@ -88,17 +90,18 @@ func genObservationID() (uint32, error) { return h.Sum32(), nil } -func NewFlowExporter(records flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, expInterval time.Duration, tempID uint16) *flowExporter { +func NewFlowExporter(records flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, expInterval time.Duration, pollInterval time.Duration, tempID uint16) *flowExporter { return &flowExporter{ records, expProcess, elemList, expInterval, + pollInterval, tempID, } } -func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expInterval time.Duration) (*flowExporter, error) { +func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expInterval time.Duration, pollInterval time.Duration) (*flowExporter, error) { // Create IPFIX exporting expProcess and initialize registries and other related entities obsID, err := genObservationID() if err != nil { @@ -118,7 +121,7 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn } expProcess.LoadRegistries() - flowExp := NewFlowExporter(records, expProcess, nil, expInterval, expProcess.NewTemplateID()) + flowExp := NewFlowExporter(records, expProcess, nil, expInterval, pollInterval, expProcess.NewTemplateID()) templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), flowExp.templateID) @@ -142,11 +145,14 @@ func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan bool) { exp.process.CloseConnToCollector() break case <-ticker.C: - // Waiting for pollDone channel is necessary because IPFIX collector computes throughput based on - // flow records received interval, therefore we need to wait for poll go routine (ConnectionStore.Run). - // pollDone provides this synchronization. Note that export interval is multiple of poll interval, and poll - // interval is at least one second long. - <-pollDone + // Waiting for expected number of pollDone signals from go routine(ConnectionStore.Run) is necessary because + // IPFIX collector computes throughput based on flow records received interval. Expected number of pollDone + // signals should be equal to the number of pollCycles to be done before starting the export cycle; it is computed + // from flow poll interval and flow export interval. Note that export interval is multiple of poll interval, + // and poll interval is at least one second long. + for i := uint(0); i < uint(exp.exportInterval/exp.pollInterval); i++ { + <-pollDone + } err := exp.flowRecords.BuildFlowRecords() if err != nil { klog.Errorf("Error when building flow records: %v", err) @@ -164,9 +170,19 @@ func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan bool) { } func (exp *flowExporter) sendFlowRecords() error { - err := exp.flowRecords.IterateFlowRecordsWithSendCB(exp.sendDataRecord, exp.templateID) + sendAndUpdateFlowRecord := func(key flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { + dataRec := ipfix.NewIPFIXDataRecord(exp.templateID) + if err := exp.sendDataRecord(dataRec, record); err != nil { + return err + } + if err := exp.flowRecords.ValidateAndUpdateStats(key, record); err != nil { + return err + } + return nil + } + err := exp.flowRecords.ForAllFlowRecordsDo(sendAndUpdateFlowRecord) if err != nil { - return fmt.Errorf("error in iterating flow records: %v", err) + return fmt.Errorf("error when iterating flow records: %v", err) } return nil } diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index d3419aa7869..630d3a62d48 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -39,6 +39,7 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { mockIPFIXExpProc, nil, 60 * time.Second, + time.Second, 256, } // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) @@ -146,6 +147,7 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { mockIPFIXExpProc, elemList, 60 * time.Second, + time.Second, 256, } // Expect calls required diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index e66a5b41a15..8d9da18873d 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -16,8 +16,6 @@ package flowrecords import ( "fmt" - "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" - "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" @@ -27,16 +25,21 @@ import ( var _ FlowRecords = new(flowRecords) type FlowRecords interface { + // BuildFlowRecords builds the flow record map from connection map in connection store BuildFlowRecords() error + // GetFlowRecordByConnKey gets the record from the flow record map given the connection key GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) - IterateFlowRecordsWithSendCB(callback flowexporter.FlowRecordSend, templateID uint16) error + // ValidateAndUpdateStats validates and updates the flow record given the connection key + ValidateAndUpdateStats(connKey flowexporter.ConnectionKey, record flowexporter.FlowRecord) error + // ForAllFlowRecordsDo executes the callback for all records in the flow record map + ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallBack) error } type flowRecords struct { - // synchronization is not required as there is no concurrency involving this object. - // Add lock when this is consumed by more than one entity concurrently. - recordsMap map[flowexporter.ConnectionKey]flowexporter.FlowRecord - connStoreBuilder connections.ConnectionStore + // synchronization is not required as there is no concurrency, involving this object. + // Add lock when this map is consumed by more than one entity concurrently. + recordsMap map[flowexporter.ConnectionKey]flowexporter.FlowRecord + connStore connections.ConnectionStore } func NewFlowRecords(connStore connections.ConnectionStore) *flowRecords { @@ -47,9 +50,9 @@ func NewFlowRecords(connStore connections.ConnectionStore) *flowRecords { } func (fr *flowRecords) BuildFlowRecords() error { - err := fr.connStoreBuilder.IterateCxnMapWithCB(fr.addOrUpdateFlowRecord) + err := fr.connStore.ForAllConnectionsDo(fr.addOrUpdateFlowRecord) if err != nil { - return fmt.Errorf("error in iterating cxn map: %v", err) + return fmt.Errorf("error when iterating connection map: %v", err) } klog.V(2).Infof("No. of flow records built: %d", len(fr.recordsMap)) return nil @@ -60,25 +63,33 @@ func (fr *flowRecords) GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey return &record, found } -func (fr *flowRecords) IterateFlowRecordsWithSendCB(sendCallback flowexporter.FlowRecordSend, templateID uint16) error { +func (fr *flowRecords) ValidateAndUpdateStats(connKey flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { + // Delete the flow record if the corresponding connection is not active, i.e., not present in conntrack table. + // Delete the corresponding connection in connectionMap as well. + if !record.Conn.IsActive { + klog.V(2).Infof("Deleting the inactive connection with key: %v", connKey) + delete(fr.recordsMap, connKey) + if err := fr.connStore.DeleteConnectionByKey(connKey); err != nil { + return err + } + } else { + // Update the stats in flow record after it is sent successfully + record.PrevPackets = record.Conn.OriginalPackets + record.PrevBytes = record.Conn.OriginalBytes + record.PrevReversePackets = record.Conn.ReversePackets + record.PrevReverseBytes = record.Conn.ReverseBytes + fr.recordsMap[connKey] = record + } + + return nil +} + +func (fr *flowRecords) ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallBack) error { for k, v := range fr.recordsMap { - if v.IsActive == true { - dataRec := ipfix.NewIPFIXDataRecord(templateID) - err := sendCallback(dataRec, v) - if err != nil { - klog.Errorf("flow record update and send failed for flow with key: %v, cxn: %v", k, v) - return err - } - // Update the flow record after it is sent successfully - v.PrevPackets = v.Conn.OriginalPackets - v.PrevBytes = v.Conn.OriginalBytes - v.PrevReversePackets = v.Conn.ReversePackets - v.PrevReverseBytes = v.Conn.ReverseBytes - v.IsActive = false - fr.recordsMap[k] = v - } else { - // Delete flow record as the corresponding connection is not present in conntrack table. - delete(fr.recordsMap, k) + err := callback(k, v) + if err != nil { + klog.Errorf("Error when executing callback for flow record") + return err } } @@ -94,11 +105,9 @@ func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, con 0, 0, 0, - true, } } else { record.Conn = &conn - record.IsActive = true } fr.recordsMap[key] = record return nil diff --git a/pkg/agent/flowexporter/ipfix/ipfix_record.go b/pkg/agent/flowexporter/ipfix/ipfix_record.go index 31d32633073..d9ee42b28a1 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_record.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_record.go @@ -16,6 +16,7 @@ package ipfix import ( "bytes" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" ) diff --git a/pkg/agent/flowexporter/types.go b/pkg/agent/flowexporter/types.go index be5e2f6c810..880c1d565b9 100644 --- a/pkg/agent/flowexporter/types.go +++ b/pkg/agent/flowexporter/types.go @@ -15,15 +15,14 @@ package flowexporter import ( - "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" "net" "time" ) type ConnectionKey [5]string -type FlowRecordUpdate func(key ConnectionKey, cxn Connection) error -type FlowRecordSend func(dataRecord ipfix.IPFIXRecord, record FlowRecord) error +type ConnectionMapCallBack func(key ConnectionKey, conn Connection) error +type FlowRecordCallBack func(key ConnectionKey, record FlowRecord) error type Tuple struct { SourceAddress net.IP @@ -40,7 +39,9 @@ type Connection struct { StartTime time.Time // For invalid and closed connections: StopTime is the time when connection was updated last. // For established connections: StopTime is latest time when it was polled. - StopTime time.Time + StopTime time.Time + // IsActive flag helps in cleaning up connections when they are not in conntrack any module more. + IsActive bool Zone uint16 StatusFlag uint32 // TODO: Have a separate field for protocol. No need to keep it in Tuple. @@ -60,5 +61,4 @@ type FlowRecord struct { PrevBytes uint64 PrevReversePackets uint64 PrevReverseBytes uint64 - IsActive bool } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 6dd420590e5..9bf5cbd9bda 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -312,7 +312,7 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } // pollAndExportInterval is added as harcoded value "1s:5s" - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#pollAndExportInterval: \"\"|pollAndExportInterval: \"1s:5s\"|g' %s", antreaYML) + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowPollAndFlowExportIntervals: \"\"|flowPollAndFlowExportIntervals: \"1s:5s\"|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) @@ -340,7 +340,7 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|pollAndExportInterval: \"1s:5s\"|#pollAndExportInterval: \"\"|g' %s", antreaYML) + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowPollAndFlowExportIntervals: \"1s:5s\"|#flowPollAndFlowExportIntervals: \"\"|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index b4cc9103739..3cf1fea5faf 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -113,7 +113,7 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { connDumperMock := connectionstest.NewMockConnTrackDumper(ctrl) ifStoreMock := interfacestoretest.NewMockInterfaceStore(ctrl) // Hardcoded poll and export intervals; they are not used - connStore := connections.NewConnectionStore(connDumperMock, ifStoreMock, time.Second, time.Second) + connStore := connections.NewConnectionStore(connDumperMock, ifStoreMock, time.Second) flowRecords := flowrecords.NewFlowRecords(connStore) // Prepare connections and interface config for test testConns, testConnKeys := createConnsForTest() @@ -154,19 +154,4 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { // Test for build flow records testBuildFlowRecords(t, flowRecords, testConns, testConnKeys) - // Delete the connections from connection store and check - connStore.FlushConnectionStore() - // Check the resulting connectionStore; connections should not be present in ConnectionStore - for i := 0; i < len(testConns); i++ { - _, found := connStore.GetConnByKey(*testConnKeys[i]) - assert.Equal(t, found, false, "testConn should not be part of connection store") - } - err = flowRecords.BuildFlowRecords() - require.Nil(t, err, fmt.Sprintf("Failed to build flow records from connection store: %v", err)) - // Make sure that records corresponding to testConns are not gone in flow records. - for i := 0; i < len(testConns); i++ { - _, found := flowRecords.GetFlowRecordByConnKey(*testConnKeys[i]) - assert.Equal(t, found, true, "testConn should not be part of flow records") - } - } From 7eb03b5900719a50798658b0937cd7157ccf58c2 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Mon, 27 Jul 2020 11:31:39 -0700 Subject: [PATCH 10/15] Improve unit test coverage --- .../connections/connections_test.go | 127 ++++++++++++++- .../connections/conntrack_linux.go | 5 +- .../connections/conntrack_test.go | 146 +++++++++++------- .../flowexporter/exporter/exporter_test.go | 24 ++- pkg/util/ip/ip.go | 1 + 5 files changed, 225 insertions(+), 78 deletions(-) diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index 4808ed77664..19b63312a62 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -117,13 +117,13 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { testFlow1Tuple := flowexporter.NewConnectionKey(&testFlow1) connStore.connections[testFlow1Tuple] = oldTestFlow1 - updateConnTests := []struct { + addOrUpdateConnTests := []struct { flow flowexporter.Connection }{ {testFlow1}, // To test update part of function {testFlow2}, // To test add part of function } - for i, test := range updateConnTests { + for i, test := range addOrUpdateConnTests { flowTuple := flowexporter.NewConnectionKey(&test.flow) var expConn flowexporter.Connection if i == 0 { @@ -138,7 +138,128 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { iStore.EXPECT().GetInterfaceByIP(test.flow.TupleReply.SourceAddress.String()).Return(interfaceFlow2, true) } connStore.addOrUpdateConn(&test.flow) - actualConn, _ := connStore.GetConnByKey(flowTuple) + actualConn, ok := connStore.GetConnByKey(flowTuple) + assert.Equal(t, ok, true, "connection should be there in connection store") assert.Equal(t, expConn, *actualConn, "Connections should be equal") } } + +func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + // Create two flows; one is already in connectionStore and other one is new + testFlows := make([]*flowexporter.Connection, 2) + testFlowKeys := make([]*flowexporter.ConnectionKey, 2) + refTime := time.Now() + // Flow-1, which is already in connectionStore + tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) + testFlows[0] = &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 50)), + StopTime: refTime, + OriginalPackets: 0xffff, + OriginalBytes: 0xbaaaaa0000000000, + ReversePackets: 0xff, + ReverseBytes: 0xbaaa, + TupleOrig: *tuple1, + TupleReply: *revTuple1, + IsActive: true, + } + // Flow-2, which is not in connectionStore + tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) + testFlows[1] = &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 20)), + StopTime: refTime, + OriginalPackets: 0xbb, + OriginalBytes: 0xcbbb, + ReversePackets: 0xbbbb, + ReverseBytes: 0xcbbbb0000000000, + TupleOrig: *tuple2, + TupleReply: *revTuple2, + IsActive: true, + } + for i, flow := range testFlows { + connKey := flowexporter.NewConnectionKey(flow) + testFlowKeys[i] = &connKey + } + // Create connectionStore + connStore := &connectionStore{ + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: nil, + ifaceStore: nil, + } + // Add flows to the Connection store + for i, flow := range testFlows { + connStore.connections[*testFlowKeys[i]] = *flow + } + + resetTwoFields := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { + conn.IsActive = false + conn.OriginalPackets = 0 + connStore.connections[key] = conn + return nil + } + connStore.ForAllConnectionsDo(resetTwoFields) + // Check isActive and OriginalPackets, if they are reset or not. + for i := 0; i < len(testFlows); i++ { + conn, ok := connStore.GetConnByKey(*testFlowKeys[i]) + assert.Equal(t, ok, true, "connection should be there in connection store") + assert.Equal(t, conn.IsActive, false, "isActive flag should be reset") + assert.Equal(t, conn.OriginalPackets, uint64(0), "OriginalPackets should be reset") + } +} + +func TestConnectionStore_DeleteConnectionByKey(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + // Create two flows; one is already in connectionStore and other one is new + testFlows := make([]*flowexporter.Connection, 2) + testFlowKeys := make([]*flowexporter.ConnectionKey, 2) + refTime := time.Now() + // Flow-1, which is already in connectionStore + tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) + testFlows[0] = &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 50)), + StopTime: refTime, + OriginalPackets: 0xffff, + OriginalBytes: 0xbaaaaa0000000000, + ReversePackets: 0xff, + ReverseBytes: 0xbaaa, + TupleOrig: *tuple1, + TupleReply: *revTuple1, + IsActive: true, + } + // Flow-2, which is not in connectionStore + tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) + testFlows[1] = &flowexporter.Connection{ + StartTime: refTime.Add(-(time.Second * 20)), + StopTime: refTime, + OriginalPackets: 0xbb, + OriginalBytes: 0xcbbb, + ReversePackets: 0xbbbb, + ReverseBytes: 0xcbbbb0000000000, + TupleOrig: *tuple2, + TupleReply: *revTuple2, + IsActive: true, + } + for i, flow := range testFlows { + connKey := flowexporter.NewConnectionKey(flow) + testFlowKeys[i] = &connKey + } + // Create connectionStore + connStore := &connectionStore{ + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: nil, + ifaceStore: nil, + } + // Add flows to the connection store. + for i, flow := range testFlows { + connStore.connections[*testFlowKeys[i]] = *flow + } + // Delete the connections in connection store. + for i := 0; i < len(testFlows); i++ { + err := connStore.DeleteConnectionByKey(*testFlowKeys[i]) + assert.Nil(t, err, "DeleteConnectionByKey should return nil") + _, exists := connStore.GetConnByKey(*testFlowKeys[i]) + assert.Equal(t, exists, false, "connection should be deleted in connection store") + } +} diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 2dea3a66e6f..e8045c0af81 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -214,7 +214,7 @@ func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Con conn := flowexporter.Connection{} flowSlice := strings.Split(flow, ",") isReply := false - inZone := true + inZone := false for _, fs := range flowSlice { // Indicator to populate reply or reverse fields if strings.Contains(fs, "reply") { @@ -277,9 +277,9 @@ func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Con continue } if zoneFilter != uint16(val) { - inZone = false break } else { + inZone = true conn.Zone = uint16(val) } } else if strings.Contains(fs, "timeout") { @@ -301,6 +301,7 @@ func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Con } } if inZone { + conn.IsActive = true antreaConns = append(antreaConns, &conn) } } diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index 5e4389cd0cf..67b5ab917f9 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -19,6 +19,7 @@ package connections import ( "net" "testing" + "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -32,76 +33,33 @@ import ( ovsctltest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl/testing" ) -var ( - tuple3 = flowexporter.Tuple{ - SourceAddress: net.IP{1, 2, 3, 4}, - DestinationAddress: net.IP{4, 3, 2, 1}, - Protocol: 6, - SourcePort: 65280, - DestinationPort: 255, - } - revTuple3 = flowexporter.Tuple{ - SourceAddress: net.IP{4, 3, 2, 1}, - DestinationAddress: net.IP{1, 2, 3, 4}, - Protocol: 6, - SourcePort: 255, - DestinationPort: 65280, - } - tuple4 = flowexporter.Tuple{ - SourceAddress: net.IP{5, 6, 7, 8}, - DestinationAddress: net.IP{8, 7, 6, 5}, - Protocol: 6, - SourcePort: 60001, - DestinationPort: 200, - } - revTuple4 = flowexporter.Tuple{ - SourceAddress: net.IP{8, 7, 6, 5}, - DestinationAddress: net.IP{5, 6, 7, 8}, - Protocol: 6, - SourcePort: 200, - DestinationPort: 60001, - } - tuple5 = flowexporter.Tuple{ - SourceAddress: net.IP{1, 2, 3, 4}, - DestinationAddress: net.IP{100, 50, 25, 5}, - Protocol: 6, - SourcePort: 60001, - DestinationPort: 200, - } - revTuple5 = flowexporter.Tuple{ - SourceAddress: net.IP{100, 50, 25, 5}, - DestinationAddress: net.IP{1, 2, 3, 4}, - Protocol: 6, - SourcePort: 200, - DestinationPort: 60001, - } -) - func TestConnTrack_DumpFlows(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Create flows to test + // Create flows for test + tuple, revTuple := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) antreaFlow := &flowexporter.Connection{ - TupleOrig: tuple3, - TupleReply: revTuple3, + TupleOrig: *tuple, + TupleReply: *revTuple, Zone: openflow.CtZone, } + tuple, revTuple = makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{100, 50, 25, 5}, 6, 60001, 200) antreaServiceFlow := &flowexporter.Connection{ - TupleOrig: tuple5, - TupleReply: revTuple5, + TupleOrig: *tuple, + TupleReply: *revTuple, Zone: openflow.CtZone, } + tuple, revTuple = makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) antreaGWFlow := &flowexporter.Connection{ - TupleOrig: tuple4, - TupleReply: revTuple4, + TupleOrig: *tuple, + TupleReply: *revTuple, Zone: openflow.CtZone, } nonAntreaFlow := &flowexporter.Connection{ - TupleOrig: tuple4, - TupleReply: revTuple4, + TupleOrig: *tuple, + TupleReply: *revTuple, Zone: 100, } - testFlows := []*flowexporter.Connection{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} // Create mock interfaces @@ -120,14 +78,86 @@ func TestConnTrack_DumpFlows(t *testing.T) { IP: net.IP{100, 50, 25, 0}, Mask: net.IPMask{255, 255, 255, 0}, } - // set expects for mocks + + // Test DumpFlows implementation of connTrackSystem + connDumperDPSystem := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathSystem, mockOVSCtlClient) + // Set expects for mocks mockCTInterfacer.EXPECT().GetConnTrack(nil).Return(nil) mockCTInterfacer.EXPECT().DumpFilter(conntrack.Filter{}).Return(testFlows, nil) - connDumper := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathSystem, mockOVSCtlClient) - conns, err := connDumper.DumpFlows(openflow.CtZone) + conns, err := connDumperDPSystem.DumpFlows(openflow.CtZone) if err != nil { t.Errorf("Dump flows function returned error: %v", err) } assert.Equal(t, 1, len(conns), "number of filtered connections should be equal") + + // Test DumpFlows implementation of connTrackNetdev + connDumperDPNetdev := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathNetdev, mockOVSCtlClient) + // Re-initialize testFlows + testFlows = []*flowexporter.Connection{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} + // Set expects for mocks + mockCTInterfacer.EXPECT().GetConnTrack(mockOVSCtlClient).Return(nil) + mockCTInterfacer.EXPECT().DumpFilter(uint16(openflow.CtZone)).Return(testFlows, nil) + + conns, err = connDumperDPNetdev.DumpFlows(openflow.CtZone) + if err != nil { + t.Errorf("Dump flows function returned error: %v", err) + } + assert.Equal(t, 1, len(conns), "number of filtered connections should be equal") +} + +func TestConnTackNetdev_DumpFilter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Create mock interfaces + mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) + conntrackNetdev := NewConnTrackNetdev() + err := conntrackNetdev.GetConnTrack(mockOVSCtlClient) + assert.Nil(t, err, "GetConnTrack call should be successful") + + // Set expect call for mock ovsCtlClient + ovsctlCmdOutput := []byte("tcp,orig=(src=127.0.0.1,dst=127.0.0.1,sport=45218,dport=2379,packets=320108,bytes=24615344),reply=(src=127.0.0.1,dst=127.0.0.1,sport=2379,dport=45218,packets=239595,bytes=24347883),start=2020-07-24T05:07:03.998,id=3750535678,status=SEEN_REPLY|ASSURED|CONFIRMED|SRC_NAT_DONE|DST_NAT_DONE,timeout=86399,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)\n" + + "tcp,orig=(src=127.0.0.1,dst=127.0.0.1,sport=45170,dport=2379,packets=80743,bytes=5416239),reply=(src=127.0.0.1,dst=127.0.0.1,sport=2379,dport=45170,packets=63361,bytes=4811261),start=2020-07-24T05:07:01.591,id=462801621,status=SEEN_REPLY|ASSURED|CONFIRMED|SRC_NAT_DONE|DST_NAT_DONE,timeout=86397,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)\n" + + "tcp,orig=(src=100.10.0.105,dst=10.96.0.1,sport=41284,dport=443,packets=343260,bytes=19340621),reply=(src=192.168.86.82,dst=100.10.0.105,sport=6443,dport=41284,packets=381035,bytes=181176472),start=2020-07-25T08:40:08.959,id=982464968,zone=65520,status=SEEN_REPLY|ASSURED|CONFIRMED|DST_NAT|DST_NAT_DONE,timeout=86399,mark=33,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)") + expConn := &flowexporter.Connection{ + ID: 982464968, + Timeout: 86399, + StartTime: time.Time{}, + StopTime: time.Time{}, + IsActive: true, + Zone: 65520, + StatusFlag: 0, + TupleOrig: flowexporter.Tuple{ + net.ParseIP("100.10.0.105"), + net.ParseIP("10.96.0.1"), + 6, + uint16(41284), + uint16(443), + }, + TupleReply: flowexporter.Tuple{ + net.ParseIP("192.168.86.82"), + net.ParseIP("100.10.0.105"), + 6, + 6443, + 41284, + }, + OriginalPackets: 0, + OriginalBytes: 0, + ReversePackets: 0, + ReverseBytes: 0, + SourcePodNamespace: "", + SourcePodName: "", + DestinationPodNamespace: "", + DestinationPodName: "", + } + mockOVSCtlClient.EXPECT().RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s").Return(ovsctlCmdOutput, nil) + + conns, err := conntrackNetdev.DumpFilter(uint16(openflow.CtZone)) + if err != nil { + t.Errorf("conntrackNetdev.DumpFilter function returned error: %v", err) + } + assert.Equal(t, len(conns), 1) + assert.Equal(t, conns[0], expConn, "filtered connection and expected connection should be same") + } diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 630d3a62d48..92a9c1e903b 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -153,25 +153,19 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { // Expect calls required var dataRecord ipfixentities.Record tempBytes := uint16(0) - for i, ie := range flowExp.elementsList { - // Could not come up with a way to exclude if else conditions as different IEs have different data types. - if i == 0 || i == 1 { - // For time elements - mockDataRec.EXPECT().AddInfoElement(ie, record1.Conn.StartTime.Unix()).Return(tempBytes, nil) - } else if i == 2 || i == 3 { - // For IP addresses + for _, ie := range flowExp.elementsList { + switch ieName := ie.Name; ieName { + case "flowStartSeconds", "flowEndSeconds": + mockDataRec.EXPECT().AddInfoElement(ie, time.Time{}.Unix()).Return(tempBytes, nil) + case "sourceIPv4Address", "destinationIPv4Address": mockDataRec.EXPECT().AddInfoElement(ie, nil).Return(tempBytes, nil) - } else if i == 4 || i == 5 { - // For transport ports + case "sourceTransportPort", "destinationTransportPort": mockDataRec.EXPECT().AddInfoElement(ie, uint16(0)).Return(tempBytes, nil) - } else if i == 6 { - // For proto identifier + case "protocolIdentifier": mockDataRec.EXPECT().AddInfoElement(ie, uint8(0)).Return(tempBytes, nil) - } else if i >= 7 && i < 15 { - // For packets and octets + case "packetTotalCount", "octetTotalCount", "packetDeltaCount", "octetDeltaCount", "reverse_PacketTotalCount", "reverse_OctetTotalCount", "reverse_PacketDeltaCount", "reverse_OctetDeltaCount": mockDataRec.EXPECT().AddInfoElement(ie, uint64(0)).Return(tempBytes, nil) - } else { - // For string elements + case "sourcePodName", "sourcePodNamespace", "sourceNodeName", "destinationPodName", "destinationPodNamespace", "destinationNodeName": mockDataRec.EXPECT().AddInfoElement(ie, "").Return(tempBytes, nil) } } diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go index 9afbbd7c85a..3a059c03bd4 100644 --- a/pkg/util/ip/ip.go +++ b/pkg/util/ip/ip.go @@ -159,6 +159,7 @@ func NetIPNetToIPNet(ipNet *net.IPNet) *v1beta1.IPNet { // LookupProtocolMap return protocol identifier given protocol name func LookupProtocolMap(name string) (uint8, error) { + name = strings.TrimSpace(name) lowerCaseStr := strings.ToLower(name) proto, found := protocols[lowerCaseStr] if !found { From a8ea1d529f85cab04467924846d3b928c97f12db Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Thu, 30 Jul 2020 12:25:25 -0700 Subject: [PATCH 11/15] Do not send double copies of flow records This will be removed when network policy info is added in flow records. --- pkg/agent/flowexporter/connections/connections.go | 9 ++++++++- .../flowexporter/connections/conntrack_linux.go | 2 ++ pkg/agent/flowexporter/connections/conntrack_test.go | 1 + pkg/agent/flowexporter/flowrecords/flow_records.go | 5 +++++ pkg/agent/flowexporter/types.go | 4 +++- test/integration/agent/flowexporter_test.go | 12 ++++++++++-- 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index a53f36d203a..509fe3cf432 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -123,6 +123,13 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodName = dIface.ContainerInterfaceConfig.PodName conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } + // Do not export flow records of connections, who destination is local pod and source is remote pod. + // We export flow records only form "source node", where the connection is originated from. This is to avoid + // 2 copies of flow records. This restriction will be removed when flow records store network policy rule ID. + // TODO: Remove this when network policy rule ID are added to flow records. + if !srcFound && dstFound { + conn.DoExport = false + } klog.V(4).Infof("New Antrea flow added: %v", conn) // Add new antrea connection to connection store cs.connections[connKey] = *conn @@ -149,7 +156,7 @@ func (cs *connectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionM return nil } -// poll returns number of filtered connections after poll cycle +// Poll returns number of filtered connections after poll cycle // TODO: Optimize polling cycle--Only poll invalid/close connection during every poll. Poll established right before export func (cs *connectionStore) Poll() (int, error) { klog.V(2).Infof("Polling conntrack") diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index e8045c0af81..70118e31994 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -302,6 +302,7 @@ func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Con } if inZone { conn.IsActive = true + conn.DoExport = true antreaConns = append(antreaConns, &conn) } } @@ -332,6 +333,7 @@ func createAntreaConn(conn *conntrack.Flow) *flowexporter.Connection { conn.Timestamp.Start, conn.Timestamp.Stop, true, + true, conn.Zone, uint32(conn.Status.Value), tupleOrig, diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index 67b5ab917f9..70eca92e0c2 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -126,6 +126,7 @@ func TestConnTackNetdev_DumpFilter(t *testing.T) { StartTime: time.Time{}, StopTime: time.Time{}, IsActive: true, + DoExport: true, Zone: 65520, StatusFlag: 0, TupleOrig: flowexporter.Tuple{ diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index 8d9da18873d..276b4066a06 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -97,6 +97,11 @@ func (fr *flowRecords) ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallB } func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { + // If DoExport flag is not set return immediately. + if !conn.DoExport { + return nil + } + record, exists := fr.recordsMap[key] if !exists { record = flowexporter.FlowRecord{ diff --git a/pkg/agent/flowexporter/types.go b/pkg/agent/flowexporter/types.go index 880c1d565b9..319e59c8f92 100644 --- a/pkg/agent/flowexporter/types.go +++ b/pkg/agent/flowexporter/types.go @@ -41,7 +41,9 @@ type Connection struct { // For established connections: StopTime is latest time when it was polled. StopTime time.Time // IsActive flag helps in cleaning up connections when they are not in conntrack any module more. - IsActive bool + IsActive bool + // DoExport flag helps in tagging connections that can be exported by Flow Exporter + DoExport bool Zone uint16 StatusFlag uint32 // TODO: Have a separate field for protocol. No need to keep it in Tuple. diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index 3cf1fea5faf..a1a44c17af8 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -54,6 +54,7 @@ func createConnsForTest() ([]*flowexporter.Connection, []*flowexporter.Connectio ReverseBytes: 0xbaaa, TupleOrig: *tuple1, TupleReply: *revTuple1, + DoExport: true, } testConnKey1 := flowexporter.NewConnectionKey(testConn1) testConns[0] = testConn1 @@ -69,6 +70,7 @@ func createConnsForTest() ([]*flowexporter.Connection, []*flowexporter.Connectio ReverseBytes: 0xcbbbb0000000000, TupleOrig: *tuple2, TupleReply: *revTuple2, + DoExport: true, } testConnKey2 := flowexporter.NewConnectionKey(testConn2) testConns[1] = testConn2 @@ -97,8 +99,12 @@ func testBuildFlowRecords(t *testing.T, flowRecords flowrecords.FlowRecords, con // Check if records in flow records are built as expected or not for i, expRecConn := range conns { actualRec, found := flowRecords.GetFlowRecordByConnKey(*connKeys[i]) - assert.Equal(t, found, true, "testConn should be part of flow records") - assert.Equal(t, actualRec.Conn, expRecConn, "testConn and connection in connection store should be equal") + if expRecConn.DoExport { + assert.Equal(t, found, true, "testConn should be part of flow records") + assert.Equal(t, actualRec.Conn, expRecConn, "testConn and connection in connection store should be equal") + } else { + assert.Equal(t, found, false, "testConn should be not part of flow records") + } } } @@ -142,9 +148,11 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { if i == 0 { expConn.SourcePodName = testIfConfigs[i].PodName expConn.SourcePodNamespace = testIfConfigs[i].PodNamespace + expConn.DoExport = true } else { expConn.DestinationPodName = testIfConfigs[i].PodName expConn.DestinationPodNamespace = testIfConfigs[i].PodNamespace + expConn.DoExport = false } actualConn, found := connStore.GetConnByKey(*testConnKeys[i]) assert.Equal(t, found, true, "testConn should be present in connection store") From 090ef090d092c524e47e63fc5236ad1700fd5f1f Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Fri, 31 Jul 2020 23:42:50 -0700 Subject: [PATCH 12/15] Addressed comments from 7/30 --- build/yamls/base/conf/antrea-agent.conf | 17 +- cmd/antrea-agent/agent.go | 40 +-- cmd/antrea-agent/config.go | 21 +- cmd/antrea-agent/main.go | 2 +- cmd/antrea-agent/options.go | 58 ++-- cmd/antrea-agent/options_test.go | 53 ++++ go.mod | 1 - go.sum | 1 - hack/update-codegen-dockerized.sh | 2 +- .../flowexporter/connections/connections.go | 95 +++--- .../connections/connections_test.go | 58 ++-- .../flowexporter/connections/conntrack.go | 176 ++++++++++- .../connections/conntrack_linux.go | 294 +++--------------- .../connections/conntrack_test.go | 63 ++-- .../connections/conntrack_windows.go | 17 +- .../flowexporter/connections/interface.go | 12 +- .../connections/testing/mock_connections.go | 59 ++-- pkg/agent/flowexporter/exporter/exporter.go | 40 +-- .../flowexporter/exporter/exporter_test.go | 6 +- .../flowexporter/flowrecords/flow_records.go | 36 +-- test/e2e/fixtures.go | 2 - test/e2e/framework.go | 17 +- test/integration/agent/flowexporter_test.go | 19 +- 23 files changed, 542 insertions(+), 547 deletions(-) create mode 100644 cmd/antrea-agent/options_test.go diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index e063b366ff2..e9b78ff7f9b 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -63,13 +63,14 @@ featureGates: # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables flow exporter that sends IPFIX # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. -# Defaults to "". #flowCollectorAddr: "" -# Provide flow exporter poll and export intervals in format "0s:0s". This determines how often flow exporter polls connections -# in conntrack module and exports IPFIX flow records that are built from connection store. -# Flow export interval should be a multiple of flow poll interval. -# Flow poll interval value should be in range [1s, ExportInterval(s)). -# Flow export interval value should be in range (PollInterval(s), 600s]. -# Defaults to "5s:60s". Follow the time units of duration. -#flowPollAndFlowExportIntervals: "" +# Provide flow poll interval in format "0s". This determines how often flow exporter dumps connections in conntrack module. +# Flow poll interval should be greater than or equal to 1s(one second). +# Follow the time units of time.Duration type. +#flowPollInterval: "5s" + +# Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to +# the flow collector. +# Flow export frequency should be greater than or equal to 1. +#flowExportFrequency: 12 diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 2909a3ec07b..f81ff150ecc 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -237,28 +237,28 @@ func run(o *Options) error { go ofClient.StartPacketInHandler(stopCh) } - // Initialize flow exporter; start go routines to poll conntrack flows and export IPFIX flow records + // Initialize flow exporter to start go routines to poll conntrack flows and export IPFIX flow records if features.DefaultFeatureGate.Enabled(features.FlowExporter) { - if o.flowCollector != nil { - var connTrackDumper connections.ConnTrackDumper - if o.config.OVSDatapathType == ovsconfig.OVSDatapathSystem { - connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackSystem(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) - } else if o.config.OVSDatapathType == ovsconfig.OVSDatapathNetdev { - connTrackDumper = connections.NewConnTrackDumper(connections.NewConnTrackNetdev(), nodeConfig, serviceCIDRNet, o.config.OVSDatapathType, agentQuerier.GetOVSCtlClient()) - } - connStore := connections.NewConnectionStore(connTrackDumper, ifaceStore, o.pollingInterval) - flowRecords := flowrecords.NewFlowRecords(connStore) - flowExporter, err := exporter.InitFlowExporter(o.flowCollector, flowRecords, o.exportInterval, o.pollingInterval) - if err != nil { - // If flow exporter cannot be initialized, then Antrea agent does not exit; only error is logged. - klog.Errorf("error when initializing flow exporter: %v", err) - } else { - // pollDone helps in synchronizing connStore.Run and flowExporter.Run go routines. - pollDone := make(chan bool) - go connStore.Run(stopCh, pollDone) - go flowExporter.Run(stopCh, pollDone) - } + connStore := connections.NewConnectionStore( + o.config.OVSDatapathType, + nodeConfig, + serviceCIDRNet, + agentQuerier.GetOVSCtlClient(), + ifaceStore, + o.pollInterval) + // pollDone helps in synchronizing connStore.Run and flowExporter.Run go routines. + pollDone := make(chan struct{}) + go connStore.Run(stopCh, pollDone) + + flowExporter, err := exporter.InitFlowExporter( + o.flowCollector, + flowrecords.NewFlowRecords(connStore), + o.config.FlowExportFrequency, + o.pollInterval) + if err != nil { + return fmt.Errorf("error when initializing flow exporter: %v", err) } + go flowExporter.Run(stopCh, pollDone) } <-stopCh diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 7d5e809b576..38c46fe2b7b 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -86,15 +86,18 @@ type AgentConfig struct { // Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener // Defaults to false. EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"` - // Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables flow exporter that sends IPFIX - // flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. + // Provide the flow collector address as string with format :[:], where proto is tcp or udp. This also + // enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto + // is given, we consider tcp as default. // Defaults to "". FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` - // Provide flow exporter poll and export intervals in format "0s:0s". This determines how often flow exporter polls connections - // in conntrack module and exports IPFIX flow records that are built from connection store. - // Flow export interval should be a multiple of flow poll interval. - // Flow poll interval value should be in range [1s, ExportInterval(s)). - // Flow export interval value should be in range (PollInterval(s), 600s]. - // Defaults to "5s:60s". Follow the time units of duration. - FlowPollAndFlowExportIntervals string `yaml:"flowPollAndFlowExportIntervals,omitempty"` + // Provide flow poll interval in format "0s". This determines how often flow exporter dumps connections in conntrack module. + // Flow poll interval should be greater than or equal to 1s(one second). + // Defaults to "5s". Follow the time units of duration. + FlowPollInterval string `yaml:"flowPollInterval,omitempty"` + // Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + // the flow collector. + // Flow export frequency should be greater than or equal to 1. + // Defaults to "12". + FlowExportFrequency uint `yaml:"flowExportFrequency,omitempty"` } diff --git a/cmd/antrea-agent/main.go b/cmd/antrea-agent/main.go index 2b9c68a5c2f..13a2d8230ca 100644 --- a/cmd/antrea-agent/main.go +++ b/cmd/antrea-agent/main.go @@ -62,7 +62,7 @@ func newAgentCommand() *cobra.Command { if err := opts.validate(args); err != nil { klog.Fatalf("Failed to validate: %v", err) } - // Not passing args again as it is already validated and not used in flow exporter config + // Not passing args again as they are already validated and are not used in flow exporter config if err := opts.validateFlowExporterConfig(); err != nil { klog.Fatalf("Failed to validate flow exporter config: %v", err) } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 9978cbb5f0e..25cb034117c 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -46,10 +46,8 @@ type Options struct { config *AgentConfig // IPFIX flow collector flowCollector net.Addr - // Flow exporter polling interval - pollingInterval time.Duration - // Flow exporter export interval - exportInterval time.Duration + // Flow exporter poll interval + pollInterval time.Duration } func newOptions() *Options { @@ -152,15 +150,22 @@ func (o *Options) setDefaults() { o.config.APIPort = apis.AntreaAgentAPIPort } - if o.config.FlowCollectorAddr != "" && o.config.FlowPollAndFlowExportIntervals == "" { - o.pollingInterval = 5 * time.Second - o.exportInterval = 60 * time.Second + if o.config.FeatureGates[string(features.FlowExporter)] { + if o.config.FlowPollInterval == "" { + o.pollInterval = 5 * time.Second + } + if o.config.FlowExportFrequency == 0 { + // This frequency value makes flow export interval as 60s + o.config.FlowExportFrequency = 12 + } } } func (o *Options) validateFlowExporterConfig() error { if features.DefaultFeatureGate.Enabled(features.FlowExporter) { - if o.config.FlowCollectorAddr != "" { + if o.config.FlowCollectorAddr == "" { + return fmt.Errorf("IPFIX flow collector address should be provided") + } else { // Check if it is TCP or UDP strSlice := strings.Split(o.config.FlowCollectorAddr, ":") var proto string @@ -175,6 +180,7 @@ func (o *Options) validateFlowExporterConfig() error { } else { return fmt.Errorf("IPFIX flow collector is given in invalid format") } + // Convert the string input in net.Addr format hostPortAddr := strSlice[0] + ":" + strSlice[1] _, _, err := net.SplitHostPort(hostPortAddr) @@ -184,37 +190,23 @@ func (o *Options) validateFlowExporterConfig() error { if proto == "udp" { o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) if err != nil { - return fmt.Errorf("IPFIX flow collector over UDP proto is not resolved: %v", err) + return fmt.Errorf("IPFIX flow collector over UDP proto cannot be resolved: %v", err) } } else { o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) if err != nil { - return fmt.Errorf("IPFIX flow collector over TCP proto is not resolved: %v", err) + return fmt.Errorf("IPFIX flow collector over TCP proto cannot be resolved: %v", err) } } - - if o.config.FlowPollAndFlowExportIntervals != "" { - intervalSlice := strings.Split(o.config.FlowPollAndFlowExportIntervals, ":") - if len(intervalSlice) != 2 { - return fmt.Errorf("flow exporter intervals %s is not in acceptable format \"OOs:OOs\"", o.config.FlowPollAndFlowExportIntervals) - } - o.pollingInterval, err = time.ParseDuration(intervalSlice[0]) - if err != nil { - return fmt.Errorf("poll interval is not provided in right format: %v", err) - } - o.exportInterval, err = time.ParseDuration(intervalSlice[1]) - if err != nil { - return fmt.Errorf("export interval is not provided in right format: %v", err) - } - if o.pollingInterval < time.Second { - return fmt.Errorf("poll interval should be minimum of one second") - } - if o.pollingInterval > o.exportInterval { - return fmt.Errorf("poll interval should be less than or equal to export interval") - } - if o.exportInterval%o.pollingInterval != 0 { - return fmt.Errorf("export interval should be a multiple of poll interval") - } + } + if o.config.FlowPollInterval != "" { + var err error + o.pollInterval, err = time.ParseDuration(o.config.FlowPollInterval) + if err != nil { + return fmt.Errorf("FlowPollInterval is not provided in right format: %v", err) + } + if o.pollInterval < time.Second { + return fmt.Errorf("FlowPollInterval should be greater than or equal to one second") } } } diff --git a/cmd/antrea-agent/options_test.go b/cmd/antrea-agent/options_test.go new file mode 100644 index 00000000000..db0dfd183db --- /dev/null +++ b/cmd/antrea-agent/options_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/vmware-tanzu/antrea/pkg/features" +) + +func TestOptions_validateFlowExporterConfig(t *testing.T) { + // Enable flow exporter + enableFlowExporter := map[string]bool{ + "FlowExporter": true, + } + features.DefaultMutableFeatureGate.SetFromMap(enableFlowExporter) + testcases := []struct { + // input + collector string + pollInterval string + // expectations + expCollectorNet string + expCollectorStr string + expPollIntervalStr string + expError error + }{ + {collector: "192.168.1.100:2002:tcp", pollInterval: "5s", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, + {collector: "192.168.1.100:2002:udp", pollInterval: "5s", expCollectorNet: "udp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, + {collector: "192.168.1.100:2002", pollInterval: "5s", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, + {collector: "192.168.1.100:2002:sctp", pollInterval: "5s", expCollectorNet: "", expCollectorStr: "", expPollIntervalStr: "", expError: fmt.Errorf("IPFIX flow collector over %s proto is not supported", "sctp")}, + {collector: "192.168.1.100:2002", pollInterval: "5ss", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "", expError: fmt.Errorf("FlowPollInterval is not provided in right format: ")}, + {collector: "192.168.1.100:2002", pollInterval: "1ms", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "", expError: fmt.Errorf("FlowPollInterval should be greater than or equal to one second")}, + } + assert.Equal(t, features.DefaultFeatureGate.Enabled(features.FlowExporter), true) + for _, tc := range testcases { + testOptions := &Options{ + config: new(AgentConfig), + } + testOptions.config.FlowCollectorAddr = tc.collector + testOptions.config.FlowPollInterval = tc.pollInterval + err := testOptions.validateFlowExporterConfig() + + if tc.expError != nil { + assert.NotNil(t, err) + } else { + assert.Equal(t, tc.expCollectorNet, testOptions.flowCollector.Network()) + assert.Equal(t, tc.expCollectorStr, testOptions.flowCollector.String()) + assert.Equal(t, tc.expPollIntervalStr, testOptions.pollInterval.String()) + } + } + +} diff --git a/go.mod b/go.mod index b800d8d41aa..5367ad8cc61 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/grpc v1.26.0 gopkg.in/yaml.v2 v2.2.8 - gotest.tools v2.2.0+incompatible k8s.io/api v0.18.4 k8s.io/apimachinery v0.18.4 k8s.io/apiserver v0.18.4 diff --git a/go.sum b/go.sum index 837ceb65b15..afa512e96d8 100644 --- a/go.sum +++ b/go.sum @@ -554,7 +554,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/update-codegen-dockerized.sh b/hack/update-codegen-dockerized.sh index c079a2a46b2..1395dacf192 100755 --- a/hack/update-codegen-dockerized.sh +++ b/hack/update-codegen-dockerized.sh @@ -92,7 +92,7 @@ MOCKGEN_TARGETS=( "pkg/agent/querier AgentQuerier" "pkg/controller/querier ControllerQuerier" "pkg/querier AgentNetworkPolicyInfoQuerier" - "pkg/agent/flowexporter/connections ConnTrackDumper,ConnTrackInterfacer" + "pkg/agent/flowexporter/connections ConnTrackDumper,NetFilterConnTrack" "pkg/agent/flowexporter/ipfix IPFIXExportingProcess,IPFIXRecord" ) diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 509fe3cf432..8ea6410f3c2 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -16,6 +16,10 @@ package connections import ( "fmt" + "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" + "net" "sync" "time" @@ -26,41 +30,31 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/openflow" ) -var _ ConnectionStore = new(connectionStore) - -type ConnectionStore interface { - // Run enables to poll conntrack connections periodically at given flowPollInterval - Run(stopCh <-chan struct{}, pollDone chan bool) - // Poll calls into conntrackDumper interface - Poll() (int, error) - // ForAllConnectionsDo execute the callback for each connection in connection map - ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error - // GetConnByKey gets the connection in connection map given the connection key - GetConnByKey(connKey flowexporter.ConnectionKey) (*flowexporter.Connection, bool) - // DeleteConnectionByKey deletes the connection in connection map given the connection key - DeleteConnectionByKey(connKey flowexporter.ConnectionKey) error -} - -type connectionStore struct { - connections map[flowexporter.ConnectionKey]flowexporter.Connection - mutex sync.Mutex - connDumper ConnTrackDumper - ifaceStore interfacestore.InterfaceStore +type ConnectionStore struct { + Connections map[flowexporter.ConnectionKey]flowexporter.Connection + ConnDumper ConnTrackDumper + IfaceStore interfacestore.InterfaceStore pollInterval time.Duration + mutex sync.Mutex } -func NewConnectionStore(ctDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration) *connectionStore { - return &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: ctDumper, - ifaceStore: ifaceStore, +func NewConnectionStore(ovsDatapathType string, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration) *ConnectionStore { + var connTrackDumper ConnTrackDumper + if ovsDatapathType == ovsconfig.OVSDatapathSystem { + connTrackDumper = NewConnTrackSystem(nodeConfig, serviceCIDR) + } else if ovsDatapathType == ovsconfig.OVSDatapathNetdev { + connTrackDumper = NewConnTrackOvsAppCtl(nodeConfig, serviceCIDR, ovsctlClient) + } + return &ConnectionStore{ + Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + ConnDumper: connTrackDumper, + IfaceStore: ifaceStore, pollInterval: pollInterval, } } -// Run polls the connTrackDumper module periodically to get connections. These connections are used -// to build connection store. -func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { +// Run enables the periodical polling of conntrack Connections, at the given flowPollInterval +func (cs *ConnectionStore) Run(stopCh <-chan struct{}, pollDone chan struct{}) { klog.Infof("Starting conntrack polling") pollTicker := time.NewTicker(cs.pollInterval) @@ -80,7 +74,7 @@ func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { // We need synchronization between ConnectionStore.Run and FlowExporter.Run go routines. // ConnectionStore.Run (connection poll) should be done to start FlowExporter.Run (connection export); pollDone signals helps enabling this. // FlowExporter.Run should be done to start ConnectionStore.Run; mutex on connection map object makes sure of this synchronization guarantee. - pollDone <- true + pollDone <- struct{}{} } } @@ -88,7 +82,7 @@ func (cs *connectionStore) Run(stopCh <-chan struct{}, pollDone chan bool) { // addOrUpdateConn updates the connection if it is already present, i.e., update timestamp, counters etc., // or adds a new Connection by 5-tuple of the flow along with local Pod and PodNameSpace. -func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { +func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { connKey := flowexporter.NewConnectionKey(conn) existingConn, exists := cs.GetConnByKey(connKey) @@ -105,12 +99,12 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn.ReversePackets = conn.ReversePackets existingConn.IsActive = true // Reassign the flow to update the map - cs.connections[connKey] = *existingConn + cs.Connections[connKey] = *existingConn klog.V(4).Infof("Antrea flow updated: %v", existingConn) } else { var srcFound, dstFound bool - sIface, srcFound := cs.ifaceStore.GetInterfaceByIP(conn.TupleOrig.SourceAddress.String()) - dIface, dstFound := cs.ifaceStore.GetInterfaceByIP(conn.TupleReply.SourceAddress.String()) + sIface, srcFound := cs.IfaceStore.GetInterfaceByIP(conn.TupleOrig.SourceAddress.String()) + dIface, dstFound := cs.IfaceStore.GetInterfaceByIP(conn.TupleReply.SourceAddress.String()) if !srcFound && !dstFound { klog.Warningf("Cannot map any of the IP %s or %s to a local Pod", conn.TupleOrig.SourceAddress.String(), conn.TupleReply.SourceAddress.String()) } @@ -123,30 +117,32 @@ func (cs *connectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodName = dIface.ContainerInterfaceConfig.PodName conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } - // Do not export flow records of connections, who destination is local pod and source is remote pod. + // Do not export flow records of Connections, whose destination is local pod and source is remote pod. // We export flow records only form "source node", where the connection is originated from. This is to avoid - // 2 copies of flow records. This restriction will be removed when flow records store network policy rule ID. + // 2 copies of flow records at flow collector. This restriction will be removed when flow records store network policy rule ID. // TODO: Remove this when network policy rule ID are added to flow records. if !srcFound && dstFound { conn.DoExport = false } klog.V(4).Infof("New Antrea flow added: %v", conn) // Add new antrea connection to connection store - cs.connections[connKey] = *conn + cs.Connections[connKey] = *conn } } -func (cs *connectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { +// GetConnByKey gets the connection in connection map given the connection key +func (cs *ConnectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { cs.mutex.Lock() defer cs.mutex.Unlock() - conn, found := cs.connections[flowTuple] + conn, found := cs.Connections[flowTuple] return &conn, found } -func (cs *connectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error { +// ForAllConnectionsDo execute the callback for each connection in connection map +func (cs *ConnectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error { cs.mutex.Lock() defer cs.mutex.Unlock() - for k, v := range cs.connections { + for k, v := range cs.Connections { err := callback(k, v) if err != nil { klog.Errorf("Callback execution failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) @@ -156,21 +152,22 @@ func (cs *connectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionM return nil } -// Poll returns number of filtered connections after poll cycle -// TODO: Optimize polling cycle--Only poll invalid/close connection during every poll. Poll established right before export -func (cs *connectionStore) Poll() (int, error) { +// Poll calls into conntrackDumper interface to dump conntrack flows +// TODO: As optimization, only poll invalid/closed Connections during every poll, and poll the established Connections right before the export. +func (cs *ConnectionStore) Poll() (int, error) { klog.V(2).Infof("Polling conntrack") - // Reset all connections in connection map before dumping flows in conntrack module. + // Reset isActive flag for all Connections in connection map before dumping flows in conntrack module. + // This is to specify that the connection and the flow record can be deleted after the next export. resetConn := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { conn.IsActive = false - cs.connections[key] = conn + cs.Connections[key] = conn return nil } // We do not expect any error as resetConn is not returning any error cs.ForAllConnectionsDo(resetConn) - filteredConns, err := cs.connDumper.DumpFlows(openflow.CtZone) + filteredConns, err := cs.ConnDumper.DumpFlows(openflow.CtZone) if err != nil { return 0, err } @@ -186,15 +183,15 @@ func (cs *connectionStore) Poll() (int, error) { return connsLen, nil } -// DeleteConnectionByKey after each IPFIX export of flow records. -func (cs *connectionStore) DeleteConnectionByKey(connKey flowexporter.ConnectionKey) error { +// DeleteConnectionByKey deletes the connection in connection map given the connection key +func (cs *ConnectionStore) DeleteConnectionByKey(connKey flowexporter.ConnectionKey) error { _, exists := cs.GetConnByKey(connKey) if !exists { return fmt.Errorf("connection with key %v doesn't exist in map", connKey) } cs.mutex.Lock() defer cs.mutex.Unlock() - delete(cs.connections, connKey) + delete(cs.Connections, connKey) return nil } diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index 19b63312a62..b497538c5f4 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -49,9 +49,9 @@ func makeTuple(srcIP *net.IP, dstIP *net.IP, protoID uint8, srcPort uint16, dstP func TestConnectionStore_addAndUpdateConn(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Create two flows; one is already in connectionStore and other one is new + // Create two flows; one is already in ConnectionStore and other one is new refTime := time.Now() - // Flow-1, which is already in connectionStore + // Flow-1, which is already in ConnectionStore tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) testFlow1 := flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 50)), @@ -64,7 +64,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { TupleReply: *revTuple1, IsActive: true, } - // Flow-2, which is not in connectionStore + // Flow-2, which is not in ConnectionStore tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) testFlow2 := flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 20)), @@ -107,15 +107,15 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { // Mock interface store with one of the couple of IPs correspond to Pods iStore := interfacestoretest.NewMockInterfaceStore(ctrl) mockCT := connectionstest.NewMockConnTrackDumper(ctrl) - // Create connectionStore - connStore := &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: mockCT, - ifaceStore: iStore, + // Create ConnectionStore + connStore := &ConnectionStore{ + Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + ConnDumper: mockCT, + IfaceStore: iStore, } // Add flow1conn to the Connection map testFlow1Tuple := flowexporter.NewConnectionKey(&testFlow1) - connStore.connections[testFlow1Tuple] = oldTestFlow1 + connStore.Connections[testFlow1Tuple] = oldTestFlow1 addOrUpdateConnTests := []struct { flow flowexporter.Connection @@ -147,11 +147,11 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Create two flows; one is already in connectionStore and other one is new + // Create two flows; one is already in ConnectionStore and other one is new testFlows := make([]*flowexporter.Connection, 2) testFlowKeys := make([]*flowexporter.ConnectionKey, 2) refTime := time.Now() - // Flow-1, which is already in connectionStore + // Flow-1, which is already in ConnectionStore tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) testFlows[0] = &flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 50)), @@ -164,7 +164,7 @@ func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { TupleReply: *revTuple1, IsActive: true, } - // Flow-2, which is not in connectionStore + // Flow-2, which is not in ConnectionStore tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) testFlows[1] = &flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 20)), @@ -181,21 +181,21 @@ func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { connKey := flowexporter.NewConnectionKey(flow) testFlowKeys[i] = &connKey } - // Create connectionStore - connStore := &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: nil, - ifaceStore: nil, + // Create ConnectionStore + connStore := &ConnectionStore{ + Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + ConnDumper: nil, + IfaceStore: nil, } // Add flows to the Connection store for i, flow := range testFlows { - connStore.connections[*testFlowKeys[i]] = *flow + connStore.Connections[*testFlowKeys[i]] = *flow } resetTwoFields := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { conn.IsActive = false conn.OriginalPackets = 0 - connStore.connections[key] = conn + connStore.Connections[key] = conn return nil } connStore.ForAllConnectionsDo(resetTwoFields) @@ -211,11 +211,11 @@ func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { func TestConnectionStore_DeleteConnectionByKey(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Create two flows; one is already in connectionStore and other one is new + // Create two flows; one is already in ConnectionStore and other one is new testFlows := make([]*flowexporter.Connection, 2) testFlowKeys := make([]*flowexporter.ConnectionKey, 2) refTime := time.Now() - // Flow-1, which is already in connectionStore + // Flow-1, which is already in ConnectionStore tuple1, revTuple1 := makeTuple(&net.IP{1, 2, 3, 4}, &net.IP{4, 3, 2, 1}, 6, 65280, 255) testFlows[0] = &flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 50)), @@ -228,7 +228,7 @@ func TestConnectionStore_DeleteConnectionByKey(t *testing.T) { TupleReply: *revTuple1, IsActive: true, } - // Flow-2, which is not in connectionStore + // Flow-2, which is not in ConnectionStore tuple2, revTuple2 := makeTuple(&net.IP{5, 6, 7, 8}, &net.IP{8, 7, 6, 5}, 6, 60001, 200) testFlows[1] = &flowexporter.Connection{ StartTime: refTime.Add(-(time.Second * 20)), @@ -245,17 +245,17 @@ func TestConnectionStore_DeleteConnectionByKey(t *testing.T) { connKey := flowexporter.NewConnectionKey(flow) testFlowKeys[i] = &connKey } - // Create connectionStore - connStore := &connectionStore{ - connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - connDumper: nil, - ifaceStore: nil, + // Create ConnectionStore + connStore := &ConnectionStore{ + Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + ConnDumper: nil, + IfaceStore: nil, } // Add flows to the connection store. for i, flow := range testFlows { - connStore.connections[*testFlowKeys[i]] = *flow + connStore.Connections[*testFlowKeys[i]] = *flow } - // Delete the connections in connection store. + // Delete the Connections in connection store. for i := 0; i < len(testFlows); i++ { err := connStore.DeleteConnectionByKey(*testFlowKeys[i]) assert.Nil(t, err, "DeleteConnectionByKey should return nil") diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index a0132d06bf3..05bb8b16de1 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -15,28 +15,188 @@ package connections import ( + "fmt" "net" + "strconv" + "strings" + + "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" + "github.com/vmware-tanzu/antrea/pkg/util/ip" ) -var _ ConnTrackDumper = new(connTrackDumper) +// connTrackOvsCtl implements ConnTrackDumper. This supports OVS userspace datapath scenarios. +var _ ConnTrackDumper = new(connTrackOvsCtl) -type connTrackDumper struct { - connTrack ConnTrackInterfacer +type connTrackOvsCtl struct { nodeConfig *config.NodeConfig serviceCIDR *net.IPNet - datapathType string ovsctlClient ovsctl.OVSCtlClient } -func NewConnTrackDumper(connTrack ConnTrackInterfacer, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, dpType string, ovsctlClient ovsctl.OVSCtlClient) *connTrackDumper { - return &connTrackDumper{ - connTrack, +func NewConnTrackOvsAppCtl(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient) *connTrackOvsCtl { + return &connTrackOvsCtl{ nodeConfig, serviceCIDR, - dpType, ovsctlClient, } } + +// DumpFlows uses "ovs-appctl dpctl/dump-conntrack" to dump conntrack flows in the Antrea ZoneID. +func (ct *connTrackOvsCtl) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { + conns, err := ct.ovsAppctlDumpConnections(zoneFilter) + if err != nil { + klog.Errorf("Error when dumping flows from conntrack: %v", err) + return nil, err + } + + filteredConns := filterAntreaConns(conns, ct.nodeConfig, ct.serviceCIDR, zoneFilter) + klog.V(2).Infof("Flow exporter considered flows: %d", len(filteredConns)) + + return filteredConns, nil +} + +func (ct *connTrackOvsCtl) ovsAppctlDumpConnections(zoneFilter uint16) ([]*flowexporter.Connection, error) { + // Dump conntrack using ovs-appctl dpctl/dump-conntrack + cmdOutput, execErr := ct.ovsctlClient.RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s") + if execErr != nil { + return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) + } + + // Parse the output to get the flows + antreaConns := make([]*flowexporter.Connection, 0) + outputFlow := strings.Split(string(cmdOutput), "\n") + var err error + for _, flow := range outputFlow { + conn := flowexporter.Connection{} + flowSlice := strings.Split(flow, ",") + isReply := false + inZone := false + for _, fs := range flowSlice { + // Indicator to populate reply or reverse fields + if strings.Contains(fs, "reply") { + isReply = true + } + if !strings.Contains(fs, "=") { + // Proto identifier + conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) + if err != nil { + klog.Errorf("Unknown protocol to convert to ID: %s", fs) + continue + } + conn.TupleReply.Protocol = conn.TupleOrig.Protocol + } else if strings.Contains(fs, "src") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "dst") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "sport") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.SourcePort = uint16(val) + } else { + conn.TupleReply.SourcePort = uint16(val) + } + } else if strings.Contains(fs, "dport") { + // dport field could be the last tuple field in ovs-dpctl output format. + fs = strings.TrimSuffix(fs, ")") + + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.DestinationPort = uint16(val) + } else { + conn.TupleReply.DestinationPort = uint16(val) + } + } else if strings.Contains(fs, "zone") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) + continue + } + if zoneFilter != uint16(val) { + break + } else { + inZone = true + conn.Zone = uint16(val) + } + } else if strings.Contains(fs, "timeout") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) + continue + } + conn.Timeout = uint32(val) + } else if strings.Contains(fs, "id") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) + continue + } + conn.ID = uint32(val) + } + } + if inZone { + conn.IsActive = true + conn.DoExport = true + antreaConns = append(antreaConns, &conn) + } + } + klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + return antreaConns, nil +} + +func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, zoneFilter uint16) []*flowexporter.Connection { + filteredConns := conns[:0] + for _, conn := range conns { + if conn.Zone != zoneFilter { + continue + } + srcIP := conn.TupleOrig.SourceAddress + dstIP := conn.TupleReply.SourceAddress + + // Only get Pod-to-Pod flows. + if srcIP.Equal(nodeConfig.GatewayConfig.IP) || dstIP.Equal(nodeConfig.GatewayConfig.IP) { + klog.V(4).Infof("Detected flow through gateway") + continue + } + + // Pod-to-Service flows w/ kube-proxy: There are two conntrack flows for every Pod-to-Service flow. + // One is with ClusterIP as source or destination, where other IP is podIP. Second conntrack flow is + // with resolved Endpoint Pod IP corresponding to ClusterIP. Both conntrack flows have same stats, which makes them duplicate. + // Ideally, we have to correlate these two Connections and maintain one connection with both Endpoint Pod IP and ClusterIP. + // To do the correlation, we need ClusterIP-to-EndpointIP mapping info, which is not available at Agent. + // Therefore, we ignore the connection with ClusterIP and keep the connection with Endpoint Pod IP. + // Conntrack flows will be different for Pod-to-Service flows w/ Antrea-proxy. This implementation will be simpler, when the + // Antrea proxy is supported. + if serviceCIDR.Contains(srcIP) || serviceCIDR.Contains(dstIP) { + continue + } + filteredConns = append(filteredConns, conn) + } + return filteredConns +} diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 70118e31994..51437762484 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -17,296 +17,98 @@ package connections import ( - "fmt" "net" - "strconv" - "strings" "github.com/ti-mo/conntrack" "k8s.io/klog/v2" + "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" - "github.com/vmware-tanzu/antrea/pkg/agent/openflow" "github.com/vmware-tanzu/antrea/pkg/agent/util/sysctl" - "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" - "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" - "github.com/vmware-tanzu/antrea/pkg/util/ip" ) -// DumpFlows opens netlink connection and dumps all the flows in Antrea ZoneID -// of conntrack table, i.e., corresponding to Antrea OVS bridge. -func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - if ctdump.datapathType == ovsconfig.OVSDatapathSystem { - // Get connection to netlink socket - err := ctdump.connTrack.GetConnTrack(nil) - if err != nil { - klog.Errorf("Error when getting netlink conn: %v", err) - return nil, err - } - } else if ctdump.datapathType == ovsconfig.OVSDatapathNetdev { - // Set ovsCtlClient to dump conntrack flows - err := ctdump.connTrack.GetConnTrack(ctdump.ovsctlClient) - if err != nil { - klog.Errorf("Error when getting ovsclient: %v", err) - return nil, err - } - } - - // ZoneID filter is not supported currently in tl-mo/conntrack library. - // Link to issue: https://github.com/ti-mo/conntrack/issues/23 - // Dump all flows in the conntrack table for now. - var conns []*flowexporter.Connection - var err error - if ctdump.datapathType == ovsconfig.OVSDatapathSystem { - conns, err = ctdump.connTrack.DumpFilter(conntrack.Filter{}) - if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) - return nil, err - } - } else if ctdump.datapathType == ovsconfig.OVSDatapathNetdev { - // This is supported for kind clusters. Ovs-appctl access in kind clusters is unstable currently. - // This will be used once the issue with Ovs-appctl is fixed on kind cluster nodes. - conns, err = ctdump.connTrack.DumpFilter(uint16(openflow.CtZone)) - if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) - return nil, err - } - } - - for i := 0; i < len(conns); i++ { - if conns[i].Zone != openflow.CtZone { - // Delete the element from the slice - conns[i] = conns[len(conns)-1] - conns[len(conns)-1] = nil - conns = conns[:len(conns)-1] - // Decrement i to iterate over swapped element - i = i - 1 - continue - } - srcIP := conns[i].TupleOrig.SourceAddress - dstIP := conns[i].TupleReply.SourceAddress - - // Only get Pod-to-Pod flows. Pod-to-ExternalService flows are ignored for now. - if srcIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) || dstIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) { - // Delete the element from the slice - conns[i] = conns[len(conns)-1] - conns[len(conns)-1] = nil - conns = conns[:len(conns)-1] - // Decrement i to iterate over swapped element - i = i - 1 - continue - } - - // Pod-to-Service flows w/ kube-proxy: There are two conntrack flows for every Pod-to-Service flow. - // One is with ClusterIP as source or destination, where other IP is podIP. Second conntrack flow is - // with resolved Endpoint Pod IP corresponding to ClusterIP. Both conntrack flows have same stats, which makes them duplicate. - // Ideally, we have to correlate these two connections and maintain one connection with both Endpoint Pod IP and ClusterIP. - // To do the correlation, we need ClusterIP-to-EndpointIP mapping info, which is not available at Agent. - // Therefore, we ignore the connection with ClusterIP and keep the connection with Endpoint Pod IP. - // Conntrack flows will be different for Pod-to-Service flows w/ Antrea-proxy. This implementation will be simpler, when the - // Antrea proxy is supported. - if ctdump.serviceCIDR.Contains(srcIP) || ctdump.serviceCIDR.Contains(dstIP) { - // Delete element from the slice - conns[i] = conns[len(conns)-1] - conns[len(conns)-1] = nil - conns = conns[:len(conns)-1] - // Decrement i to iterate over swapped element - i = i - 1 - continue - } - } - klog.V(2).Infof("No. of flow exporter considered flows in Antrea zoneID: %d", len(conns)) - - return conns, nil -} - -// connTrackSystem implements ConnTrackInterfacer -var _ ConnTrackInterfacer = new(connTrackSystem) -var _ ConnTrackInterfacer = new(connTrackNetdev) +// connTrackSystem implements ConnTrackDumper. This is for linux kernel datapath. +var _ ConnTrackDumper = new(connTrackSystem) type connTrackSystem struct { - netlinkConn *conntrack.Conn + nodeConfig *config.NodeConfig + serviceCIDR *net.IPNet + connTrack NetFilterConnTrack } -func NewConnTrackSystem() *connTrackSystem { - // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of connections. +func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackSystem { + // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of Connections. // Do not handle error and continue with creation of interfacer object as we can still dump flows with no stats. // If log says permission error, please ensure net.netfilter.nf_conntrack_acct to be set to 1. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_acct", 1) - // Ensure net.netfilter.nf_conntrack_timestamp value to be 1. This will enable flow exporter to export timestamps of connections. + // Ensure net.netfilter.nf_conntrack_timestamp value to be 1. This will enable flow exporter to export timestamps of Connections. // Do not handle error and continue with creation of interfacer object as we can still dump flows with no timestamps. // If log says permission error, please ensure net.netfilter.nf_conntrack_timestamp to be set to 1. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_timestamp", 1) - return &connTrackSystem{} + return &connTrackSystem{ + nodeConfig, + serviceCIDR, + &netFilterConnTrack{}, + } } -type connTrackNetdev struct { - ovsCtl ovsctl.OVSCtlClient +// DumpFlows opens netlink connection and dumps all the flows in Antrea ZoneID of conntrack table. +func (ct *connTrackSystem) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { + // Get connection to netlink socket + err := ct.connTrack.Dial() + if err != nil { + klog.Errorf("Error when getting netlink socket: %v", err) + return nil, err + } + + // ZoneID filter is not supported currently in tl-mo/conntrack library. + // Link to issue: https://github.com/ti-mo/conntrack/issues/23 + // Dump all flows in the conntrack table for now. + conns, err := ct.connTrack.DumpFilter(conntrack.Filter{}) + if err != nil { + klog.Errorf("Error when dumping flows from conntrack: %v", err) + return nil, err + } + filteredConns := filterAntreaConns(conns, ct.nodeConfig, ct.serviceCIDR, zoneFilter) + klog.V(2).Infof("No. of flow exporter considered flows in Antrea zoneID: %d", len(filteredConns)) + + return filteredConns, nil } -func NewConnTrackNetdev() *connTrackNetdev { - return &connTrackNetdev{} +// NetFilterConnTrack interface helps for testing the code that contains the third party library functions ("github.com/ti-mo/conntrack") +type NetFilterConnTrack interface { + Dial() error + DumpFilter(filter conntrack.Filter) ([]*flowexporter.Connection, error) } -func (ctnl *connTrackSystem) GetConnTrack(config interface{}) error { - if config != nil { - return fmt.Errorf("this function does not expect any netlink config") - } +type netFilterConnTrack struct { + netlinkConn *conntrack.Conn +} + +func (nfct *netFilterConnTrack) Dial() error { // Get netlink client in current namespace conn, err := conntrack.Dial(nil) if err != nil { - klog.Errorf("Error when dialing conntrack: %v", err) return err } - ctnl.netlinkConn = conn + nfct.netlinkConn = conn return nil } -func (ctnl *connTrackSystem) DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) { - netlinkFilter, ok := filter.(conntrack.Filter) - if !ok { - return nil, fmt.Errorf("error: filter should be of type conntrack.Filter") - } - conns, err := ctnl.netlinkConn.DumpFilter(netlinkFilter) +func (nfct *netFilterConnTrack) DumpFilter(filter conntrack.Filter) ([]*flowexporter.Connection, error) { + conns, err := nfct.netlinkConn.DumpFilter(filter) if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) return nil, err } antreaConns := make([]*flowexporter.Connection, len(conns)) for i, conn := range conns { antreaConns[i] = createAntreaConn(&conn) } - conns = nil klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) - ctnl.netlinkConn.Close() - return antreaConns, nil -} - -func (ctnd *connTrackNetdev) GetConnTrack(config interface{}) error { - client, ok := config.(ovsctl.OVSCtlClient) - if !ok { - return fmt.Errorf("config should be ovsCtlClient of type OVSCtlClient") - } - ctnd.ovsCtl = client - return nil -} - -func (ctnd *connTrackNetdev) DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) { - zoneFilter, ok := filter.(uint16) - if !ok { - return nil, fmt.Errorf("filter should be of type uint16") - } - - // Dump conntrack using ovs-appctl dpctl/dump-conntrack - cmdOutput, execErr := ctnd.ovsCtl.RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s") - if execErr != nil { - return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) - } - - // Parse the output to get the flows - antreaConns := make([]*flowexporter.Connection, 0) - outputFlow := strings.Split(string(cmdOutput), "\n") - var err error - for _, flow := range outputFlow { - conn := flowexporter.Connection{} - flowSlice := strings.Split(flow, ",") - isReply := false - inZone := false - for _, fs := range flowSlice { - // Indicator to populate reply or reverse fields - if strings.Contains(fs, "reply") { - isReply = true - } - if !strings.Contains(fs, "=") { - // Proto identifier - conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) - if err != nil { - klog.Errorf("Unknown protocol to convert to ID: %s", fs) - continue - } - conn.TupleReply.Protocol = conn.TupleOrig.Protocol - } else if strings.Contains(fs, "src") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "dst") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "sport") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.SourcePort = uint16(val) - } else { - conn.TupleReply.SourcePort = uint16(val) - } - } else if strings.Contains(fs, "dport") { - // dport field could be the last tuple field in ovs-dpctl output format. - fs = strings.TrimSuffix(fs, ")") - - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.DestinationPort = uint16(val) - } else { - conn.TupleReply.DestinationPort = uint16(val) - } - } else if strings.Contains(fs, "zone") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) - continue - } - if zoneFilter != uint16(val) { - break - } else { - inZone = true - conn.Zone = uint16(val) - } - } else if strings.Contains(fs, "timeout") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) - continue - } - conn.Timeout = uint32(val) - } else if strings.Contains(fs, "id") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) - continue - } - conn.ID = uint32(val) - } - } - if inZone { - conn.IsActive = true - conn.DoExport = true - antreaConns = append(antreaConns, &conn) - } - } - klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + nfct.netlinkConn.Close() return antreaConns, nil } diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index 70eca92e0c2..ed5106246aa 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -29,11 +29,10 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" connectionstest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections/testing" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" - "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" ovsctltest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl/testing" ) -func TestConnTrack_DumpFlows(t *testing.T) { +func TestConnTrackSystem_DumpFlows(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Create flows for test @@ -62,9 +61,6 @@ func TestConnTrack_DumpFlows(t *testing.T) { } testFlows := []*flowexporter.Connection{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} - // Create mock interfaces - mockCTInterfacer := connectionstest.NewMockConnTrackInterfacer(ctrl) - mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) // Create nodeConfig and gateWayConfig // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP gwConfig := &config.GatewayConfig{ @@ -78,47 +74,48 @@ func TestConnTrack_DumpFlows(t *testing.T) { IP: net.IP{100, 50, 25, 0}, Mask: net.IPMask{255, 255, 255, 0}, } - - // Test DumpFlows implementation of connTrackSystem - connDumperDPSystem := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathSystem, mockOVSCtlClient) - // Set expects for mocks - mockCTInterfacer.EXPECT().GetConnTrack(nil).Return(nil) - mockCTInterfacer.EXPECT().DumpFilter(conntrack.Filter{}).Return(testFlows, nil) - - conns, err := connDumperDPSystem.DumpFlows(openflow.CtZone) - if err != nil { - t.Errorf("Dump flows function returned error: %v", err) + // Test the DumpFlows implementation of connTrackSystem + mockNetlinkCT := connectionstest.NewMockNetFilterConnTrack(ctrl) + connDumperDPSystem := &connTrackSystem{ + nodeConfig, + serviceCIDR, + mockNetlinkCT, } - assert.Equal(t, 1, len(conns), "number of filtered connections should be equal") - - // Test DumpFlows implementation of connTrackNetdev - connDumperDPNetdev := NewConnTrackDumper(mockCTInterfacer, nodeConfig, serviceCIDR, ovsconfig.OVSDatapathNetdev, mockOVSCtlClient) - // Re-initialize testFlows - testFlows = []*flowexporter.Connection{antreaFlow, antreaServiceFlow, antreaGWFlow, nonAntreaFlow} // Set expects for mocks - mockCTInterfacer.EXPECT().GetConnTrack(mockOVSCtlClient).Return(nil) - mockCTInterfacer.EXPECT().DumpFilter(uint16(openflow.CtZone)).Return(testFlows, nil) + mockNetlinkCT.EXPECT().Dial().Return(nil) + mockNetlinkCT.EXPECT().DumpFilter(conntrack.Filter{}).Return(testFlows, nil) - conns, err = connDumperDPNetdev.DumpFlows(openflow.CtZone) + conns, err := connDumperDPSystem.DumpFlows(openflow.CtZone) if err != nil { t.Errorf("Dump flows function returned error: %v", err) } - assert.Equal(t, 1, len(conns), "number of filtered connections should be equal") + assert.Equal(t, 1, len(conns), "number of filtered Connections should be equal") } -func TestConnTackNetdev_DumpFilter(t *testing.T) { +func TestConnTackOvsAppCtl_DumpFlows(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Create mock interfaces + // Create mock interface mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) - conntrackNetdev := NewConnTrackNetdev() - err := conntrackNetdev.GetConnTrack(mockOVSCtlClient) - assert.Nil(t, err, "GetConnTrack call should be successful") + // Create nodeConfig and gateWayConfig + // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP + gwConfig := &config.GatewayConfig{ + IP: net.IP{8, 7, 6, 5}, + } + nodeConfig := &config.NodeConfig{ + GatewayConfig: gwConfig, + } + // Create serviceCIDR + serviceCIDR := &net.IPNet{ + IP: net.IP{100, 50, 25, 0}, + Mask: net.IPMask{255, 255, 255, 0}, + } + connDumper := NewConnTrackOvsAppCtl(nodeConfig, serviceCIDR, mockOVSCtlClient) // Set expect call for mock ovsCtlClient ovsctlCmdOutput := []byte("tcp,orig=(src=127.0.0.1,dst=127.0.0.1,sport=45218,dport=2379,packets=320108,bytes=24615344),reply=(src=127.0.0.1,dst=127.0.0.1,sport=2379,dport=45218,packets=239595,bytes=24347883),start=2020-07-24T05:07:03.998,id=3750535678,status=SEEN_REPLY|ASSURED|CONFIRMED|SRC_NAT_DONE|DST_NAT_DONE,timeout=86399,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)\n" + - "tcp,orig=(src=127.0.0.1,dst=127.0.0.1,sport=45170,dport=2379,packets=80743,bytes=5416239),reply=(src=127.0.0.1,dst=127.0.0.1,sport=2379,dport=45170,packets=63361,bytes=4811261),start=2020-07-24T05:07:01.591,id=462801621,status=SEEN_REPLY|ASSURED|CONFIRMED|SRC_NAT_DONE|DST_NAT_DONE,timeout=86397,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)\n" + + "tcp,orig=(src=127.0.0.1,dst=8.7.6.5,sport=45170,dport=2379,packets=80743,bytes=5416239),reply=(src=8.7.6.5,dst=127.0.0.1,sport=2379,dport=45170,packets=63361,bytes=4811261),start=2020-07-24T05:07:01.591,id=462801621,zone=65520,status=SEEN_REPLY|ASSURED|CONFIRMED|SRC_NAT_DONE|DST_NAT_DONE,timeout=86397,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)\n" + "tcp,orig=(src=100.10.0.105,dst=10.96.0.1,sport=41284,dport=443,packets=343260,bytes=19340621),reply=(src=192.168.86.82,dst=100.10.0.105,sport=6443,dport=41284,packets=381035,bytes=181176472),start=2020-07-25T08:40:08.959,id=982464968,zone=65520,status=SEEN_REPLY|ASSURED|CONFIRMED|DST_NAT|DST_NAT_DONE,timeout=86399,mark=33,protoinfo=(state_orig=ESTABLISHED,state_reply=ESTABLISHED,wscale_orig=7,wscale_reply=7,flags_orig=WINDOW_SCALE|SACK_PERM|MAXACK_SET,flags_reply=WINDOW_SCALE|SACK_PERM|MAXACK_SET)") expConn := &flowexporter.Connection{ ID: 982464968, @@ -154,9 +151,9 @@ func TestConnTackNetdev_DumpFilter(t *testing.T) { } mockOVSCtlClient.EXPECT().RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s").Return(ovsctlCmdOutput, nil) - conns, err := conntrackNetdev.DumpFilter(uint16(openflow.CtZone)) + conns, err := connDumper.DumpFlows(uint16(openflow.CtZone)) if err != nil { - t.Errorf("conntrackNetdev.DumpFilter function returned error: %v", err) + t.Errorf("conntrackNetdev.DumpConnections function returned error: %v", err) } assert.Equal(t, len(conns), 1) assert.Equal(t, conns[0], expConn, "filtered connection and expected connection should be same") diff --git a/pkg/agent/flowexporter/connections/conntrack_windows.go b/pkg/agent/flowexporter/connections/conntrack_windows.go index c4d1f81bd47..74ff4e26db4 100644 --- a/pkg/agent/flowexporter/connections/conntrack_windows.go +++ b/pkg/agent/flowexporter/connections/conntrack_windows.go @@ -17,18 +17,21 @@ package connections import ( + "net" + + "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ) -func (cp *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - return nil, nil -} +// TODO: Implement ConnTrackDumper for windows. +var _ ConnTrackDumper = new(connTrackSystem) -// TODO: Implement ConnTrackInterfacer when flow exporter is supported for windows. -func NewConnTrackSystem() ConnTrackInterfacer { +type connTrackSystem struct{} + +func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackSystem { return nil } -func NewConnTrackNetdev() ConnTrackInterfacer { - return nil +func (ct *connTrackSystem) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { + return nil, nil } diff --git a/pkg/agent/flowexporter/connections/interface.go b/pkg/agent/flowexporter/connections/interface.go index d510be271d3..f13506e7ab0 100644 --- a/pkg/agent/flowexporter/connections/interface.go +++ b/pkg/agent/flowexporter/connections/interface.go @@ -18,15 +18,9 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ) -// ConnTrackDumper is an interface that is used to dump connections from -// conntrack module. +// ConnTrackDumper is an interface that is used to dump Connections from conntrack module. This supports dumping through +// netfilter socket (OVS kernel datapath) and ovs-appctl command (OVS userspace datapath). +// In future, support will be extended to Windows. type ConnTrackDumper interface { DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) } - -// ConnTrackInterfacer is an interface created to consume the required dump functions from either the third party -// conntrack library or internal packages depending on OVS datapath type or OS. -type ConnTrackInterfacer interface { - GetConnTrack(config interface{}) error // suggest a different name for config if it is not appropriate - DumpFilter(filter interface{}) ([]*flowexporter.Connection, error) -} diff --git a/pkg/agent/flowexporter/connections/testing/mock_connections.go b/pkg/agent/flowexporter/connections/testing/mock_connections.go index 6a26b3cc296..8a11d028a73 100644 --- a/pkg/agent/flowexporter/connections/testing/mock_connections.go +++ b/pkg/agent/flowexporter/connections/testing/mock_connections.go @@ -14,13 +14,14 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections (interfaces: ConnTrackDumper,ConnTrackInterfacer) +// Source: github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections (interfaces: ConnTrackDumper,NetFilterConnTrack) // Package testing is a generated GoMock package. package testing import ( gomock "github.com/golang/mock/gomock" + conntrack "github.com/ti-mo/conntrack" flowexporter "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" reflect "reflect" ) @@ -63,54 +64,54 @@ func (mr *MockConnTrackDumperMockRecorder) DumpFlows(arg0 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpFlows", reflect.TypeOf((*MockConnTrackDumper)(nil).DumpFlows), arg0) } -// MockConnTrackInterfacer is a mock of ConnTrackInterfacer interface -type MockConnTrackInterfacer struct { +// MockNetFilterConnTrack is a mock of NetFilterConnTrack interface +type MockNetFilterConnTrack struct { ctrl *gomock.Controller - recorder *MockConnTrackInterfacerMockRecorder + recorder *MockNetFilterConnTrackMockRecorder } -// MockConnTrackInterfacerMockRecorder is the mock recorder for MockConnTrackInterfacer -type MockConnTrackInterfacerMockRecorder struct { - mock *MockConnTrackInterfacer +// MockNetFilterConnTrackMockRecorder is the mock recorder for MockNetFilterConnTrack +type MockNetFilterConnTrackMockRecorder struct { + mock *MockNetFilterConnTrack } -// NewMockConnTrackInterfacer creates a new mock instance -func NewMockConnTrackInterfacer(ctrl *gomock.Controller) *MockConnTrackInterfacer { - mock := &MockConnTrackInterfacer{ctrl: ctrl} - mock.recorder = &MockConnTrackInterfacerMockRecorder{mock} +// NewMockNetFilterConnTrack creates a new mock instance +func NewMockNetFilterConnTrack(ctrl *gomock.Controller) *MockNetFilterConnTrack { + mock := &MockNetFilterConnTrack{ctrl: ctrl} + mock.recorder = &MockNetFilterConnTrackMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use -func (m *MockConnTrackInterfacer) EXPECT() *MockConnTrackInterfacerMockRecorder { +func (m *MockNetFilterConnTrack) EXPECT() *MockNetFilterConnTrackMockRecorder { return m.recorder } -// DumpFilter mocks base method -func (m *MockConnTrackInterfacer) DumpFilter(arg0 interface{}) ([]*flowexporter.Connection, error) { +// Dial mocks base method +func (m *MockNetFilterConnTrack) Dial() error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DumpFilter", arg0) - ret0, _ := ret[0].([]*flowexporter.Connection) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "Dial") + ret0, _ := ret[0].(error) + return ret0 } -// DumpFilter indicates an expected call of DumpFilter -func (mr *MockConnTrackInterfacerMockRecorder) DumpFilter(arg0 interface{}) *gomock.Call { +// Dial indicates an expected call of Dial +func (mr *MockNetFilterConnTrackMockRecorder) Dial() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpFilter", reflect.TypeOf((*MockConnTrackInterfacer)(nil).DumpFilter), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Dial", reflect.TypeOf((*MockNetFilterConnTrack)(nil).Dial)) } -// GetConnTrack mocks base method -func (m *MockConnTrackInterfacer) GetConnTrack(arg0 interface{}) error { +// DumpFilter mocks base method +func (m *MockNetFilterConnTrack) DumpFilter(arg0 conntrack.Filter) ([]*flowexporter.Connection, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConnTrack", arg0) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "DumpFilter", arg0) + ret0, _ := ret[0].([]*flowexporter.Connection) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// GetConnTrack indicates an expected call of GetConnTrack -func (mr *MockConnTrackInterfacerMockRecorder) GetConnTrack(arg0 interface{}) *gomock.Call { +// DumpFilter indicates an expected call of DumpFilter +func (mr *MockNetFilterConnTrackMockRecorder) DumpFilter(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConnTrack", reflect.TypeOf((*MockConnTrackInterfacer)(nil).GetConnTrack), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpFilter", reflect.TypeOf((*MockNetFilterConnTrack)(nil).DumpFilter), arg0) } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 1dd444c2820..8295f16da45 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -64,20 +64,13 @@ var ( } ) -var _ FlowExporter = new(flowExporter) - -type FlowExporter interface { - // Run enables to export flow records periodically at given export interval - Run(stopCh <-chan struct{}, pollDone <-chan bool) -} - type flowExporter struct { - flowRecords flowrecords.FlowRecords - process ipfix.IPFIXExportingProcess - elementsList []*ipfixentities.InfoElement - exportInterval time.Duration - pollInterval time.Duration - templateID uint16 + flowRecords *flowrecords.FlowRecords + process ipfix.IPFIXExportingProcess + elementsList []*ipfixentities.InfoElement + exportFrequency uint + pollInterval time.Duration + templateID uint16 } func genObservationID() (uint32, error) { @@ -90,18 +83,18 @@ func genObservationID() (uint32, error) { return h.Sum32(), nil } -func NewFlowExporter(records flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, expInterval time.Duration, pollInterval time.Duration, tempID uint16) *flowExporter { +func NewFlowExporter(records *flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, exportFrequency uint, pollInterval time.Duration, tempID uint16) *flowExporter { return &flowExporter{ records, expProcess, elemList, - expInterval, + exportFrequency, pollInterval, tempID, } } -func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expInterval time.Duration, pollInterval time.Duration) (*flowExporter, error) { +func InitFlowExporter(collector net.Addr, records *flowrecords.FlowRecords, exportFrequency uint, pollInterval time.Duration) (*flowExporter, error) { // Create IPFIX exporting expProcess and initialize registries and other related entities obsID, err := genObservationID() if err != nil { @@ -121,7 +114,7 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn } expProcess.LoadRegistries() - flowExp := NewFlowExporter(records, expProcess, nil, expInterval, pollInterval, expProcess.NewTemplateID()) + flowExp := NewFlowExporter(records, expProcess, nil, exportFrequency, pollInterval, expProcess.NewTemplateID()) templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), flowExp.templateID) @@ -134,9 +127,10 @@ func InitFlowExporter(collector net.Addr, records flowrecords.FlowRecords, expIn return flowExp, nil } -func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan bool) { +// Run enables to export flow records periodically at a given flow export frequency +func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan struct{}) { klog.Infof("Start exporting IPFIX flow records") - ticker := time.NewTicker(exp.exportInterval) + ticker := time.NewTicker(time.Duration(exp.exportFrequency) * exp.pollInterval) defer ticker.Stop() for { @@ -146,11 +140,9 @@ func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan bool) { break case <-ticker.C: // Waiting for expected number of pollDone signals from go routine(ConnectionStore.Run) is necessary because - // IPFIX collector computes throughput based on flow records received interval. Expected number of pollDone - // signals should be equal to the number of pollCycles to be done before starting the export cycle; it is computed - // from flow poll interval and flow export interval. Note that export interval is multiple of poll interval, - // and poll interval is at least one second long. - for i := uint(0); i < uint(exp.exportInterval/exp.pollInterval); i++ { + // IPFIX collector computes throughput based on flow records received interval. Number of pollDone + // signals should be equal to export frequency before starting the export cycle. + for i := uint(0); i < exp.exportFrequency; i++ { <-pollDone } err := exp.flowRecords.BuildFlowRecords() diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 92a9c1e903b..4b18949181d 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -21,7 +21,7 @@ import ( "unicode" "github.com/golang/mock/gomock" - "gotest.tools/assert" + "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ipfixtest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix/testing" @@ -38,7 +38,7 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { nil, mockIPFIXExpProc, nil, - 60 * time.Second, + 12, time.Second, 256, } @@ -146,7 +146,7 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { nil, mockIPFIXExpProc, elemList, - 60 * time.Second, + 12, time.Second, 256, } diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index 276b4066a06..f403ab6dfdf 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -22,34 +22,21 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections" ) -var _ FlowRecords = new(flowRecords) - -type FlowRecords interface { - // BuildFlowRecords builds the flow record map from connection map in connection store - BuildFlowRecords() error - // GetFlowRecordByConnKey gets the record from the flow record map given the connection key - GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) - // ValidateAndUpdateStats validates and updates the flow record given the connection key - ValidateAndUpdateStats(connKey flowexporter.ConnectionKey, record flowexporter.FlowRecord) error - // ForAllFlowRecordsDo executes the callback for all records in the flow record map - ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallBack) error -} - -type flowRecords struct { - // synchronization is not required as there is no concurrency, involving this object. +type FlowRecords struct { // Add lock when this map is consumed by more than one entity concurrently. recordsMap map[flowexporter.ConnectionKey]flowexporter.FlowRecord - connStore connections.ConnectionStore + connStore *connections.ConnectionStore } -func NewFlowRecords(connStore connections.ConnectionStore) *flowRecords { - return &flowRecords{ +func NewFlowRecords(connStore *connections.ConnectionStore) *FlowRecords { + return &FlowRecords{ make(map[flowexporter.ConnectionKey]flowexporter.FlowRecord), connStore, } } -func (fr *flowRecords) BuildFlowRecords() error { +// BuildFlowRecords builds the flow record map from connection map in connection store +func (fr *FlowRecords) BuildFlowRecords() error { err := fr.connStore.ForAllConnectionsDo(fr.addOrUpdateFlowRecord) if err != nil { return fmt.Errorf("error when iterating connection map: %v", err) @@ -58,12 +45,14 @@ func (fr *flowRecords) BuildFlowRecords() error { return nil } -func (fr *flowRecords) GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) { +// GetFlowRecordByConnKey gets the record from the flow record map given the connection key +func (fr *FlowRecords) GetFlowRecordByConnKey(connKey flowexporter.ConnectionKey) (*flowexporter.FlowRecord, bool) { record, found := fr.recordsMap[connKey] return &record, found } -func (fr *flowRecords) ValidateAndUpdateStats(connKey flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { +// ValidateAndUpdateStats validates and updates the flow record given the connection key +func (fr *FlowRecords) ValidateAndUpdateStats(connKey flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { // Delete the flow record if the corresponding connection is not active, i.e., not present in conntrack table. // Delete the corresponding connection in connectionMap as well. if !record.Conn.IsActive { @@ -84,7 +73,8 @@ func (fr *flowRecords) ValidateAndUpdateStats(connKey flowexporter.ConnectionKey return nil } -func (fr *flowRecords) ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallBack) error { +// ForAllFlowRecordsDo executes the callback for all records in the flow record map +func (fr *FlowRecords) ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallBack) error { for k, v := range fr.recordsMap { err := callback(k, v) if err != nil { @@ -96,7 +86,7 @@ func (fr *flowRecords) ForAllFlowRecordsDo(callback flowexporter.FlowRecordCallB return nil } -func (fr *flowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { +func (fr *FlowRecords) addOrUpdateFlowRecord(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { // If DoExport flag is not set return immediately. if !conn.DoExport { return nil diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 46bf5faf336..20c2ea60129 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -83,8 +83,6 @@ func setupTest(tb testing.TB) (*TestData, error) { func setupTestWithIPFIXCollector(tb testing.TB) (*TestData, error) { data := &TestData{} tb.Logf("Creating K8s clientset") - // TODO: it is probably not needed to re-create the clientset in each test, maybe we could - // just keep it in clusterInfo? if err := data.createClient(); err != nil { return nil, err } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 9bf5cbd9bda..e8bee0837ae 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -311,8 +311,14 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } - // pollAndExportInterval is added as harcoded value "1s:5s" - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowPollAndFlowExportIntervals: \"\"|flowPollAndFlowExportIntervals: \"1s:5s\"|g' %s", antreaYML) + // flowPollInterval is added as harcoded value "1s" + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowPollInterval: \"5s\"|flowPollInterval: \"1s\"|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + // exportFrequency is added as harcoded value "5" + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowExportFrequency: 12|flowExportFrequency: 5|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) @@ -340,7 +346,12 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowPollAndFlowExportIntervals: \"1s:5s\"|#flowPollAndFlowExportIntervals: \"\"|g' %s", antreaYML) + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowPollInterval: \"1s\"|#flowPollInterval: \"5s\"|g' %s", antreaYML) + rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + } + cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowExportFrequency: 5|#flowExportFrequency: 12|g' %s", antreaYML) rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index a1a44c17af8..ab93f07b923 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -93,7 +93,7 @@ func prepareInterfaceConfigs(contID, podName, podNS, ifName string, ip *net.IP) return iface } -func testBuildFlowRecords(t *testing.T, flowRecords flowrecords.FlowRecords, conns []*flowexporter.Connection, connKeys []*flowexporter.ConnectionKey) { +func testBuildFlowRecords(t *testing.T, flowRecords *flowrecords.FlowRecords, conns []*flowexporter.Connection, connKeys []*flowexporter.ConnectionKey) { err := flowRecords.BuildFlowRecords() require.Nil(t, err, fmt.Sprintf("Failed to build flow records from connection store: %v", err)) // Check if records in flow records are built as expected or not @@ -115,18 +115,20 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { ctrl := mock.NewController(t) defer ctrl.Finish() - // Create ConnectionStore, FlowRecords and associated mocks - connDumperMock := connectionstest.NewMockConnTrackDumper(ctrl) - ifStoreMock := interfacestoretest.NewMockInterfaceStore(ctrl) - // Hardcoded poll and export intervals; they are not used - connStore := connections.NewConnectionStore(connDumperMock, ifStoreMock, time.Second) - flowRecords := flowrecords.NewFlowRecords(connStore) // Prepare connections and interface config for test testConns, testConnKeys := createConnsForTest() testIfConfigs := make([]*interfacestore.InterfaceConfig, 2) testIfConfigs[0] = prepareInterfaceConfigs("1", "pod1", "ns1", "interface1", &testConns[0].TupleOrig.SourceAddress) testIfConfigs[1] = prepareInterfaceConfigs("2", "pod2", "ns2", "interface2", &testConns[1].TupleOrig.DestinationAddress) - + // Create ConnectionStore, FlowRecords and associated mocks + connDumperMock := connectionstest.NewMockConnTrackDumper(ctrl) + ifStoreMock := interfacestoretest.NewMockInterfaceStore(ctrl) + // Hardcoded poll and export intervals; they are not used + connStore := &connections.ConnectionStore{ + Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection, 2), + ConnDumper: connDumperMock, + IfaceStore: ifStoreMock, + } // Expect calls for connStore.poll and other callees connDumperMock.EXPECT().DumpFlows(uint16(openflow.CtZone)).Return(testConns, nil) for i, testConn := range testConns { @@ -160,6 +162,7 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { } // Test for build flow records + flowRecords := flowrecords.NewFlowRecords(connStore) testBuildFlowRecords(t, flowRecords, testConns, testConnKeys) } From 719f8e0a67a212143a45dc3e73c9c6a338f34487 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Fri, 7 Aug 2020 13:36:02 -0700 Subject: [PATCH 13/15] Address comments 8/6 --- build/yamls/antrea-aks.yml | 21 ++- build/yamls/antrea-eks.yml | 27 +-- build/yamls/antrea-gke.yml | 27 +-- build/yamls/antrea-ipsec.yml | 27 +-- build/yamls/antrea.yml | 27 +-- build/yamls/base/conf/antrea-agent.conf | 11 +- cmd/antrea-agent/agent.go | 5 +- cmd/antrea-agent/main.go | 4 - cmd/antrea-agent/options.go | 21 ++- .../flowexporter/connections/connections.go | 50 ++--- .../connections/connections_test.go | 44 ++--- .../flowexporter/connections/conntrack.go | 157 ++-------------- .../connections/conntrack_linux.go | 4 +- .../flowexporter/connections/conntrack_ovs.go | 174 ++++++++++++++++++ .../connections/conntrack_test.go | 2 +- .../connections/conntrack_windows.go | 15 +- .../flowexporter/connections/interface.go | 2 +- pkg/agent/flowexporter/exporter/exporter.go | 6 +- .../flowexporter/exporter/exporter_test.go | 31 ++-- .../flowexporter/flowrecords/flow_records.go | 1 + pkg/agent/flowexporter/ipfix/ipfix_process.go | 1 + pkg/agent/flowexporter/ipfix/ipfix_record.go | 13 +- test/e2e/fixtures.go | 8 +- test/e2e/flowexporter_test.go | 43 ++++- test/e2e/framework.go | 68 ++----- test/e2e/providers/exec/docker.go | 2 +- test/integration/agent/flowexporter_test.go | 12 +- 27 files changed, 417 insertions(+), 386 deletions(-) create mode 100644 pkg/agent/flowexporter/connections/conntrack_ovs.go diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 472cb0eed41..a7640fca644 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -722,6 +722,21 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false + + # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables + # the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, + # we consider tcp as default. + #flowCollectorAddr: "" + + # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. + # Flow poll interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowPollInterval: "5s" + + # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + # the flow collector. + # Flow export frequency should be greater than or equal to 1. + #flowExportFrequency: 12 antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -770,7 +785,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b namespace: kube-system --- apiVersion: v1 @@ -876,7 +891,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b name: antrea-config - name: antrea-controller-tls secret: @@ -1091,7 +1106,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index ca4de457e3b..03d5eacd24f 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -723,17 +723,20 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false - # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX - # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. - # Defaults to "". + # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables + # the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, + # we consider tcp as default. #flowCollectorAddr: "" - # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections - # in conntrack module and exports IPFIX flow records that are built from connection store. - # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. - # Any value in range (PollInterval(s), 600s] for export interval is acceptable. - # Defaults to "5s:60s". Follow the time units of duration. - #pollAndExportInterval: "" + # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. + # Flow poll interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowPollInterval: "5s" + + # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + # the flow collector. + # Flow export frequency should be greater than or equal to 1. + #flowExportFrequency: 12 antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -782,7 +785,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b namespace: kube-system --- apiVersion: v1 @@ -888,7 +891,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b name: antrea-config - name: antrea-controller-tls secret: @@ -1105,7 +1108,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-24f6gdd4fb + name: antrea-config-62554ht95b name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index b42370e2fae..c73c819ef0d 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -723,17 +723,20 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false - # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX - # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. - # Defaults to "". + # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables + # the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, + # we consider tcp as default. #flowCollectorAddr: "" - # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections - # in conntrack module and exports IPFIX flow records that are built from connection store. - # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. - # Any value in range (PollInterval(s), 600s] for export interval is acceptable. - # Defaults to "5s:60s". Follow the time units of duration. - #pollAndExportInterval: "" + # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. + # Flow poll interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowPollInterval: "5s" + + # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + # the flow collector. + # Flow export frequency should be greater than or equal to 1. + #flowExportFrequency: 12 antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -782,7 +785,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-8gtt9dfdgg + name: antrea-config-9gt8khtcth namespace: kube-system --- apiVersion: v1 @@ -888,7 +891,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-8gtt9dfdgg + name: antrea-config-9gt8khtcth name: antrea-config - name: antrea-controller-tls secret: @@ -1103,7 +1106,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-8gtt9dfdgg + name: antrea-config-9gt8khtcth name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 1dc1d9d6485..77a75576920 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -723,17 +723,20 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false - # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX - # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. - # Defaults to "". + # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables + # the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, + # we consider tcp as default. #flowCollectorAddr: "" - # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections - # in conntrack module and exports IPFIX flow records that are built from connection store. - # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. - # Any value in range (PollInterval(s), 600s] for export interval is acceptable. - # Defaults to "5s:60s". Follow the time units of duration. - #pollAndExportInterval: "" + # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. + # Flow poll interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowPollInterval: "5s" + + # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + # the flow collector. + # Flow export frequency should be greater than or equal to 1. + #flowExportFrequency: 12 antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -782,7 +785,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-76fcd44cth + name: antrea-config-gf96hhfdg8 namespace: kube-system --- apiVersion: v1 @@ -897,7 +900,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-76fcd44cth + name: antrea-config-gf96hhfdg8 name: antrea-config - name: antrea-controller-tls secret: @@ -1147,7 +1150,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-76fcd44cth + name: antrea-config-gf96hhfdg8 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index a15409aa95e..e7e3ce728e3 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -723,17 +723,20 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false - # Provide flow collector address as string with format IP:port:L4(tcp or udp). This also enables flow exporter that sends IPFIX - # flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. - # Defaults to "". + # Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables + # the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, + # we consider tcp as default. #flowCollectorAddr: "" - # Provide flow exporter poll and export interval in format "0s:0s". This determines how often flow exporter polls connections - # in conntrack module and exports IPFIX flow records that are built from connection store. - # Any value in range [1s, ExportInterval(s)) for poll interval is acceptable. - # Any value in range (PollInterval(s), 600s] for export interval is acceptable. - # Defaults to "5s:60s". Follow the time units of duration. - #pollAndExportInterval: "" + # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. + # Flow poll interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowPollInterval: "5s" + + # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to + # the flow collector. + # Flow export frequency should be greater than or equal to 1. + #flowExportFrequency: 12 antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -782,7 +785,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-c6467gmm8c + name: antrea-config-mk822kf995 namespace: kube-system --- apiVersion: v1 @@ -888,7 +891,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-c6467gmm8c + name: antrea-config-mk822kf995 name: antrea-config - name: antrea-controller-tls secret: @@ -1103,7 +1106,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-c6467gmm8c + name: antrea-config-mk822kf995 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index e9b78ff7f9b..4022d17157e 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -61,13 +61,14 @@ featureGates: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: false -# Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables flow exporter that sends IPFIX -# flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, we consider tcp as default. +# Provide flow collector address as string with format :[:], where proto is tcp or udp. This also enables +# the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto is given, +# we consider tcp as default. #flowCollectorAddr: "" -# Provide flow poll interval in format "0s". This determines how often flow exporter dumps connections in conntrack module. -# Flow poll interval should be greater than or equal to 1s(one second). -# Follow the time units of time.Duration type. +# Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. +# Flow poll interval should be greater than or equal to 1s (one second). +# Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". #flowPollInterval: "5s" # Provide flow export frequency, which is the number of poll cycles elapsed before flow exporter exports flow records to diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index f81ff150ecc..9a71ca0bbfd 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -240,10 +240,7 @@ func run(o *Options) error { // Initialize flow exporter to start go routines to poll conntrack flows and export IPFIX flow records if features.DefaultFeatureGate.Enabled(features.FlowExporter) { connStore := connections.NewConnectionStore( - o.config.OVSDatapathType, - nodeConfig, - serviceCIDRNet, - agentQuerier.GetOVSCtlClient(), + connections.InitializeConnTrackDumper(nodeConfig, serviceCIDRNet, agentQuerier.GetOVSCtlClient(), o.config.OVSDatapathType), ifaceStore, o.pollInterval) // pollDone helps in synchronizing connStore.Run and flowExporter.Run go routines. diff --git a/cmd/antrea-agent/main.go b/cmd/antrea-agent/main.go index 13a2d8230ca..9e641d9225c 100644 --- a/cmd/antrea-agent/main.go +++ b/cmd/antrea-agent/main.go @@ -62,10 +62,6 @@ func newAgentCommand() *cobra.Command { if err := opts.validate(args); err != nil { klog.Fatalf("Failed to validate: %v", err) } - // Not passing args again as they are already validated and are not used in flow exporter config - if err := opts.validateFlowExporterConfig(); err != nil { - klog.Fatalf("Failed to validate flow exporter config: %v", err) - } if err := run(opts); err != nil { klog.Fatalf("Error running agent: %v", err) } diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 25cb034117c..89c71a408fb 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -32,11 +32,13 @@ import ( ) const ( - defaultOVSBridge = "br-int" - defaultHostGateway = "antrea-gw0" - defaultHostProcPathPrefix = "/host" - defaultServiceCIDR = "10.96.0.0/12" - defaultTunnelType = ovsconfig.GeneveTunnel + defaultOVSBridge = "br-int" + defaultHostGateway = "antrea-gw0" + defaultHostProcPathPrefix = "/host" + defaultServiceCIDR = "10.96.0.0/12" + defaultTunnelType = ovsconfig.GeneveTunnel + defaultFlowPollInterval = 5 * time.Second + defaultFlowExportFrequency = 12 ) type Options struct { @@ -101,6 +103,9 @@ func (o *Options) validate(args []string) error { if encapMode.SupportsNoEncap() && o.config.EnableIPSecTunnel { return fmt.Errorf("IPSec tunnel may only be enabled on %s mode", config.TrafficEncapModeEncap) } + if err := o.validateFlowExporterConfig(); err != nil { + return fmt.Errorf("Failed to validate flow exporter config: %v", err) + } return nil } @@ -152,11 +157,11 @@ func (o *Options) setDefaults() { if o.config.FeatureGates[string(features.FlowExporter)] { if o.config.FlowPollInterval == "" { - o.pollInterval = 5 * time.Second + o.pollInterval = defaultFlowPollInterval } if o.config.FlowExportFrequency == 0 { - // This frequency value makes flow export interval as 60s - o.config.FlowExportFrequency = 12 + // This frequency value makes flow export interval as 60s by default. + o.config.FlowExportFrequency = defaultFlowExportFrequency } } } diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 8ea6410f3c2..30c2a01811c 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -16,10 +16,6 @@ package connections import ( "fmt" - "github.com/vmware-tanzu/antrea/pkg/agent/config" - "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" - "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" - "net" "sync" "time" @@ -31,29 +27,23 @@ import ( ) type ConnectionStore struct { - Connections map[flowexporter.ConnectionKey]flowexporter.Connection - ConnDumper ConnTrackDumper - IfaceStore interfacestore.InterfaceStore + connections map[flowexporter.ConnectionKey]flowexporter.Connection + connDumper ConnTrackDumper + ifaceStore interfacestore.InterfaceStore pollInterval time.Duration mutex sync.Mutex } -func NewConnectionStore(ovsDatapathType string, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration) *ConnectionStore { - var connTrackDumper ConnTrackDumper - if ovsDatapathType == ovsconfig.OVSDatapathSystem { - connTrackDumper = NewConnTrackSystem(nodeConfig, serviceCIDR) - } else if ovsDatapathType == ovsconfig.OVSDatapathNetdev { - connTrackDumper = NewConnTrackOvsAppCtl(nodeConfig, serviceCIDR, ovsctlClient) - } +func NewConnectionStore(connTrackDumper ConnTrackDumper, ifaceStore interfacestore.InterfaceStore, pollInterval time.Duration) *ConnectionStore { return &ConnectionStore{ - Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - ConnDumper: connTrackDumper, - IfaceStore: ifaceStore, + connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), + connDumper: connTrackDumper, + ifaceStore: ifaceStore, pollInterval: pollInterval, } } -// Run enables the periodical polling of conntrack Connections, at the given flowPollInterval +// Run enables the periodical polling of conntrack connections, at the given flowPollInterval func (cs *ConnectionStore) Run(stopCh <-chan struct{}, pollDone chan struct{}) { klog.Infof("Starting conntrack polling") @@ -99,12 +89,12 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { existingConn.ReversePackets = conn.ReversePackets existingConn.IsActive = true // Reassign the flow to update the map - cs.Connections[connKey] = *existingConn + cs.connections[connKey] = *existingConn klog.V(4).Infof("Antrea flow updated: %v", existingConn) } else { var srcFound, dstFound bool - sIface, srcFound := cs.IfaceStore.GetInterfaceByIP(conn.TupleOrig.SourceAddress.String()) - dIface, dstFound := cs.IfaceStore.GetInterfaceByIP(conn.TupleReply.SourceAddress.String()) + sIface, srcFound := cs.ifaceStore.GetInterfaceByIP(conn.TupleOrig.SourceAddress.String()) + dIface, dstFound := cs.ifaceStore.GetInterfaceByIP(conn.TupleReply.SourceAddress.String()) if !srcFound && !dstFound { klog.Warningf("Cannot map any of the IP %s or %s to a local Pod", conn.TupleOrig.SourceAddress.String(), conn.TupleReply.SourceAddress.String()) } @@ -117,7 +107,7 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodName = dIface.ContainerInterfaceConfig.PodName conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } - // Do not export flow records of Connections, whose destination is local pod and source is remote pod. + // Do not export flow records of connections whose destination is local pod and source is remote pod. // We export flow records only form "source node", where the connection is originated from. This is to avoid // 2 copies of flow records at flow collector. This restriction will be removed when flow records store network policy rule ID. // TODO: Remove this when network policy rule ID are added to flow records. @@ -126,7 +116,7 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { } klog.V(4).Infof("New Antrea flow added: %v", conn) // Add new antrea connection to connection store - cs.Connections[connKey] = *conn + cs.connections[connKey] = *conn } } @@ -134,7 +124,7 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { func (cs *ConnectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (*flowexporter.Connection, bool) { cs.mutex.Lock() defer cs.mutex.Unlock() - conn, found := cs.Connections[flowTuple] + conn, found := cs.connections[flowTuple] return &conn, found } @@ -142,7 +132,7 @@ func (cs *ConnectionStore) GetConnByKey(flowTuple flowexporter.ConnectionKey) (* func (cs *ConnectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionMapCallBack) error { cs.mutex.Lock() defer cs.mutex.Unlock() - for k, v := range cs.Connections { + for k, v := range cs.connections { err := callback(k, v) if err != nil { klog.Errorf("Callback execution failed for flow with key: %v, conn: %v, k, v: %v", k, v, err) @@ -153,21 +143,21 @@ func (cs *ConnectionStore) ForAllConnectionsDo(callback flowexporter.ConnectionM } // Poll calls into conntrackDumper interface to dump conntrack flows -// TODO: As optimization, only poll invalid/closed Connections during every poll, and poll the established Connections right before the export. +// TODO: As optimization, only poll invalid/closed connections during every poll, and poll the established connections right before the export. func (cs *ConnectionStore) Poll() (int, error) { klog.V(2).Infof("Polling conntrack") - // Reset isActive flag for all Connections in connection map before dumping flows in conntrack module. + // Reset isActive flag for all connections in connection map before dumping flows in conntrack module. // This is to specify that the connection and the flow record can be deleted after the next export. resetConn := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { conn.IsActive = false - cs.Connections[key] = conn + cs.connections[key] = conn return nil } // We do not expect any error as resetConn is not returning any error cs.ForAllConnectionsDo(resetConn) - filteredConns, err := cs.ConnDumper.DumpFlows(openflow.CtZone) + filteredConns, err := cs.connDumper.DumpFlows(openflow.CtZone) if err != nil { return 0, err } @@ -191,7 +181,7 @@ func (cs *ConnectionStore) DeleteConnectionByKey(connKey flowexporter.Connection } cs.mutex.Lock() defer cs.mutex.Unlock() - delete(cs.Connections, connKey) + delete(cs.connections, connKey) return nil } diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index b497538c5f4..b4233b1b82e 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -28,6 +28,8 @@ import ( interfacestoretest "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore/testing" ) +const testPollInterval = 0 // Not used in these tests, hence 0. + func makeTuple(srcIP *net.IP, dstIP *net.IP, protoID uint8, srcPort uint16, dstPort uint16) (*flowexporter.Tuple, *flowexporter.Tuple) { tuple := &flowexporter.Tuple{ SourceAddress: *srcIP, @@ -104,18 +106,12 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { IP: net.IP{8, 7, 6, 5}, ContainerInterfaceConfig: podConfigFlow2, } - // Mock interface store with one of the couple of IPs correspond to Pods - iStore := interfacestoretest.NewMockInterfaceStore(ctrl) - mockCT := connectionstest.NewMockConnTrackDumper(ctrl) - // Create ConnectionStore - connStore := &ConnectionStore{ - Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - ConnDumper: mockCT, - IfaceStore: iStore, - } + mockIfaceStore := interfacestoretest.NewMockInterfaceStore(ctrl) + mockConnDumper := connectionstest.NewMockConnTrackDumper(ctrl) + connStore := NewConnectionStore(mockConnDumper, mockIfaceStore, testPollInterval) // Add flow1conn to the Connection map testFlow1Tuple := flowexporter.NewConnectionKey(&testFlow1) - connStore.Connections[testFlow1Tuple] = oldTestFlow1 + connStore.connections[testFlow1Tuple] = oldTestFlow1 addOrUpdateConnTests := []struct { flow flowexporter.Connection @@ -134,8 +130,8 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { expConn = test.flow expConn.DestinationPodNamespace = "ns2" expConn.DestinationPodName = "pod2" - iStore.EXPECT().GetInterfaceByIP(test.flow.TupleOrig.SourceAddress.String()).Return(nil, false) - iStore.EXPECT().GetInterfaceByIP(test.flow.TupleReply.SourceAddress.String()).Return(interfaceFlow2, true) + mockIfaceStore.EXPECT().GetInterfaceByIP(test.flow.TupleOrig.SourceAddress.String()).Return(nil, false) + mockIfaceStore.EXPECT().GetInterfaceByIP(test.flow.TupleReply.SourceAddress.String()).Return(interfaceFlow2, true) } connStore.addOrUpdateConn(&test.flow) actualConn, ok := connStore.GetConnByKey(flowTuple) @@ -182,20 +178,18 @@ func TestConnectionStore_ForAllConnectionsDo(t *testing.T) { testFlowKeys[i] = &connKey } // Create ConnectionStore - connStore := &ConnectionStore{ - Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - ConnDumper: nil, - IfaceStore: nil, - } + mockIfaceStore := interfacestoretest.NewMockInterfaceStore(ctrl) + mockConnDumper := connectionstest.NewMockConnTrackDumper(ctrl) + connStore := NewConnectionStore(mockConnDumper, mockIfaceStore, testPollInterval) // Add flows to the Connection store for i, flow := range testFlows { - connStore.Connections[*testFlowKeys[i]] = *flow + connStore.connections[*testFlowKeys[i]] = *flow } resetTwoFields := func(key flowexporter.ConnectionKey, conn flowexporter.Connection) error { conn.IsActive = false conn.OriginalPackets = 0 - connStore.Connections[key] = conn + connStore.connections[key] = conn return nil } connStore.ForAllConnectionsDo(resetTwoFields) @@ -246,16 +240,14 @@ func TestConnectionStore_DeleteConnectionByKey(t *testing.T) { testFlowKeys[i] = &connKey } // Create ConnectionStore - connStore := &ConnectionStore{ - Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection), - ConnDumper: nil, - IfaceStore: nil, - } + mockIfaceStore := interfacestoretest.NewMockInterfaceStore(ctrl) + mockConnDumper := connectionstest.NewMockConnTrackDumper(ctrl) + connStore := NewConnectionStore(mockConnDumper, mockIfaceStore, testPollInterval) // Add flows to the connection store. for i, flow := range testFlows { - connStore.Connections[*testFlowKeys[i]] = *flow + connStore.connections[*testFlowKeys[i]] = *flow } - // Delete the Connections in connection store. + // Delete the connections in connection store. for i := 0; i < len(testFlows); i++ { err := connStore.DeleteConnectionByKey(*testFlowKeys[i]) assert.Nil(t, err, "DeleteConnectionByKey should return nil") diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index 05bb8b16de1..ec28f20c0df 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -15,159 +15,24 @@ package connections import ( - "fmt" "net" - "strconv" - "strings" "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" - "github.com/vmware-tanzu/antrea/pkg/util/ip" ) -// connTrackOvsCtl implements ConnTrackDumper. This supports OVS userspace datapath scenarios. -var _ ConnTrackDumper = new(connTrackOvsCtl) - -type connTrackOvsCtl struct { - nodeConfig *config.NodeConfig - serviceCIDR *net.IPNet - ovsctlClient ovsctl.OVSCtlClient -} - -func NewConnTrackOvsAppCtl(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient) *connTrackOvsCtl { - return &connTrackOvsCtl{ - nodeConfig, - serviceCIDR, - ovsctlClient, - } -} - -// DumpFlows uses "ovs-appctl dpctl/dump-conntrack" to dump conntrack flows in the Antrea ZoneID. -func (ct *connTrackOvsCtl) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - conns, err := ct.ovsAppctlDumpConnections(zoneFilter) - if err != nil { - klog.Errorf("Error when dumping flows from conntrack: %v", err) - return nil, err - } - - filteredConns := filterAntreaConns(conns, ct.nodeConfig, ct.serviceCIDR, zoneFilter) - klog.V(2).Infof("Flow exporter considered flows: %d", len(filteredConns)) - - return filteredConns, nil -} - -func (ct *connTrackOvsCtl) ovsAppctlDumpConnections(zoneFilter uint16) ([]*flowexporter.Connection, error) { - // Dump conntrack using ovs-appctl dpctl/dump-conntrack - cmdOutput, execErr := ct.ovsctlClient.RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s") - if execErr != nil { - return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) - } - - // Parse the output to get the flows - antreaConns := make([]*flowexporter.Connection, 0) - outputFlow := strings.Split(string(cmdOutput), "\n") - var err error - for _, flow := range outputFlow { - conn := flowexporter.Connection{} - flowSlice := strings.Split(flow, ",") - isReply := false - inZone := false - for _, fs := range flowSlice { - // Indicator to populate reply or reverse fields - if strings.Contains(fs, "reply") { - isReply = true - } - if !strings.Contains(fs, "=") { - // Proto identifier - conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) - if err != nil { - klog.Errorf("Unknown protocol to convert to ID: %s", fs) - continue - } - conn.TupleReply.Protocol = conn.TupleOrig.Protocol - } else if strings.Contains(fs, "src") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "dst") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "sport") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.SourcePort = uint16(val) - } else { - conn.TupleReply.SourcePort = uint16(val) - } - } else if strings.Contains(fs, "dport") { - // dport field could be the last tuple field in ovs-dpctl output format. - fs = strings.TrimSuffix(fs, ")") - - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.DestinationPort = uint16(val) - } else { - conn.TupleReply.DestinationPort = uint16(val) - } - } else if strings.Contains(fs, "zone") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) - continue - } - if zoneFilter != uint16(val) { - break - } else { - inZone = true - conn.Zone = uint16(val) - } - } else if strings.Contains(fs, "timeout") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) - continue - } - conn.Timeout = uint32(val) - } else if strings.Contains(fs, "id") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) - continue - } - conn.ID = uint32(val) - } - } - if inZone { - conn.IsActive = true - conn.DoExport = true - antreaConns = append(antreaConns, &conn) - } +func InitializeConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient, ovsDatapathType string) ConnTrackDumper { + var connTrackDumper ConnTrackDumper + if ovsDatapathType == ovsconfig.OVSDatapathSystem { + connTrackDumper = NewConnTrackSystem(nodeConfig, serviceCIDR) + } else if ovsDatapathType == ovsconfig.OVSDatapathNetdev { + connTrackDumper = NewConnTrackOvsAppCtl(nodeConfig, serviceCIDR, ovsctlClient) } - klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) - return antreaConns, nil + return connTrackDumper } func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, zoneFilter uint16) []*flowexporter.Connection { @@ -181,14 +46,14 @@ func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.Node // Only get Pod-to-Pod flows. if srcIP.Equal(nodeConfig.GatewayConfig.IP) || dstIP.Equal(nodeConfig.GatewayConfig.IP) { - klog.V(4).Infof("Detected flow through gateway") + klog.V(4).Infof("Detected flow through gateway :%v", conn) continue } // Pod-to-Service flows w/ kube-proxy: There are two conntrack flows for every Pod-to-Service flow. // One is with ClusterIP as source or destination, where other IP is podIP. Second conntrack flow is - // with resolved Endpoint Pod IP corresponding to ClusterIP. Both conntrack flows have same stats, which makes them duplicate. - // Ideally, we have to correlate these two Connections and maintain one connection with both Endpoint Pod IP and ClusterIP. + // with resolved Endpoint Pod IP corresponding to ClusterIP. Both conntrack flows have same stats, which makes them duplicates. + // Ideally, we have to correlate these two connections and maintain one connection with both Endpoint Pod IP and ClusterIP. // To do the correlation, we need ClusterIP-to-EndpointIP mapping info, which is not available at Agent. // Therefore, we ignore the connection with ClusterIP and keep the connection with Endpoint Pod IP. // Conntrack flows will be different for Pod-to-Service flows w/ Antrea-proxy. This implementation will be simpler, when the diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index 51437762484..d6b7c2fac20 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -37,11 +37,11 @@ type connTrackSystem struct { } func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackSystem { - // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of Connections. + // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of connections. // Do not handle error and continue with creation of interfacer object as we can still dump flows with no stats. // If log says permission error, please ensure net.netfilter.nf_conntrack_acct to be set to 1. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_acct", 1) - // Ensure net.netfilter.nf_conntrack_timestamp value to be 1. This will enable flow exporter to export timestamps of Connections. + // Ensure net.netfilter.nf_conntrack_timestamp value to be 1. This will enable flow exporter to export timestamps of connections. // Do not handle error and continue with creation of interfacer object as we can still dump flows with no timestamps. // If log says permission error, please ensure net.netfilter.nf_conntrack_timestamp to be set to 1. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_timestamp", 1) diff --git a/pkg/agent/flowexporter/connections/conntrack_ovs.go b/pkg/agent/flowexporter/connections/conntrack_ovs.go new file mode 100644 index 00000000000..99722fd07da --- /dev/null +++ b/pkg/agent/flowexporter/connections/conntrack_ovs.go @@ -0,0 +1,174 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connections + +import ( + "fmt" + "net" + "strconv" + "strings" + + "k8s.io/klog" + + "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" + "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" + "github.com/vmware-tanzu/antrea/pkg/util/ip" +) + +// connTrackOvsCtl implements ConnTrackDumper. This supports OVS userspace datapath scenarios. +var _ ConnTrackDumper = new(connTrackOvsCtl) + +type connTrackOvsCtl struct { + nodeConfig *config.NodeConfig + serviceCIDR *net.IPNet + ovsctlClient ovsctl.OVSCtlClient +} + +func NewConnTrackOvsAppCtl(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient) *connTrackOvsCtl { + if ovsctlClient == nil { + return nil + } + return &connTrackOvsCtl{ + nodeConfig, + serviceCIDR, + ovsctlClient, + } +} + +// DumpFlows uses "ovs-appctl dpctl/dump-conntrack" to dump conntrack flows in the Antrea ZoneID. +func (ct *connTrackOvsCtl) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { + conns, err := ct.ovsAppctlDumpConnections(zoneFilter) + if err != nil { + klog.Errorf("Error when dumping flows from conntrack: %v", err) + return nil, err + } + + filteredConns := filterAntreaConns(conns, ct.nodeConfig, ct.serviceCIDR, zoneFilter) + klog.V(2).Infof("Flow exporter considered flows: %d", len(filteredConns)) + + return filteredConns, nil +} + +func (ct *connTrackOvsCtl) ovsAppctlDumpConnections(zoneFilter uint16) ([]*flowexporter.Connection, error) { + // Dump conntrack using ovs-appctl dpctl/dump-conntrack + cmdOutput, execErr := ct.ovsctlClient.RunAppctlCmd("dpctl/dump-conntrack", false, "-m", "-s") + if execErr != nil { + return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) + } + + // Parse the output to get the flows + antreaConns := make([]*flowexporter.Connection, 0) + outputFlow := strings.Split(string(cmdOutput), "\n") + var err error + for _, flow := range outputFlow { + conn := flowexporter.Connection{} + flowSlice := strings.Split(flow, ",") + isReply := false + inZone := false + for _, fs := range flowSlice { + // Indicator to populate reply or reverse fields + if strings.Contains(fs, "reply") { + isReply = true + } + if !strings.Contains(fs, "=") { + // Proto identifier + conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) + if err != nil { + klog.Errorf("Unknown protocol to convert to ID: %s", fs) + continue + } + conn.TupleReply.Protocol = conn.TupleOrig.Protocol + } else if strings.Contains(fs, "src") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "dst") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "sport") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.SourcePort = uint16(val) + } else { + conn.TupleReply.SourcePort = uint16(val) + } + } else if strings.Contains(fs, "dport") { + // dport field could be the last tuple field in ovs-dpctl output format. + fs = strings.TrimSuffix(fs, ")") + + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) + continue + } + if !isReply { + conn.TupleOrig.DestinationPort = uint16(val) + } else { + conn.TupleReply.DestinationPort = uint16(val) + } + } else if strings.Contains(fs, "zone") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) + continue + } + if zoneFilter != uint16(val) { + break + } else { + inZone = true + conn.Zone = uint16(val) + } + } else if strings.Contains(fs, "timeout") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) + continue + } + conn.Timeout = uint32(val) + } else if strings.Contains(fs, "id") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) + continue + } + conn.ID = uint32(val) + } + } + if inZone { + conn.IsActive = true + conn.DoExport = true + antreaConns = append(antreaConns, &conn) + } + } + klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + return antreaConns, nil +} diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index ed5106246aa..ae57e512b00 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -89,7 +89,7 @@ func TestConnTrackSystem_DumpFlows(t *testing.T) { if err != nil { t.Errorf("Dump flows function returned error: %v", err) } - assert.Equal(t, 1, len(conns), "number of filtered Connections should be equal") + assert.Equal(t, 1, len(conns), "number of filtered connections should be equal") } func TestConnTackOvsAppCtl_DumpFlows(t *testing.T) { diff --git a/pkg/agent/flowexporter/connections/conntrack_windows.go b/pkg/agent/flowexporter/connections/conntrack_windows.go index 74ff4e26db4..84e3e5812b0 100644 --- a/pkg/agent/flowexporter/connections/conntrack_windows.go +++ b/pkg/agent/flowexporter/connections/conntrack_windows.go @@ -20,18 +20,9 @@ import ( "net" "github.com/vmware-tanzu/antrea/pkg/agent/config" - "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ) -// TODO: Implement ConnTrackDumper for windows. -var _ ConnTrackDumper = new(connTrackSystem) - -type connTrackSystem struct{} - -func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackSystem { - return nil -} - -func (ct *connTrackSystem) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connection, error) { - return nil, nil +// TODO: Support FlowExporter feature for windows. We have to pass ovsctlClient when supported. +func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackOvsCtl { + return NewConnTrackOvsAppCtl(nodeConfig, serviceCIDR, nil) } diff --git a/pkg/agent/flowexporter/connections/interface.go b/pkg/agent/flowexporter/connections/interface.go index f13506e7ab0..1f9ea4ed911 100644 --- a/pkg/agent/flowexporter/connections/interface.go +++ b/pkg/agent/flowexporter/connections/interface.go @@ -18,7 +18,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ) -// ConnTrackDumper is an interface that is used to dump Connections from conntrack module. This supports dumping through +// ConnTrackDumper is an interface that is used to dump connections from conntrack module. This supports dumping through // netfilter socket (OVS kernel datapath) and ovs-appctl command (OVS userspace datapath). // In future, support will be extended to Windows. type ConnTrackDumper interface { diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 8295f16da45..3448ac01b52 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -16,8 +16,6 @@ package exporter import ( "fmt" - "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" - "github.com/vmware-tanzu/antrea/pkg/util/env" "hash/fnv" "net" "strings" @@ -29,6 +27,8 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" + "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" + "github.com/vmware-tanzu/antrea/pkg/util/env" ) var ( @@ -46,8 +46,6 @@ var ( "octetDeltaCount", } // Substring "reverse" is an indication to get reverse element of go-ipfix library. - // Specifically using GetReverseInfoElement, which is part of implementations of GetIANARegistryInfoElement and - // GetAntreaRegistryInfoElement. IANAReverseInfoElements = []string{ "reverse_PacketTotalCount", "reverse_OctetTotalCount", diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 4b18949181d..5d5f6516087 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -22,10 +22,17 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" ipfixtest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix/testing" - ipfixentities "github.com/vmware/go-ipfix/pkg/entities" +) + +const ( + testTemplateID = 256 + testFlowPollInterval = time.Second + testFlowExportFrequency = 12 + antreaEnterpriseRegistry = 29305 ) func TestFlowExporter_sendTemplateRecord(t *testing.T) { @@ -38,18 +45,18 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { nil, mockIPFIXExpProc, nil, - 12, - time.Second, - 256, + testFlowExportFrequency, + testFlowPollInterval, + testTemplateID, } // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) - // Need only element name and other are dummys + // Only the element name is needed, other arguments have dummy values. elemList := make([]*ipfixentities.InfoElement, 0) for _, ie := range IANAInfoElements { elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) } for _, ie := range IANAReverseInfoElements { - elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 29305, 0)) + elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, antreaEnterpriseRegistry, 0)) } for _, ie := range AntreaInfoElements { elemList = append(elemList, ipfixentities.NewInfoElement(ie, 0, 0, 0, 0)) @@ -76,8 +83,8 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { } mockTempRec.EXPECT().GetRecord().Return(templateRecord) mockTempRec.EXPECT().GetTemplateElements().Return(elemList) - // Passing 0 for sentBytes as it is not used anywhere in the test. In reality, this IPFIX message size - // for template record of above elements. + // Passing 0 for sentBytes as it is not used anywhere in the test. If this not a call to mock, the actual sentBytes + // above elements: IANAInfoElements, IANAReverseInfoElements and AntreaInfoElements. mockIPFIXExpProc.EXPECT().AddRecordAndSendMsg(ipfixentities.Template, templateRecord).Return(0, nil) _, err := flowExp.sendTemplateRecord(mockTempRec) @@ -135,7 +142,7 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { elemList[i] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) } for i, ie := range IANAReverseInfoElements { - elemList[i+len(IANAInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 29305, 0) + elemList[i+len(IANAInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, antreaEnterpriseRegistry, 0) } for i, ie := range AntreaInfoElements { elemList[i+len(IANAInfoElements)+len(IANAReverseInfoElements)] = ipfixentities.NewInfoElement(ie, 0, 0, 0, 0) @@ -146,9 +153,9 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { nil, mockIPFIXExpProc, elemList, - 12, - time.Second, - 256, + testFlowExportFrequency, + testFlowPollInterval, + testTemplateID, } // Expect calls required var dataRecord ipfixentities.Record diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index f403ab6dfdf..fc2f4970c77 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -16,6 +16,7 @@ package flowrecords import ( "fmt" + "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" diff --git a/pkg/agent/flowexporter/ipfix/ipfix_process.go b/pkg/agent/flowexporter/ipfix/ipfix_process.go index 56036e99077..3834d952be6 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_process.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_process.go @@ -25,6 +25,7 @@ import ( var _ IPFIXExportingProcess = new(ipfixExportingProcess) +// IPFIXExportingProcess interface is added to facilitate unit testing without involving the code from go-ipfix library. type IPFIXExportingProcess interface { LoadRegistries() GetIANARegistryInfoElement(name string, isReverse bool) (*ipfixentities.InfoElement, error) diff --git a/pkg/agent/flowexporter/ipfix/ipfix_record.go b/pkg/agent/flowexporter/ipfix/ipfix_record.go index d9ee42b28a1..73a4e079c59 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_record.go +++ b/pkg/agent/flowexporter/ipfix/ipfix_record.go @@ -23,6 +23,7 @@ import ( var _ IPFIXRecord = new(ipfixDataRecord) var _ IPFIXRecord = new(ipfixTemplateRecord) +// IPFIXRecord interface is added to facilitate unit testing without involving the code from go-ipfix library. type IPFIXRecord interface { GetRecord() ipfixentities.Record PrepareRecord() (uint16, error) @@ -54,13 +55,11 @@ func (dr *ipfixDataRecord) GetRecord() ipfixentities.Record { } func (dr *ipfixDataRecord) PrepareRecord() (uint16, error) { - addedBytes, err := dr.dataRecord.PrepareRecord() - return addedBytes, err + return dr.dataRecord.PrepareRecord() } func (dr *ipfixDataRecord) AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) { - addedBytes, err := dr.dataRecord.AddInfoElement(element, val) - return addedBytes, err + return dr.dataRecord.AddInfoElement(element, val) } func (dr *ipfixDataRecord) GetBuffer() *bytes.Buffer { @@ -80,13 +79,11 @@ func (tr *ipfixTemplateRecord) GetRecord() ipfixentities.Record { } func (tr *ipfixTemplateRecord) PrepareRecord() (uint16, error) { - addedBytes, err := tr.templateRecord.PrepareRecord() - return addedBytes, err + return tr.templateRecord.PrepareRecord() } func (tr *ipfixTemplateRecord) AddInfoElement(element *ipfixentities.InfoElement, val interface{}) (uint16, error) { - addedBytes, err := tr.templateRecord.AddInfoElement(element, val) - return addedBytes, err + return tr.templateRecord.AddInfoElement(element, val) } func (tr *ipfixTemplateRecord) GetBuffer() *bytes.Buffer { diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 20c2ea60129..8f271b6e69a 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -94,16 +94,12 @@ func setupTestWithIPFIXCollector(tb testing.TB) (*TestData, error) { if err := data.createPodOnNode("ipfix-collector", masterNodeName(), ipfixCollectorImage, nil, nil, nil, nil, true); err != nil { tb.Fatalf("Error when creating the ipfix collector Pod: %v", err) } - ipfixCollIP, err := data.podWaitForIP(defaultTimeout, "ipfix-collector", testNamespace) + ipfixCollectorIP, err := data.podWaitForIP(defaultTimeout, "ipfix-collector", testNamespace) if err != nil { tb.Fatalf("Error when waiting to get ipfix collector Pod IP: %v", err) } tb.Logf("Applying Antrea YAML with ipfix collector address") - if err := data.deployAntreaFlowExporter(ipfixCollIP + ":" + ipfixCollectorPort + ":tcp"); err != nil { - return data, err - } - tb.Logf("Waiting for all Antrea DaemonSet Pods") - if err := data.waitForAntreaDaemonSetPods(defaultTimeout); err != nil { + if err := data.deployAntreaFlowExporter(ipfixCollectorIP + ":" + ipfixCollectorPort + ":tcp"); err != nil { return data, err } tb.Logf("Checking CoreDNS deployment") diff --git a/test/e2e/flowexporter_test.go b/test/e2e/flowexporter_test.go index 1ebee0cb0b9..b195afa8276 100644 --- a/test/e2e/flowexporter_test.go +++ b/test/e2e/flowexporter_test.go @@ -17,7 +17,6 @@ package e2e import ( "encoding/hex" "fmt" - "math" "regexp" "strconv" "strings" @@ -66,7 +65,41 @@ func TestFlowExporter(t *testing.T) { t.Fatalf("error when getting logs %v, rc: %v", err, rc) } - // Parse through IPFIX collector output + /* Parse through IPFIX collector output. Sample output (with truncated fields) is given below: + IPFIX-HDR: + version=10, length=158 + unixtime=1596608557 (2020-08-04 23:22:37 PDT) + seqno=51965, odid=4093457084 + DATA RECORD: + template id: 256 + nfields: 21 + sourceIPv4Address: 100.10.0.117 + destinationIPv4Address: 100.10.1.128 + sourceTransportPort: 44586 + destinationTransportPort: 8080 + protocolIdentifier: 6 + packetTotalCount: 7 + octetTotalCount: 420 + packetDeltaCount: 0 + octetDeltaCount: 0 + 55829_101: 0x7765622d636c69 + IPFIX-HDR: + version=10, length=119 + unixtime=1596608558 (2020-08-04 23:22:38 PDT) + seqno=159, odid=1269807227 + DATA RECORD: + template id: 256 + nfields: 21 + sourceIPv4Address: 100.10.0.114 + destinationIPv4Address: 100.10.1.127 + sourceTransportPort: 42872 + destinationTransportPort: 8080 + protocolIdentifier: 6 + packetTotalCount: 7 + octetTotalCount: 420 + packetDeltaCount: 0 + octetDeltaCount: 0 + */ re := regexp.MustCompile("(?m)^.*" + "#" + ".*$[\r\n]+") collectorOutput = re.ReplaceAllString(collectorOutput, "") collectorOutput = strings.TrimSpace(collectorOutput) @@ -109,18 +142,16 @@ func TestFlowExporter(t *testing.T) { t.Fatalf("Error in converting octetDeltaCount to int type") } // compute the bandwidth using 5s as interval - recBandwidth := (deltaBytes * 8.0) / (5.0 * math.Pow10(9)) + recBandwidth := (deltaBytes * 8.0) / float64((int64(5.0))*time.Second.Nanoseconds()) // bandwidth from iperf output bwSlice := strings.Split(bandwidth, " ") iperfBandwidth, err := strconv.ParseFloat(bwSlice[0], 64) if err != nil { t.Fatalf("Error in converting iperf bandwidth to float64 type") } - // Check if at least the first digit is equal, i.e., 42 Gb/s and 48 Gb/s are considered equal - // we cannot guarantee both will be exactly same. Logging both values to give visibility. t.Logf("Iperf bandwidth: %v", iperfBandwidth) t.Logf("IPFIX record bandwidth: %v", recBandwidth) - assert.Equal(t, int(recBandwidth/10), int(float64(iperfBandwidth)/10), "Iperf bandwidth and IPFIX record bandwidth should be similar") + assert.InEpsilonf(t, recBandwidth, iperfBandwidth, 5, "Difference between Iperf bandwidth and IPFIX record bandwidth should be less than 5Gb/s") break } } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e8bee0837ae..93186622699 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -304,62 +304,28 @@ func (data *TestData) deployAntreaIPSec() error { // deployAntreaFlowExporter deploys Antrea with flow exporter config params enabled. func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { - // May be better to change this from configmap rather than directly changing antrea manifest? - // This is to add ipfixCollector address and pollAndExportInterval config params to antrea agent configmap - cmd := fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowCollectorAddr: \"\"|flowCollectorAddr: \"%s\"|g' %s", ipfixCollector, antreaYML) - rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) - } - // flowPollInterval is added as harcoded value "1s" - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowPollInterval: \"5s\"|flowPollInterval: \"1s\"|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) - } - // exportFrequency is added as harcoded value "5" - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|#flowExportFrequency: 12|flowExportFrequency: 5|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + // Enable flow exporter feature and add related config params to antrea agent configmap. + configMap, err := data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + return fmt.Errorf("failed to get ConfigMap: %v", err) } - // Turn on FlowExporter feature in featureGates - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|# FlowExporter: false| FlowExporter: true|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + + antreaAgentConf, _ := configMap.Data["antrea-agent.conf"] + antreaAgentConf = strings.Replace(antreaAgentConf, "# FlowExporter: false", " FlowExporter: true", 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "#flowCollectorAddr: \"\"", fmt.Sprintf("flowCollectorAddr: \"%s\"", ipfixCollector), 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "#flowPollInterval: \"5s\"", "flowPollInterval: \"1s\"", 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "#flowExportFrequency: 12", "flowExportFrequency: 5", 1) + configMap.Data["antrea-agent.conf"] = antreaAgentConf + + if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err) } // Delete and re-deploy antrea for config map settings to take effect. - // Question: Can end-to-end tests run in parallel? Is there an issue deleting Antrea daemon set? // TODO: Remove this when configmap can be changed runtime - if err := data.deleteAntrea(defaultTimeout); err != nil { - return err - } - if err := data.deployAntreaCommon(antreaYML, ""); err != nil { - return err - } - - // Change the yaml file back for other tests - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowCollectorAddr: \"%s\"|#flowCollectorAddr: \"\"|g' %s", ipfixCollector, antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) - } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowPollInterval: \"1s\"|#flowPollInterval: \"5s\"|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) - } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's|flowExportFrequency: 5|#flowExportFrequency: 12|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s back on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) - } - cmd = fmt.Sprintf("/bin/sh -c sed -i.bak -E 's| FlowExporter: true|# FlowExporter: false|g' %s", antreaYML) - rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), cmd) - if err != nil || rc != 0 { - return fmt.Errorf("error when changing yamlFile %s on the master Node %s: %v rc: %v", antreaYML, masterNodeName(), err, rc) + err = data.restartAntreaAgentPods(defaultTimeout) + if err != nil { + return fmt.Errorf("error when restarting antrea-agent Pod: %v", err) } return nil diff --git a/test/e2e/providers/exec/docker.go b/test/e2e/providers/exec/docker.go index a5d0bd63f14..97b00790b12 100644 --- a/test/e2e/providers/exec/docker.go +++ b/test/e2e/providers/exec/docker.go @@ -34,7 +34,7 @@ func RunDockerExecCommand(container string, cmd string, workdir string) ( args = append(args, "exec", "-w", workdir, "-t", container) if strings.Contains(cmd, "/bin/sh") { // Just split in to "/bin/sh" "-c" and "actual_cmd" - // This is useful for passing piped commands in to exec + // This is useful for passing piped commands in to os/exec interface. args = append(args, strings.SplitN(cmd, " ", 3)...) } else { args = append(args, strings.Fields(cmd)...) diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index ab93f07b923..d7fbe4a258e 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -2,7 +2,6 @@ package agent import ( "fmt" - "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" "net" "testing" "time" @@ -15,10 +14,13 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections" connectionstest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/connections/testing" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" + "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" interfacestoretest "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore/testing" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" ) +const testPollInterval = 0 // Not used in the test, hence 0. + func makeTuple(srcIP *net.IP, dstIP *net.IP, protoID uint8, srcPort uint16, dstPort uint16) (*flowexporter.Tuple, *flowexporter.Tuple) { tuple := &flowexporter.Tuple{ SourceAddress: *srcIP, @@ -123,12 +125,7 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { // Create ConnectionStore, FlowRecords and associated mocks connDumperMock := connectionstest.NewMockConnTrackDumper(ctrl) ifStoreMock := interfacestoretest.NewMockInterfaceStore(ctrl) - // Hardcoded poll and export intervals; they are not used - connStore := &connections.ConnectionStore{ - Connections: make(map[flowexporter.ConnectionKey]flowexporter.Connection, 2), - ConnDumper: connDumperMock, - IfaceStore: ifStoreMock, - } + connStore := connections.NewConnectionStore(connDumperMock, ifStoreMock, testPollInterval) // Expect calls for connStore.poll and other callees connDumperMock.EXPECT().DumpFlows(uint16(openflow.CtZone)).Return(testConns, nil) for i, testConn := range testConns { @@ -164,5 +161,4 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { // Test for build flow records flowRecords := flowrecords.NewFlowRecords(connStore) testBuildFlowRecords(t, flowRecords, testConns, testConnKeys) - } From 841469c252be5c22b9ee058403d8ea1d9b9b30c7 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Fri, 7 Aug 2020 19:04:40 -0700 Subject: [PATCH 14/15] Addressed comments Major change is handling of error in exporter go routine. --- cmd/antrea-agent/agent.go | 13 +- go.mod | 2 +- go.sum | 4 +- .../flowexporter/connections/connections.go | 7 +- .../flowexporter/connections/conntrack.go | 1 + .../connections/conntrack_linux.go | 10 +- .../flowexporter/connections/conntrack_ovs.go | 219 ++++++++++-------- pkg/agent/flowexporter/exporter/exporter.go | 99 ++++---- .../flowexporter/exporter/exporter_test.go | 5 +- .../flowexporter/flowrecords/flow_records.go | 8 +- pkg/util/ip/ip.go | 21 -- plugins/octant/go.sum | 2 +- test/e2e/framework.go | 14 ++ 13 files changed, 205 insertions(+), 200 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 9a71ca0bbfd..b1d7fb88468 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -19,6 +19,7 @@ import ( "net" "time" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" "k8s.io/klog/v2" @@ -243,19 +244,13 @@ func run(o *Options) error { connections.InitializeConnTrackDumper(nodeConfig, serviceCIDRNet, agentQuerier.GetOVSCtlClient(), o.config.OVSDatapathType), ifaceStore, o.pollInterval) - // pollDone helps in synchronizing connStore.Run and flowExporter.Run go routines. pollDone := make(chan struct{}) go connStore.Run(stopCh, pollDone) - flowExporter, err := exporter.InitFlowExporter( - o.flowCollector, + flowExporter := exporter.NewFlowExporter( flowrecords.NewFlowRecords(connStore), - o.config.FlowExportFrequency, - o.pollInterval) - if err != nil { - return fmt.Errorf("error when initializing flow exporter: %v", err) - } - go flowExporter.Run(stopCh, pollDone) + o.config.FlowExportFrequency) + go wait.Until(func() { flowExporter.CheckAndDoExport(o.flowCollector, pollDone) }, o.pollInterval, stopCh) } <-stopCh diff --git a/go.mod b/go.mod index 5367ad8cc61..5ac3f2b9c39 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/stretchr/testify v1.5.1 github.com/ti-mo/conntrack v0.3.0 github.com/vishvananda/netlink v1.1.0 - github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f + github.com/vmware/go-ipfix v0.0.0-20200808032647-11daf237d1dc golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index afa512e96d8..87e41ab24f3 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f h1:XyyczLRk8+6YqYXE8v20XjbVtK415KR114IrjX9THpQ= -github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= +github.com/vmware/go-ipfix v0.0.0-20200808032647-11daf237d1dc h1:lytkY3WfWgOyyaOlgj/3Y5Fkwc9ENff2qg6Ul4FYriE= +github.com/vmware/go-ipfix v0.0.0-20200808032647-11daf237d1dc/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e h1:NM4NTe6Z+mF5IYlYAiEdRlY8XcMY4P6VlYqgsBhpojQ= github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 30c2a01811c..f09c325cd22 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -62,10 +62,9 @@ func (cs *ConnectionStore) Run(stopCh <-chan struct{}, pollDone chan struct{}) { klog.Errorf("Error during conntrack poll cycle: %v", err) } // We need synchronization between ConnectionStore.Run and FlowExporter.Run go routines. - // ConnectionStore.Run (connection poll) should be done to start FlowExporter.Run (connection export); pollDone signals helps enabling this. + // ConnectionStore.Run (connection poll) should be done to start FlowExporter.Run (connection export); pollDone signal helps enabling this. // FlowExporter.Run should be done to start ConnectionStore.Run; mutex on connection map object makes sure of this synchronization guarantee. pollDone <- struct{}{} - } } } @@ -98,7 +97,7 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { if !srcFound && !dstFound { klog.Warningf("Cannot map any of the IP %s or %s to a local Pod", conn.TupleOrig.SourceAddress.String(), conn.TupleReply.SourceAddress.String()) } - // sourceIP/destinationIP are mapped only to local pods and not remote pods. + // sourceIP/destinationIP are mapped only to local Pods and not remote Pods. if srcFound && sIface.Type == interfacestore.ContainerInterface { conn.SourcePodName = sIface.ContainerInterfaceConfig.PodName conn.SourcePodNamespace = sIface.ContainerInterfaceConfig.PodNamespace @@ -107,7 +106,7 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodName = dIface.ContainerInterfaceConfig.PodName conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } - // Do not export flow records of connections whose destination is local pod and source is remote pod. + // Do not export flow records of connections whose destination is local Pod and source is remote Pod. // We export flow records only form "source node", where the connection is originated from. This is to avoid // 2 copies of flow records at flow collector. This restriction will be removed when flow records store network policy rule ID. // TODO: Remove this when network policy rule ID are added to flow records. diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index ec28f20c0df..593836c33ed 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -25,6 +25,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" ) +// InitializeConnTrackDumper initialize the ConnTrackDumper interface for different OS and datapath types. func InitializeConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient, ovsDatapathType string) ConnTrackDumper { var connTrackDumper ConnTrackDumper if ovsDatapathType == ovsconfig.OVSDatapathSystem { diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index d6b7c2fac20..f3e238bf5b4 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -38,12 +38,10 @@ type connTrackSystem struct { func NewConnTrackSystem(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet) *connTrackSystem { // Ensure net.netfilter.nf_conntrack_acct value to be 1. This will enable flow exporter to export stats of connections. - // Do not handle error and continue with creation of interfacer object as we can still dump flows with no stats. - // If log says permission error, please ensure net.netfilter.nf_conntrack_acct to be set to 1. + // Do not fail, but continue after logging error as we can still dump flows with no stats. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_acct", 1) // Ensure net.netfilter.nf_conntrack_timestamp value to be 1. This will enable flow exporter to export timestamps of connections. - // Do not handle error and continue with creation of interfacer object as we can still dump flows with no timestamps. - // If log says permission error, please ensure net.netfilter.nf_conntrack_timestamp to be set to 1. + // Do not fail, but continue after logging error as we can still dump flows with no timestamps. sysctl.EnsureSysctlNetValue("netfilter/nf_conntrack_timestamp", 1) return &connTrackSystem{ @@ -103,7 +101,7 @@ func (nfct *netFilterConnTrack) DumpFilter(filter conntrack.Filter) ([]*flowexpo } antreaConns := make([]*flowexporter.Connection, len(conns)) for i, conn := range conns { - antreaConns[i] = createAntreaConn(&conn) + antreaConns[i] = netlinkFlowToAntreaConnection(&conn) } klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) @@ -112,7 +110,7 @@ func (nfct *netFilterConnTrack) DumpFilter(filter conntrack.Filter) ([]*flowexpo return antreaConns, nil } -func createAntreaConn(conn *conntrack.Flow) *flowexporter.Connection { +func netlinkFlowToAntreaConnection(conn *conntrack.Flow) *flowexporter.Connection { tupleOrig := flowexporter.Tuple{ SourceAddress: conn.TupleOrig.IP.SourceAddress, DestinationAddress: conn.TupleOrig.IP.DestinationAddress, diff --git a/pkg/agent/flowexporter/connections/conntrack_ovs.go b/pkg/agent/flowexporter/connections/conntrack_ovs.go index 99722fd07da..38cd6be2c0d 100644 --- a/pkg/agent/flowexporter/connections/conntrack_ovs.go +++ b/pkg/agent/flowexporter/connections/conntrack_ovs.go @@ -25,9 +25,17 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" - "github.com/vmware-tanzu/antrea/pkg/util/ip" ) +// Following map is for converting protocol name (string) to protocol identifier +var protocols = map[string]uint8{ + "icmp": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "ipv6-icmp": 58, +} + // connTrackOvsCtl implements ConnTrackDumper. This supports OVS userspace datapath scenarios. var _ ConnTrackDumper = new(connTrackOvsCtl) @@ -69,106 +77,127 @@ func (ct *connTrackOvsCtl) ovsAppctlDumpConnections(zoneFilter uint16) ([]*flowe return nil, fmt.Errorf("error when executing dump-conntrack command: %v", execErr) } - // Parse the output to get the flows + // Parse the output to get the flow strings and convert them to Antrea connections. antreaConns := make([]*flowexporter.Connection, 0) outputFlow := strings.Split(string(cmdOutput), "\n") - var err error for _, flow := range outputFlow { - conn := flowexporter.Connection{} - flowSlice := strings.Split(flow, ",") - isReply := false - inZone := false - for _, fs := range flowSlice { - // Indicator to populate reply or reverse fields - if strings.Contains(fs, "reply") { - isReply = true - } - if !strings.Contains(fs, "=") { - // Proto identifier - conn.TupleOrig.Protocol, err = ip.LookupProtocolMap(fs) - if err != nil { - klog.Errorf("Unknown protocol to convert to ID: %s", fs) - continue - } - conn.TupleReply.Protocol = conn.TupleOrig.Protocol - } else if strings.Contains(fs, "src") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "dst") { - fields := strings.Split(fs, "=") - if !isReply { - conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } else { - conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) - } - } else if strings.Contains(fs, "sport") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of sport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.SourcePort = uint16(val) - } else { - conn.TupleReply.SourcePort = uint16(val) - } - } else if strings.Contains(fs, "dport") { - // dport field could be the last tuple field in ovs-dpctl output format. - fs = strings.TrimSuffix(fs, ")") - - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of dport: %s to int failed", fields[len(fields)-1]) - continue - } - if !isReply { - conn.TupleOrig.DestinationPort = uint16(val) - } else { - conn.TupleReply.DestinationPort = uint16(val) - } - } else if strings.Contains(fs, "zone") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of zone: %s to int failed", fields[len(fields)-1]) - continue - } - if zoneFilter != uint16(val) { - break - } else { - inZone = true - conn.Zone = uint16(val) - } - } else if strings.Contains(fs, "timeout") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of timeout: %s to int failed", fields[len(fields)-1]) - continue - } - conn.Timeout = uint32(val) - } else if strings.Contains(fs, "id") { - fields := strings.Split(fs, "=") - val, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - klog.Errorf("Conversion of id: %s to int failed", fields[len(fields)-1]) - continue - } - conn.ID = uint32(val) - } + conn, err := flowStringToAntreaConnection(flow, zoneFilter) + if err != nil { + klog.Warningf("Ignoring the flow from conntrack dump due to the error: %v", err) + continue } - if inZone { - conn.IsActive = true - conn.DoExport = true - antreaConns = append(antreaConns, &conn) + if conn != nil { + antreaConns = append(antreaConns, conn) } } klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) return antreaConns, nil } + +// flowStringToAntreaConnection parses the flow string and converts to Antrea connection. +// Example of flow string: +// tcp,orig=(src=10.10.1.2,dst=10.96.0.1,sport=42540,dport=443),reply=(src=10.96.0.1,dst=10.10.1.2,sport=443,dport=42540),zone=65520,protoinfo=(state=TIME_WAIT) +func flowStringToAntreaConnection(flow string, zoneFilter uint16) (*flowexporter.Connection, error) { + conn := flowexporter.Connection{} + flowSlice := strings.Split(flow, ",") + isReply := false + inZone := false + var err error + for _, fs := range flowSlice { + // Indicator to populate reply or reverse fields + if strings.Contains(fs, "reply") { + isReply = true + } + if !strings.Contains(fs, "=") { + // Proto identifier + conn.TupleOrig.Protocol, err = lookupProtocolMap(fs) + if err != nil { + return nil, err + } + conn.TupleReply.Protocol = conn.TupleOrig.Protocol + } else if strings.Contains(fs, "src") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.SourceAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "dst") { + fields := strings.Split(fs, "=") + if !isReply { + conn.TupleOrig.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } else { + conn.TupleReply.DestinationAddress = net.ParseIP(fields[len(fields)-1]) + } + } else if strings.Contains(fs, "sport") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + return nil, fmt.Errorf("conversion of sport %s to int failed", fields[len(fields)-1]) + } + if !isReply { + conn.TupleOrig.SourcePort = uint16(val) + } else { + conn.TupleReply.SourcePort = uint16(val) + } + } else if strings.Contains(fs, "dport") { + // dport field could be the last tuple field in ovs-dpctl output format. + fs = strings.TrimSuffix(fs, ")") + + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + return nil, fmt.Errorf("conversion of dport %s to int failed", fields[len(fields)-1]) + } + if !isReply { + conn.TupleOrig.DestinationPort = uint16(val) + } else { + conn.TupleReply.DestinationPort = uint16(val) + } + } else if strings.Contains(fs, "zone") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + return nil, fmt.Errorf("conversion of zone %s to int failed", fields[len(fields)-1]) + } + if zoneFilter != uint16(val) { + break + } else { + inZone = true + conn.Zone = uint16(val) + } + } else if strings.Contains(fs, "timeout") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + return nil, fmt.Errorf("conversion of timeout %s to int failed", fields[len(fields)-1]) + } + conn.Timeout = uint32(val) + } else if strings.Contains(fs, "id") { + fields := strings.Split(fs, "=") + val, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + return nil, fmt.Errorf("conversion of id %s to int failed", fields[len(fields)-1]) + } + conn.ID = uint32(val) + } + } + if !inZone { + return nil, nil + } + conn.IsActive = true + conn.DoExport = true + + return &conn, nil +} + +// lookupProtocolMap returns protocol identifier given protocol name +func lookupProtocolMap(name string) (uint8, error) { + name = strings.TrimSpace(name) + lowerCaseStr := strings.ToLower(name) + proto, found := protocols[lowerCaseStr] + if !found { + return 0, fmt.Errorf("unknown IP protocol specified: %s", name) + } + return proto, nil +} diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 3448ac01b52..612189bdf62 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -19,7 +19,6 @@ import ( "hash/fnv" "net" "strings" - "time" "unicode" ipfixentities "github.com/vmware/go-ipfix/pkg/entities" @@ -67,7 +66,7 @@ type flowExporter struct { process ipfix.IPFIXExportingProcess elementsList []*ipfixentities.InfoElement exportFrequency uint - pollInterval time.Duration + pollCycle uint templateID uint16 } @@ -81,22 +80,52 @@ func genObservationID() (uint32, error) { return h.Sum32(), nil } -func NewFlowExporter(records *flowrecords.FlowRecords, expProcess ipfix.IPFIXExportingProcess, elemList []*ipfixentities.InfoElement, exportFrequency uint, pollInterval time.Duration, tempID uint16) *flowExporter { +func NewFlowExporter(records *flowrecords.FlowRecords, exportFrequency uint) *flowExporter { return &flowExporter{ records, - expProcess, - elemList, + nil, + nil, exportFrequency, - pollInterval, - tempID, + 0, + 0, } } -func InitFlowExporter(collector net.Addr, records *flowrecords.FlowRecords, exportFrequency uint, pollInterval time.Duration) (*flowExporter, error) { - // Create IPFIX exporting expProcess and initialize registries and other related entities +// CheckAndDoExport enables us to export flow records periodically at a given flow export frequency. +func (exp *flowExporter) CheckAndDoExport(collector net.Addr, pollDone chan struct{}) { + // Number of pollDone signals received or poll cycles should be equal to export frequency before starting the export cycle. + // This is necessary because IPFIX collector computes throughput based on flow records received interval. + <-pollDone + exp.pollCycle++ + if exp.pollCycle%exp.exportFrequency == 0 { + if exp.process == nil { + err := exp.initFlowExporter(collector) + if err != nil { + klog.Errorf("Error when initializing flow exporter: %v", err) + return + } + } + exp.flowRecords.BuildFlowRecords() + err := exp.sendFlowRecords() + if err != nil { + klog.Errorf("Error when sending flow records: %v", err) + // If there is an error when sending flow records because of intermittent connectivity, we reset the connection + // to IPFIX collector and retry in the next export cycle to reinitialize the connection and send flow records. + exp.process.CloseConnToCollector() + exp.process = nil + } + exp.pollCycle = 0 + klog.V(2).Infof("Successfully exported IPFIX flow records") + } + + return +} + +func (exp *flowExporter) initFlowExporter(collector net.Addr) error { + // Create IPFIX exporting expProcess, initialize registries and other related entities obsID, err := genObservationID() if err != nil { - return nil, fmt.Errorf("cannot generate obsID for IPFIX ipfixexport: %v", err) + return fmt.Errorf("cannot generate obsID for IPFIX ipfixexport: %v", err) } var expProcess ipfix.IPFIXExportingProcess @@ -108,55 +137,21 @@ func InitFlowExporter(collector net.Addr, records *flowrecords.FlowRecords, expo expProcess, err = ipfix.NewIPFIXExportingProcess(collector, obsID, 1800) } if err != nil { - return nil, fmt.Errorf("error while initializing IPFIX exporting expProcess: %v", err) + return err } - expProcess.LoadRegistries() - - flowExp := NewFlowExporter(records, expProcess, nil, exportFrequency, pollInterval, expProcess.NewTemplateID()) + exp.process = expProcess + exp.templateID = expProcess.NewTemplateID() - templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), flowExp.templateID) + expProcess.LoadRegistries() + templateRec := ipfix.NewIPFIXTemplateRecord(uint16(len(IANAInfoElements)+len(IANAReverseInfoElements)+len(AntreaInfoElements)), exp.templateID) - sentBytes, err := flowExp.sendTemplateRecord(templateRec) + sentBytes, err := exp.sendTemplateRecord(templateRec) if err != nil { - return nil, fmt.Errorf("error while creating and sending template record through IPFIX process: %v", err) + return err } klog.V(2).Infof("Initialized flow exporter and sent %d bytes size of template record", sentBytes) - return flowExp, nil -} - -// Run enables to export flow records periodically at a given flow export frequency -func (exp *flowExporter) Run(stopCh <-chan struct{}, pollDone <-chan struct{}) { - klog.Infof("Start exporting IPFIX flow records") - ticker := time.NewTicker(time.Duration(exp.exportFrequency) * exp.pollInterval) - defer ticker.Stop() - - for { - select { - case <-stopCh: - exp.process.CloseConnToCollector() - break - case <-ticker.C: - // Waiting for expected number of pollDone signals from go routine(ConnectionStore.Run) is necessary because - // IPFIX collector computes throughput based on flow records received interval. Number of pollDone - // signals should be equal to export frequency before starting the export cycle. - for i := uint(0); i < exp.exportFrequency; i++ { - <-pollDone - } - err := exp.flowRecords.BuildFlowRecords() - if err != nil { - klog.Errorf("Error when building flow records: %v", err) - exp.process.CloseConnToCollector() - break - } - err = exp.sendFlowRecords() - if err != nil { - klog.Errorf("Error when sending flow records: %v", err) - exp.process.CloseConnToCollector() - break - } - } - } + return nil } func (exp *flowExporter) sendFlowRecords() error { diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 5d5f6516087..92ae0283d30 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -30,7 +30,6 @@ import ( const ( testTemplateID = 256 - testFlowPollInterval = time.Second testFlowExportFrequency = 12 antreaEnterpriseRegistry = 29305 ) @@ -46,7 +45,7 @@ func TestFlowExporter_sendTemplateRecord(t *testing.T) { mockIPFIXExpProc, nil, testFlowExportFrequency, - testFlowPollInterval, + 0, testTemplateID, } // Following consists of all elements that are in IANAInfoElements and AntreaInfoElements (globals) @@ -154,7 +153,7 @@ func TestFlowExporter_sendDataRecord(t *testing.T) { mockIPFIXExpProc, elemList, testFlowExportFrequency, - testFlowPollInterval, + 0, testTemplateID, } // Expect calls required diff --git a/pkg/agent/flowexporter/flowrecords/flow_records.go b/pkg/agent/flowexporter/flowrecords/flow_records.go index fc2f4970c77..cfc168a476f 100644 --- a/pkg/agent/flowexporter/flowrecords/flow_records.go +++ b/pkg/agent/flowexporter/flowrecords/flow_records.go @@ -15,8 +15,6 @@ package flowrecords import ( - "fmt" - "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" @@ -38,10 +36,8 @@ func NewFlowRecords(connStore *connections.ConnectionStore) *FlowRecords { // BuildFlowRecords builds the flow record map from connection map in connection store func (fr *FlowRecords) BuildFlowRecords() error { - err := fr.connStore.ForAllConnectionsDo(fr.addOrUpdateFlowRecord) - if err != nil { - return fmt.Errorf("error when iterating connection map: %v", err) - } + // fr.addOrUpdateFlowRecord method does not return any error, hence no error handling required. + fr.connStore.ForAllConnectionsDo(fr.addOrUpdateFlowRecord) klog.V(2).Infof("No. of flow records built: %d", len(fr.recordsMap)) return nil } diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go index 3a059c03bd4..e8e658d5be0 100644 --- a/pkg/util/ip/ip.go +++ b/pkg/util/ip/ip.go @@ -19,7 +19,6 @@ import ( "fmt" "net" "sort" - "strings" "github.com/vmware-tanzu/antrea/pkg/apis/networking/v1beta1" ) @@ -29,15 +28,6 @@ const ( v6BitLen = 8 * net.IPv6len ) -// Following map is for converting protocol name (string) to protocol identifier -var protocols = map[string]uint8{ - "icmp": 1, - "igmp": 2, - "tcp": 6, - "udp": 17, - "ipv6-icmp": 58, -} - // This function takes in one allow CIDR and multiple except CIDRs and gives diff CIDRs // in allowCIDR eliminating except CIDRs. It currently supports only IPv4. except CIDR input // can be changed. @@ -156,14 +146,3 @@ func NetIPNetToIPNet(ipNet *net.IPNet) *v1beta1.IPNet { prefix, _ := ipNet.Mask.Size() return &v1beta1.IPNet{IP: v1beta1.IPAddress(ipNet.IP), PrefixLength: int32(prefix)} } - -// LookupProtocolMap return protocol identifier given protocol name -func LookupProtocolMap(name string) (uint8, error) { - name = strings.TrimSpace(name) - lowerCaseStr := strings.ToLower(name) - proto, found := protocols[lowerCaseStr] - if !found { - return 0, fmt.Errorf("unknown IP protocol specified: %s", name) - } - return proto, nil -} diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index 382954524dc..8eb824573cb 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -469,7 +469,7 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware-tanzu/octant v0.13.1 h1:hz4JDnAA7xDkFjF4VEbt5SrSRrG26FCxKXXBGapf6Nc= github.com/vmware-tanzu/octant v0.13.1/go.mod h1:4q+wrV4tmUwAdMjvYOujSTtZbE4+zm0n5mb7FjvN0I0= -github.com/vmware/go-ipfix v0.0.0-20200715175325-6ade358dcb5f/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= +github.com/vmware/go-ipfix v0.0.0-20200808032647-11daf237d1dc/go.mod h1:8suqePBGCX20vEh/4/ekuRjX4BsZ2zYWcD22NpAWHVU= github.com/wenyingd/ofnet v0.0.0-20200601065543-2c7a62482f16/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 93186622699..83f945138a3 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -328,6 +328,20 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { return fmt.Errorf("error when restarting antrea-agent Pod: %v", err) } + // Just to be safe disabling the FlowExporter feature for subsequent tests. + configMap, err = data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + return fmt.Errorf("failed to get ConfigMap: %v", err) + } + + antreaAgentConf, _ = configMap.Data["antrea-agent.conf"] + antreaAgentConf = strings.Replace(antreaAgentConf, " FlowExporter: true", " FlowExporter: false", 1) + configMap.Data["antrea-agent.conf"] = antreaAgentConf + + if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err) + } + return nil } From 0ef571abe551ac2dd75c4628f430d3eabcdf7175 Mon Sep 17 00:00:00 2001 From: Srikar Tati Date: Mon, 10 Aug 2020 17:08:52 -0700 Subject: [PATCH 15/15] Addreseed the comments --- cmd/antrea-agent/agent.go | 2 +- .../flowexporter/connections/conntrack.go | 3 +- .../flowexporter/connections/conntrack_ovs.go | 6 +- pkg/agent/flowexporter/exporter/exporter.go | 63 ++++++++++++------- test/e2e/flowexporter_test.go | 2 +- test/e2e/framework.go | 17 +---- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index b1d7fb88468..2d1d64a1459 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -250,7 +250,7 @@ func run(o *Options) error { flowExporter := exporter.NewFlowExporter( flowrecords.NewFlowRecords(connStore), o.config.FlowExportFrequency) - go wait.Until(func() { flowExporter.CheckAndDoExport(o.flowCollector, pollDone) }, o.pollInterval, stopCh) + go wait.Until(func() { flowExporter.Export(o.flowCollector, stopCh, pollDone) }, 0, stopCh) } <-stopCh diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index 593836c33ed..270e56e3817 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -25,7 +25,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/ovs/ovsctl" ) -// InitializeConnTrackDumper initialize the ConnTrackDumper interface for different OS and datapath types. +// InitializeConnTrackDumper initializes the ConnTrackDumper interface for different OS and datapath types. func InitializeConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, ovsctlClient ovsctl.OVSCtlClient, ovsDatapathType string) ConnTrackDumper { var connTrackDumper ConnTrackDumper if ovsDatapathType == ovsconfig.OVSDatapathSystem { @@ -60,6 +60,7 @@ func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.Node // Conntrack flows will be different for Pod-to-Service flows w/ Antrea-proxy. This implementation will be simpler, when the // Antrea proxy is supported. if serviceCIDR.Contains(srcIP) || serviceCIDR.Contains(dstIP) { + klog.V(4).Infof("Detected a flow with Cluster IP :%v", conn) continue } filteredConns = append(filteredConns, conn) diff --git a/pkg/agent/flowexporter/connections/conntrack_ovs.go b/pkg/agent/flowexporter/connections/conntrack_ovs.go index 38cd6be2c0d..deb770f09e6 100644 --- a/pkg/agent/flowexporter/connections/conntrack_ovs.go +++ b/pkg/agent/flowexporter/connections/conntrack_ovs.go @@ -65,7 +65,7 @@ func (ct *connTrackOvsCtl) DumpFlows(zoneFilter uint16) ([]*flowexporter.Connect } filteredConns := filterAntreaConns(conns, ct.nodeConfig, ct.serviceCIDR, zoneFilter) - klog.V(2).Infof("Flow exporter considered flows: %d", len(filteredConns)) + klog.V(2).Infof("FlowExporter considered flows: %d", len(filteredConns)) return filteredConns, nil } @@ -83,14 +83,14 @@ func (ct *connTrackOvsCtl) ovsAppctlDumpConnections(zoneFilter uint16) ([]*flowe for _, flow := range outputFlow { conn, err := flowStringToAntreaConnection(flow, zoneFilter) if err != nil { - klog.Warningf("Ignoring the flow from conntrack dump due to the error: %v", err) + klog.V(4).Infof("Ignoring the flow from conntrack dump due to parsing error: %v", err) continue } if conn != nil { antreaConns = append(antreaConns, conn) } } - klog.V(2).Infof("Finished dumping -- total no. of flows in conntrack: %d", len(antreaConns)) + klog.V(2).Infof("FlowExporter considered flows in conntrack: %d", len(antreaConns)) return antreaConns, nil } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 612189bdf62..d986f24321f 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -91,34 +91,49 @@ func NewFlowExporter(records *flowrecords.FlowRecords, exportFrequency uint) *fl } } -// CheckAndDoExport enables us to export flow records periodically at a given flow export frequency. -func (exp *flowExporter) CheckAndDoExport(collector net.Addr, pollDone chan struct{}) { - // Number of pollDone signals received or poll cycles should be equal to export frequency before starting the export cycle. - // This is necessary because IPFIX collector computes throughput based on flow records received interval. - <-pollDone - exp.pollCycle++ - if exp.pollCycle%exp.exportFrequency == 0 { - if exp.process == nil { - err := exp.initFlowExporter(collector) - if err != nil { - klog.Errorf("Error when initializing flow exporter: %v", err) - return +// DoExport enables us to export flow records periodically at a given flow export frequency. +func (exp *flowExporter) Export(collector net.Addr, stopCh <-chan struct{}, pollDone <-chan struct{}) { + for { + select { + case <-stopCh: + return + case <-pollDone: + // Number of pollDone signals received or poll cycles should be equal to export frequency before starting + // the export cycle. This is necessary because IPFIX collector computes throughput based on flow records received interval. + exp.pollCycle++ + if exp.pollCycle%exp.exportFrequency == 0 { + // Retry to connect to IPFIX collector if the exporting process gets reset + if exp.process == nil { + err := exp.initFlowExporter(collector) + if err != nil { + klog.Errorf("Error when initializing flow exporter: %v", err) + // There could be other errors while initializing flow exporter other than connecting to IPFIX collector, + // therefore closing the connection and resetting the process. + if exp.process != nil { + exp.process.CloseConnToCollector() + exp.process = nil + } + return + } + } + // Build and send flow records to IPFIX collector. + exp.flowRecords.BuildFlowRecords() + err := exp.sendFlowRecords() + if err != nil { + klog.Errorf("Error when sending flow records: %v", err) + // If there is an error when sending flow records because of intermittent connectivity, we reset the connection + // to IPFIX collector and retry in the next export cycle to reinitialize the connection and send flow records. + exp.process.CloseConnToCollector() + exp.process = nil + return + } + + exp.pollCycle = 0 + klog.V(2).Infof("Successfully exported IPFIX flow records") } } - exp.flowRecords.BuildFlowRecords() - err := exp.sendFlowRecords() - if err != nil { - klog.Errorf("Error when sending flow records: %v", err) - // If there is an error when sending flow records because of intermittent connectivity, we reset the connection - // to IPFIX collector and retry in the next export cycle to reinitialize the connection and send flow records. - exp.process.CloseConnToCollector() - exp.process = nil - } - exp.pollCycle = 0 - klog.V(2).Infof("Successfully exported IPFIX flow records") } - return } func (exp *flowExporter) initFlowExporter(collector net.Addr) error { diff --git a/test/e2e/flowexporter_test.go b/test/e2e/flowexporter_test.go index b195afa8276..96fbd822cc9 100644 --- a/test/e2e/flowexporter_test.go +++ b/test/e2e/flowexporter_test.go @@ -62,7 +62,7 @@ func TestFlowExporter(t *testing.T) { rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs ipfix-collector -n antrea-test")) if err != nil || rc != 0 { - t.Fatalf("error when getting logs %v, rc: %v", err, rc) + t.Fatalf("Error when getting logs %v, rc: %v", err, rc) } /* Parse through IPFIX collector output. Sample output (with truncated fields) is given below: diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 83f945138a3..c344eddab76 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -311,12 +311,11 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { } antreaAgentConf, _ := configMap.Data["antrea-agent.conf"] - antreaAgentConf = strings.Replace(antreaAgentConf, "# FlowExporter: false", " FlowExporter: true", 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "# FlowExporter: false", " FlowExporter: true", 1) antreaAgentConf = strings.Replace(antreaAgentConf, "#flowCollectorAddr: \"\"", fmt.Sprintf("flowCollectorAddr: \"%s\"", ipfixCollector), 1) antreaAgentConf = strings.Replace(antreaAgentConf, "#flowPollInterval: \"5s\"", "flowPollInterval: \"1s\"", 1) antreaAgentConf = strings.Replace(antreaAgentConf, "#flowExportFrequency: 12", "flowExportFrequency: 5", 1) configMap.Data["antrea-agent.conf"] = antreaAgentConf - if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err) } @@ -328,20 +327,6 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { return fmt.Errorf("error when restarting antrea-agent Pod: %v", err) } - // Just to be safe disabling the FlowExporter feature for subsequent tests. - configMap, err = data.GetAntreaConfigMap(antreaNamespace) - if err != nil { - return fmt.Errorf("failed to get ConfigMap: %v", err) - } - - antreaAgentConf, _ = configMap.Data["antrea-agent.conf"] - antreaAgentConf = strings.Replace(antreaAgentConf, " FlowExporter: true", " FlowExporter: false", 1) - configMap.Data["antrea-agent.conf"] = antreaAgentConf - - if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err) - } - return nil }